diff --git a/Cargo.lock b/Cargo.lock index 86de4f7..2961b33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,86 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "anstream" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" + +[[package]] +name = "anstyle-parse" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "async-compression" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd", + "zstd-safe", +] + [[package]] name = "async-trait" version = "0.1.82" @@ -89,14 +169,14 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "axum" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", "axum-macros", - "base64 0.21.7", + "base64", "bytes", "futures-util", "http", @@ -127,9 +207,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", @@ -178,12 +258,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -217,6 +291,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -235,6 +330,23 @@ version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" + +[[package]] +name = "cc" +version = "1.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -276,12 +388,58 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clap" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + [[package]] name = "cobs" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" +[[package]] +name = "colorchoice" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -307,6 +465,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.1.3" @@ -519,6 +686,12 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "ff" version = "0.13.0" @@ -535,6 +708,16 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "flate2" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fluvio-wasm-timer" version = "0.2.5" @@ -714,6 +897,22 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "byteorder", + "num-traits", +] + [[package]] name = "heapless" version = "0.7.17" @@ -728,6 +927,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.9" @@ -880,6 +1085,16 @@ dependencies = [ "tower-service", ] +[[package]] +name = "indexmap" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "inout" version = "0.1.3" @@ -901,12 +1116,27 @@ dependencies = [ "web-sys", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.70" @@ -963,7 +1193,7 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" name = "meal-core" version = "0.1.0" dependencies = [ - "base64 0.22.1", + "base64", "console_error_panic_hook", "nanoid", "openmls", @@ -1042,6 +1272,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "object" version = "0.36.4" @@ -1274,6 +1513,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + [[package]] name = "poly1305" version = "0.8.0" @@ -1337,6 +1582,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pumpe" +version = "0.1.0" +dependencies = [ + "axum", + "camino", + "clap", + "http-body-util", + "thiserror", + "tokio", + "tower 0.5.1", + "tower-http", +] + [[package]] name = "quote" version = "1.0.37" @@ -1617,6 +1876,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -1686,6 +1951,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -1798,9 +2069,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.23.1" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" dependencies = [ "futures-util", "log", @@ -1844,9 +2115,13 @@ checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" dependencies = [ "futures-core", "futures-util", + "hdrhistogram", + "indexmap", "pin-project-lite", + "slab", "sync_wrapper 0.1.2", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -1858,8 +2133,10 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" dependencies = [ + "async-compression", "bitflags 2.6.0", "bytes", + "futures-core", "futures-util", "http", "http-body", @@ -1953,9 +2230,9 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" dependencies = [ "byteorder", "bytes", @@ -2006,6 +2283,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "valuable" version = "0.1.0" @@ -2248,3 +2531,31 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 5530fba..8487259 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["delivery-service", "core"] +members = ["delivery-service", "core", "pumpe"] resolver = "2" diff --git a/pumpe/Cargo.toml b/pumpe/Cargo.toml new file mode 100644 index 0000000..7afd29f --- /dev/null +++ b/pumpe/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "pumpe" +version = "0.1.0" +edition = "2021" + +[dependencies] +axum = { version = "0.7.7", features = ["macros"] } +camino = "1.1.9" +clap = { version = "4.5.18", features = ["derive"] } +http-body-util = "0.1.2" +# http = "1.1.0" +thiserror = "1.0.64" +tokio = { version = "1.40.0", features = ["full"] } +tower = { version = "0.5.1", features = ["full"] } +tower-http = { version = "0.6.1", features = ["compression-full", "fs"] } diff --git a/pumpe/README.md b/pumpe/README.md new file mode 100644 index 0000000..2e3900d --- /dev/null +++ b/pumpe/README.md @@ -0,0 +1,3 @@ +# An air compressor is just a pump + +A tool to precompress files to reduce download sizes. It uses the [tower-http compression module](https://docs.rs/tower-http/latest/tower_http/compression/index.html) to achieve the same compression results as if it would be compressed on the fly. diff --git a/pumpe/src/main.rs b/pumpe/src/main.rs new file mode 100644 index 0000000..a6a5cf5 --- /dev/null +++ b/pumpe/src/main.rs @@ -0,0 +1,155 @@ +use std::{ + collections::{HashSet, VecDeque}, + env, + fmt::format, + fs, + path::{Path, PathBuf}, + rc::Rc, + sync::Arc, +}; + +use axum::{ + body::{Body, Bytes}, + extract::{self, path, State}, + http::{self, header, request}, + response::IntoResponse, + routing::get, + Router, +}; +use camino::Utf8PathBuf; +use clap::{error::ErrorKind, CommandFactory, Parser}; +use http_body_util::BodyExt; +use thiserror::Error; +use tower::{ServiceBuilder, ServiceExt}; +use tower_http::{ + compression::{Compression, CompressionLayer}, + services::ServeDir, +}; + +#[derive(Parser, Debug)] +#[command()] +struct Arguments { + #[arg(num_args = 1)] + directory: Utf8PathBuf, +} + +#[derive(Error, Debug)] +enum PumpeError { + #[error(transparent)] + IoError(#[from] std::io::Error), +} + +fn visit_directory(path: Utf8PathBuf) -> Result, PumpeError> { + let mut files = Vec::new(); + let mut directory_queue = VecDeque::new(); + directory_queue.push_back(path.clone()); + + // Breadth first traversal + while let Some(path) = directory_queue.pop_front() { + for entry in path.read_dir_utf8()? { + let entry = entry?; + let path = entry.path().to_path_buf(); + if path.is_dir() { + directory_queue.push_back(path); + + continue; + } + + files.push(path); + } + } + + Ok(files.into()) +} + +#[tokio::main] +async fn main() -> Result<(), PumpeError> { + // Helpful: https://www.rustadventure.dev/introducing-clap/clap-v4/accepting-file-paths-as-arguments-in-clap + let directory = Arguments::parse().directory; + + if !directory.exists() { + let mut command = Arguments::command(); + command.error( + ErrorKind::ValueValidation, + format!("Path does not exist: {}", directory), + ); + } + + if !directory.is_dir() { + let mut command = Arguments::command(); + command.error( + ErrorKind::ValueValidation, + format!("Path is not a directory: {}", directory), + ); + } + + // List all files in directory + let files = visit_directory(directory.clone())?; + + //TODO use service builder + let app: Router = Router::new() + // .fallback_service(ServeDir::new(arguments.directory.as_path())) + .fallback_service(ServeDir::new(directory.clone())) + .layer(CompressionLayer::new()); + + // app.map_request() + // let request = http::Request::get("/app/dist/index.html") + // .header(header::CONTENT_ENCODING, "gzip") + // .body(Body::empty()) + // .unwrap(); + // let result = app.clone().oneshot(request).await; + // println!("Result: {:?}", result); + + // Strip source path as that is used as the root to serve files from + // for file in files.iter_mut() { + // *file = file.strip_prefix(&path).unwrap().to_path_buf(); + // } + + fn request(path: &str, encoding: &str) -> http::Request { + http::Request::get(path) + .header(header::ACCEPT_ENCODING, encoding) + .body(Body::empty()) + .unwrap() + } + + async fn write_response(response: http::Response, target: &Utf8PathBuf) { + let data = response.into_body().collect().await.unwrap().to_bytes(); + fs::write(target, data).unwrap(); + } + + async fn compress( + app: Router, + file: &Utf8PathBuf, + target_directory: impl AsRef, + (encoding_extension, encoding): (&str, &str), + ) { + // Request + let path = file.strip_prefix(target_directory).unwrap(); + let path = Utf8PathBuf::from("/").join(path); + let request = request(path.as_str(), encoding); + let response = app.clone().oneshot(request).await.unwrap(); + + // Write + let mut target = file.clone(); + let extension = target.extension().unwrap_or_default(); + target.set_extension(format!("{extension}.{encoding_extension}",)); + write_response(response, &target).await; + } + + for file in files.iter() { + // Detect if file is already compressed. This is not precise as it does not check the content but it is enough for our usecase + if file + .extension() + .is_some_and(|extension| matches!(extension, "br" | "gz" | "zst" | "zz")) + { + continue; + } + + compress(app.clone(), file, directory.as_path(), ("gz", "gzip")).await; + compress(app.clone(), file, directory.as_path(), ("br", "br")).await; + compress(app.clone(), file, directory.as_path(), ("zst", "zstd")).await; + compress(app.clone(), file, directory.as_path(), ("zz", "deflate")).await; + } + + Ok(()) +}