From 4ab5ded6c5e3be80aabafcb72b6e9a26793d4fde Mon Sep 17 00:00:00 2001 From: Eric Rodrigues Pires Date: Thu, 25 Dec 2025 09:57:40 -0300 Subject: [PATCH] Use `yoke` for zero-copy on duperq --- Cargo.lock | 4 ++- Cargo.toml | 1 + duper/CHANGELOG.md | 6 +++++ duper/Cargo.toml | 4 ++- duper/src/ast.rs | 1 + duperq/CHANGELOG.md | 6 +++++ duperq/Cargo.toml | 3 ++- duperq/src/formatter.rs | 6 +++-- duperq/src/main.rs | 42 +++++++++++++++++++++++++----- duperq/src/processor.rs | 39 +++++++++++++++++---------- duperq/src/query.rs | 19 +++++++++----- duperq/tests/integration/common.rs | 13 ++++++--- flake.lock | 23 +++++++++++++++- flake.nix | 17 ++++++++++-- 14 files changed, 145 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6546e9..677a53e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1004,7 +1004,7 @@ checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" [[package]] name = "duper" -version = "0.6.0" +version = "0.6.1" dependencies = [ "ariadne", "base64", @@ -1018,6 +1018,7 @@ dependencies = [ "serde", "serde_core", "unicode-general-category", + "yoke", ] [[package]] @@ -1131,6 +1132,7 @@ dependencies = [ "regex", "smol", "temporal_rs", + "yoke", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 51a6720..f9f15d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ tree-sitter = { version = "^0.25", features = ["wasm"] } tree-sitter-duper = { version = "^0.1", path = "tree-sitter-duper" } uuid = { version = "^1", features = ["serde"] } wasm-bindgen = "=0.2.100" +yoke = "^0.8" [profile.release] lto = true diff --git a/duper/CHANGELOG.md b/duper/CHANGELOG.md index 6226fa1..32ad6c1 100644 --- a/duper/CHANGELOG.md +++ b/duper/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### Added + +- Add `yoke::Yokeable` derive to `DuperValue` (gated behind the `yoke` feature). + ## 0.6.0 (2025-12-23) ### Changed diff --git a/duper/Cargo.toml b/duper/Cargo.toml index 55d2b71..2ab6cb5 100644 --- a/duper/Cargo.toml +++ b/duper/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "duper" description = "The format that's super!" -version = "0.6.0" +version = "0.6.1" edition = "2024" rust-version = "1.88" license = "MIT" @@ -16,6 +16,7 @@ keywords = ["duper", "parser", "serialization"] default = [] ansi = ["dep:owo-colors"] serde = ["dep:serde_core"] +yoke = ["dep:yoke"] [dependencies] ariadne = { version = "0.6", features = ["auto-color"] } @@ -28,6 +29,7 @@ owo-colors = { version = "4", optional = true } ryu = "1" serde_core = { workspace = true, optional = true } unicode-general-category = "1" +yoke = { workspace = true, optional = true } [dev-dependencies] insta = { workspace = true } diff --git a/duper/src/ast.rs b/duper/src/ast.rs index 17d47df..f9ec38b 100644 --- a/duper/src/ast.rs +++ b/duper/src/ast.rs @@ -21,6 +21,7 @@ pub type DuperFloat = decorum::R64; pub struct DuperIdentifier<'a>(pub(crate) Cow<'a, str>); /// A Duper value. +#[cfg_attr(feature = "yoke", derive(yoke::Yokeable))] #[derive(Debug, Clone)] pub enum DuperValue<'a> { /// An object: `{...}` diff --git a/duperq/CHANGELOG.md b/duperq/CHANGELOG.md index 535beca..2a885ca 100644 --- a/duperq/CHANGELOG.md +++ b/duperq/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### Changed + +- Use `yoke` to prevent unnecessary cloning of Duper values. + ## 0.1.2 (2025-12-23) ### Changed diff --git a/duperq/Cargo.toml b/duperq/Cargo.toml index df126d5..c229ca6 100644 --- a/duperq/Cargo.toml +++ b/duperq/Cargo.toml @@ -18,12 +18,13 @@ async-trait = "0.1.89" chumsky = { workspace = true } clap = { workspace = true } color-eyre = { version = "0.6.5", default-features = false } -duper = { workspace = true, features = ["serde"] } +duper = { workspace = true, features = ["serde", "yoke"] } futures = { workspace = true } num_cpus = "1.17.0" regex = { workspace = true } smol = "2.0.2" temporal_rs = { workspace = true } +yoke = { workspace = true } [dev-dependencies] glob = "0.3.3" diff --git a/duperq/src/formatter.rs b/duperq/src/formatter.rs index 173adce..f351472 100644 --- a/duperq/src/formatter.rs +++ b/duperq/src/formatter.rs @@ -8,6 +8,7 @@ use duper::{ }, visitor::DuperVisitor, }; +use yoke::Yoke; use crate::{accessor::DuperAccessor, types::DuperType}; @@ -35,13 +36,14 @@ impl Formatter { } } - pub(crate) fn format(&mut self, value: DuperValue<'static>) -> String { + pub(crate) fn format(&mut self, value: Yoke, String>) -> String { let mut buf = String::new(); + let inner = value.get(); for atom in &self.atoms { match atom { FormatterAtom::Fixed(fixed) => buf.push_str(fixed), FormatterAtom::Dynamic(duper_accessor, typ) => { - match duper_accessor.access(&value).next() { + match duper_accessor.access(inner).next() { Some(value) => { if let Some(typ) = typ { if let Some(value) = typ.cast(value) { diff --git a/duperq/src/main.rs b/duperq/src/main.rs index 59c3959..fe36773 100644 --- a/duperq/src/main.rs +++ b/duperq/src/main.rs @@ -11,6 +11,7 @@ use smol::{ }; use duperq::query; +use yoke::Yoke; #[derive(Parser)] #[command(version, about, long_about = None)] @@ -68,12 +69,21 @@ fn main() -> color_eyre::Result<()> { let stdin = BufReader::new(Unblock::new(std::io::stdin())); let mut lines = stdin.lines(); while let Some(Ok(line)) = lines.next().await { - match DuperParser::parse_duper_trunk(&line) { - Ok(trunk) => sink.process(trunk.static_clone()).await, - Err(errors) => { + match Yoke::try_attach_to_cart(line, |input| { + DuperParser::parse_duper_trunk(input).map_err(move |err| { + ( + err.into_iter() + .map(|rich| rich.into_owned()) + .collect::>(), + input.to_string(), + ) + }) + }) { + Ok(yoke) => sink.process(yoke).await, + Err((errors, string)) => { if !cli.disable_stderr && let Ok(parse_error) = - DuperParser::prettify_error(&line, &errors, None) + DuperParser::prettify_error(&string, &errors, None) { let _ = stderr .write_all(eyre!(parse_error).to_string().as_bytes()) @@ -82,6 +92,15 @@ fn main() -> color_eyre::Result<()> { } } } + // if !cli.disable_stderr + // && let Ok(parse_error) = + // DuperParser::prettify_error(&line, &errors, None) + // { + // let _ = stderr + // .write_all(eyre!(parse_error).to_string().as_bytes()) + // .await; + // let _ = stderr.flush().await; + // } } sink.close().await; })); @@ -124,9 +143,18 @@ fn main() -> color_eyre::Result<()> { tasks.push(executor.spawn(async move { while let Ok(input) = file_receiver.recv().await { match input { - Ok((pathbuf, string)) => match DuperParser::parse_duper_trunk(&string) { - Ok(trunk) => sink.process(trunk.static_clone()).await, - Err(errors) => { + Ok((pathbuf, string)) => match Yoke::try_attach_to_cart(string, |input| { + DuperParser::parse_duper_trunk(input).map_err(move |err| { + ( + err.into_iter() + .map(|rich| rich.into_owned()) + .collect::>(), + input.to_string(), + ) + }) + }) { + Ok(yoke) => sink.process(yoke).await, + Err((errors, string)) => { if !cli.disable_stderr && let Ok(parse_error) = DuperParser::prettify_error( &string, diff --git a/duperq/src/processor.rs b/duperq/src/processor.rs index 88b1836..44f871f 100644 --- a/duperq/src/processor.rs +++ b/duperq/src/processor.rs @@ -4,26 +4,27 @@ use smol::{ channel, io::{AsyncWrite, AsyncWriteExt}, }; +use yoke::Yoke; use crate::filter::DuperFilter; #[async_trait(?Send)] /// An opaque layer that processes a [`DuperValue`] asynchronously. pub trait Processor { - async fn process(&mut self, value: DuperValue<'static>); + async fn process(&mut self, value: Yoke, String>); async fn close(&mut self) {} } pub(crate) struct FilterProcessor { filter: Box, - sender: channel::Sender>, + sender: channel::Sender, String>>, is_open: bool, } impl FilterProcessor { pub(crate) fn new( - sender: channel::Sender>, + sender: channel::Sender, String>>, filter: Box, ) -> Self { Self { @@ -36,8 +37,9 @@ impl FilterProcessor { #[async_trait(?Send)] impl Processor for FilterProcessor { - async fn process(&mut self, value: DuperValue<'static>) { - if self.is_open && self.filter.filter(&value) && self.sender.send(value).await.is_err() { + async fn process(&mut self, value: Yoke, String>) { + if self.is_open && self.filter.filter(value.get()) && self.sender.send(value).await.is_err() + { self.is_open = false; } } @@ -45,18 +47,21 @@ impl Processor for FilterProcessor { pub(crate) struct TakeProcessor { available: usize, - sender: channel::Sender>, + sender: channel::Sender, String>>, } impl TakeProcessor { - pub(crate) fn new(sender: channel::Sender>, available: usize) -> Self { + pub(crate) fn new( + sender: channel::Sender, String>>, + available: usize, + ) -> Self { Self { sender, available } } } #[async_trait(?Send)] impl Processor for TakeProcessor { - async fn process(&mut self, value: DuperValue<'static>) { + async fn process(&mut self, value: Yoke, String>) { if self.available > 0 { if self.sender.send(value).await.is_err() { self.available = 0; @@ -72,12 +77,15 @@ impl Processor for TakeProcessor { pub(crate) struct SkipProcessor { to_skip: usize, - sender: channel::Sender>, + sender: channel::Sender, String>>, is_open: bool, } impl SkipProcessor { - pub(crate) fn new(sender: channel::Sender>, to_skip: usize) -> Self { + pub(crate) fn new( + sender: channel::Sender, String>>, + to_skip: usize, + ) -> Self { Self { sender, to_skip, @@ -88,7 +96,7 @@ impl SkipProcessor { #[async_trait(?Send)] impl Processor for SkipProcessor { - async fn process(&mut self, value: DuperValue<'static>) { + async fn process(&mut self, value: Yoke, String>) { if self.to_skip > 0 { self.to_skip = self.to_skip.saturating_sub(1); } else if self.is_open && self.sender.send(value).await.is_err() { @@ -99,11 +107,14 @@ impl Processor for SkipProcessor { pub(crate) struct OutputProcessor { output: O, - printer: Box) -> Vec>, + printer: Box, String>) -> Vec>, } impl OutputProcessor { - pub(crate) fn new(output: O, printer: Box) -> Vec>) -> Self { + pub(crate) fn new( + output: O, + printer: Box, String>) -> Vec>, + ) -> Self { Self { output, printer } } } @@ -113,7 +124,7 @@ impl Processor for OutputProcessor where O: AsyncWrite + Unpin + 'static, { - async fn process(&mut self, value: DuperValue<'static>) { + async fn process(&mut self, value: Yoke, String>) { self.output .write_all((self.printer)(value).as_ref()) .await diff --git a/duperq/src/query.rs b/duperq/src/query.rs index a440c57..1ac4226 100644 --- a/duperq/src/query.rs +++ b/duperq/src/query.rs @@ -5,6 +5,7 @@ use duper::{ parser::{identified_value, integer, object_key, quoted_string}, }; use smol::{channel, io::AsyncWrite}; +use yoke::Yoke; use crate::{ accessor::{ @@ -28,7 +29,7 @@ pub fn query<'a, O>() -> impl Parser< 'a, &'a str, ( - Vec>>>, + Vec, String>>>>, CreateProcessorFn, ), extra::Err>, @@ -43,7 +44,7 @@ where let mut ansi = Ansi::default(); Box::new(OutputProcessor::new( output, - Box::new(move |value| ansi.to_ansi(&value).unwrap_or_default()), + Box::new(move |value| ansi.to_ansi(value.get()).unwrap_or_default()), )) as Box }) as CreateProcessorFn }), @@ -52,7 +53,7 @@ where let mut pretty_printer = PrettyPrinter::default(); Box::new(OutputProcessor::new( output, - Box::new(move |value| pretty_printer.pretty_print(&value).into_bytes()), + Box::new(move |value| pretty_printer.pretty_print(value.get()).into_bytes()), )) as Box }) as CreateProcessorFn }), @@ -63,7 +64,7 @@ where just("filter").padded().ignore_then(filter()).map(|filter| { Box::new(move |sender| { Box::new(FilterProcessor::new(sender, filter)) as Box - }) as CreateProcessorFn>> + }) as CreateProcessorFn, String>>> }), just("take") .padded() @@ -73,7 +74,9 @@ where Ok(Box::new(move |sender| { Box::new(TakeProcessor::new(sender, take as usize)) as Box }) - as CreateProcessorFn>>) + as CreateProcessorFn< + channel::Sender, String>>, + >) } else { Err(Rich::custom( span, @@ -89,7 +92,9 @@ where Ok(Box::new(move |sender| { Box::new(SkipProcessor::new(sender, skip as usize)) as Box }) - as CreateProcessorFn>>) + as CreateProcessorFn< + channel::Sender, String>>, + >) } else { Err(Rich::custom(span, "skip parameter must be positive")) } @@ -109,7 +114,7 @@ where let mut serializer = Serializer::default(); Box::new(OutputProcessor::new( output, - Box::new(move |value| serializer.serialize(&value).into_bytes()), + Box::new(move |value| serializer.serialize(value.get()).into_bytes()), )) as Box }) as CreateProcessorFn }) diff --git a/duperq/tests/integration/common.rs b/duperq/tests/integration/common.rs index 38d6db8..b5e990b 100644 --- a/duperq/tests/integration/common.rs +++ b/duperq/tests/integration/common.rs @@ -4,18 +4,25 @@ use piper::{Reader, pipe}; use smol::LocalExecutor; use duperq::query; +use yoke::Yoke; -pub(crate) fn parse_duper_values(values: &[&'static str]) -> Vec> { +pub(crate) fn parse_duper_values( + values: &[&'static str], +) -> Vec, String>> { values .into_iter() - .map(|input| DuperParser::parse_duper_trunk(input).unwrap()) + .map(|input| { + Yoke::attach_to_cart(input.to_string(), |input| { + DuperParser::parse_duper_trunk(input).unwrap() + }) + }) .collect() } pub(crate) fn get_query_output_reader( executor: &LocalExecutor<'_>, duperq_query: &str, - values: Vec>, + values: Vec, String>>, ) -> (Reader, impl Future) { let (reader, writer) = pipe(32_768); let (pipeline_fns, output) = query().parse(duperq_query).unwrap(); diff --git a/flake.lock b/flake.lock index 8beefa6..98b7127 100644 --- a/flake.lock +++ b/flake.lock @@ -52,7 +52,28 @@ "inputs": { "crane": "crane", "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1766630657, + "narHash": "sha256-wW15buPGU29v0XuAmDkc30+d5j4Tmg/V8AkpHH+hDWY=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "3bf67c5e473f29ca79ff15904f3072d87cf6d087", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" } }, "systems": { diff --git a/flake.nix b/flake.nix index d7ea6c9..759e8b6 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,10 @@ nixpkgs.url = "nixpkgs/nixpkgs-unstable"; crane.url = "github:ipetkov/crane"; flake-utils.url = "github:numtide/flake-utils"; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = @@ -13,16 +17,25 @@ nixpkgs, crane, flake-utils, + rust-overlay, ... }: flake-utils.lib.eachDefaultSystem ( system: let - pkgs = nixpkgs.legacyPackages.${system}; + rustChannel = "stable"; + rustVersion = "latest"; + + pkgs = import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; inherit (pkgs) lib; - craneLib = crane.mkLib pkgs; + craneLib = (crane.mkLib pkgs).overrideToolchain ( + pkgs: pkgs.rust-bin.${rustChannel}.${rustVersion}.default + ); cargo-rail = craneLib.buildPackage rec { pname = "cargo-rail"; -- 2.51.2