From 4de0ef872b05328ca5154ea9e20c4e9fdc60bd69 Mon Sep 17 00:00:00 2001 From: Graham Barber Date: Thu, 28 Aug 2025 20:39:49 -0700 Subject: [PATCH] add proof-of-concept STT --- src-tauri/Cargo.lock | 342 ++++++++++++++++++++++++++++- src-tauri/Cargo.toml | 5 + src-tauri/src/cartesia/client.rs | 46 ++++ src-tauri/src/cartesia/commands.rs | 13 ++ src-tauri/src/cartesia/mod.rs | 3 + src-tauri/src/cartesia/stt.rs | 107 +++++++++ src-tauri/src/devices/input.rs | 72 +++++- src-tauri/src/lib.rs | 7 +- src-tauri/src/state.rs | 29 ++- src/routes/+page.svelte | 9 +- 10 files changed, 610 insertions(+), 23 deletions(-) create mode 100644 src-tauri/src/cartesia/client.rs create mode 100644 src-tauri/src/cartesia/commands.rs create mode 100644 src-tauri/src/cartesia/mod.rs create mode 100644 src-tauri/src/cartesia/stt.rs diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index c6b81d4..4ab9123 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -543,6 +543,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -566,9 +576,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1" dependencies = [ "bitflags 2.9.3", - "core-foundation", + "core-foundation 0.10.1", "core-graphics-types", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -579,7 +589,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" dependencies = [ "bitflags 2.9.3", - "core-foundation", + "core-foundation 0.10.1", "libc", ] @@ -738,12 +748,131 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "dasp" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7381b67da416b639690ac77c73b86a7b5e64a29e31d1f75fb3b1102301ef355a" +dependencies = [ + "dasp_envelope", + "dasp_frame", + "dasp_interpolate", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", + "dasp_signal", + "dasp_slice", + "dasp_window", +] + +[[package]] +name = "dasp_envelope" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ec617ce7016f101a87fe85ed44180839744265fae73bb4aa43e7ece1b7668b6" +dependencies = [ + "dasp_frame", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", +] + +[[package]] +name = "dasp_frame" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a3937f5fe2135702897535c8d4a5553f8b116f76c1529088797f2eee7c5cd6" +dependencies = [ + "dasp_sample", +] + +[[package]] +name = "dasp_interpolate" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc975a6563bb7ca7ec0a6c784ead49983a21c24835b0bc96eea11ee407c7486" +dependencies = [ + "dasp_frame", + "dasp_ring_buffer", + "dasp_sample", +] + +[[package]] +name = "dasp_peak" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf88559d79c21f3d8523d91250c397f9a15b5fc72fbb3f87fdb0a37b79915bf" +dependencies = [ + "dasp_frame", + "dasp_sample", +] + +[[package]] +name = "dasp_ring_buffer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07d79e19b89618a543c4adec9c5a347fe378a19041699b3278e616e387511ea1" + +[[package]] +name = "dasp_rms" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6c5dcb30b7e5014486e2822537ea2beae50b19722ffe2ed7549ab03774575aa" +dependencies = [ + "dasp_frame", + "dasp_ring_buffer", + "dasp_sample", +] + [[package]] name = "dasp_sample" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "dasp_signal" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa1ab7d01689c6ed4eae3d38fe1cea08cba761573fbd2d592528d55b421077e7" +dependencies = [ + "dasp_envelope", + "dasp_frame", + "dasp_interpolate", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", + "dasp_window", +] + +[[package]] +name = "dasp_slice" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e1c7335d58e7baedafa516cb361360ff38d6f4d3f9d9d5ee2a2fc8e27178fa1" +dependencies = [ + "dasp_frame", + "dasp_sample", +] + +[[package]] +name = "dasp_window" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ded7b88821d2ce4e8b842c9f1c86ac911891ab89443cc1de750cae764c5076" +dependencies = [ + "dasp_sample", +] + +[[package]] +name = "data-encoding" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" + [[package]] name = "deranged" version = "0.4.0" @@ -1019,6 +1148,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -1026,7 +1164,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -1040,6 +1178,12 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -2096,6 +2240,8 @@ name = "miwiwi" version = "0.1.0" dependencies = [ "cpal", + "dasp", + "futures-util", "serde", "serde_json", "tauri", @@ -2104,6 +2250,9 @@ dependencies = [ "tauri-plugin-positioner", "tauri-plugin-single-instance", "tauri-plugin-window-state", + "tokio", + "tokio-tungstenite", + "tungstenite", ] [[package]] @@ -2127,6 +2276,23 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndk" version = "0.9.0" @@ -2531,6 +2697,50 @@ dependencies = [ "pathdiff", ] +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags 2.9.3", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -2962,6 +3172,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -2982,6 +3202,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + [[package]] name = "rand_core" version = "0.5.1" @@ -3000,6 +3230,15 @@ dependencies = [ "getrandom 0.2.16", ] +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -3177,6 +3416,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "schemars" version = "0.8.22" @@ -3234,6 +3482,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.9.3", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "selectors" version = "0.24.0" @@ -3420,6 +3691,17 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -3495,7 +3777,7 @@ dependencies = [ "bytemuck", "cfg_aliases", "core-graphics", - "foreign-types", + "foreign-types 0.5.0", "js-sys", "log", "objc2 0.5.2", @@ -3651,7 +3933,7 @@ checksum = "4daa814018fecdfb977b59a094df4bd43b42e8e21f88fddfc05807e6f46efaaf" dependencies = [ "bitflags 2.9.3", "block2 0.6.1", - "core-foundation", + "core-foundation 0.10.1", "core-graphics", "crossbeam-channel", "dispatch", @@ -4121,6 +4403,30 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" +dependencies = [ + "futures-util", + "log", + "native-tls", + "tokio", + "tokio-native-tls", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.16" @@ -4333,6 +4639,24 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "native-tls", + "rand 0.9.2", + "sha1", + "thiserror 2.0.16", + "utf-8", +] + [[package]] name = "typeid" version = "1.0.3" @@ -4457,6 +4781,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version-compare" version = "0.2.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index b18ef58..54ec28d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -23,6 +23,11 @@ tauri-plugin-opener = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" cpal = "0.16.0" +tokio-tungstenite = { version = "0.27.0", features = ["native-tls"] } +tungstenite = "0.27.0" +futures-util = "0.3.31" +tokio = "1.47.1" +dasp = { version = "0.11.0", features = ["all"] } [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies] tauri-plugin-positioner = "2" diff --git a/src-tauri/src/cartesia/client.rs b/src-tauri/src/cartesia/client.rs new file mode 100644 index 0000000..890867f --- /dev/null +++ b/src-tauri/src/cartesia/client.rs @@ -0,0 +1,46 @@ +use std::sync::Arc; +use tauri::{async_runtime::RwLock, Url}; +use tokio::net::TcpStream; +use tokio_tungstenite::{ + connect_async, tungstenite::client::IntoClientRequest, MaybeTlsStream, WebSocketStream, +}; + +#[derive(Clone)] +pub struct CartesiaClient { + api_key: Arc>, +} + +impl CartesiaClient { + pub fn new(api_key: String) -> Self { + Self { + api_key: Arc::new(RwLock::new(api_key)), + } + } + + pub async fn open_stt_connection(&self) -> WebSocketStream> { + let api_key = self.api_key.read().await.clone(); + let mut request = Url::parse_with_params( + "wss://api.cartesia.ai/stt/websocket", + &[ + ("model", "ink-whisper"), + ("encoding", "pcm_s16le"), + ("sample_rate", "16000"), + ], + ) + .expect("failed to parse STT connection URL") + .as_str() + .into_client_request() + .expect("failed to instantiate STT WebSocket request"); + + let headers = request.headers_mut(); + + headers.insert("X-API-Key", api_key.parse().unwrap()); + headers.insert("Cartesia-Version", "2025-04-16".parse().unwrap()); + + let (stream, _) = connect_async(request) + .await + .expect("failed to open STT websocket connection"); + + stream + } +} diff --git a/src-tauri/src/cartesia/commands.rs b/src-tauri/src/cartesia/commands.rs new file mode 100644 index 0000000..0c6216d --- /dev/null +++ b/src-tauri/src/cartesia/commands.rs @@ -0,0 +1,13 @@ +use crate::state::AppState; + +#[tauri::command] +pub async fn start_stt(state: tauri::State<'_, AppState>) -> Result<(), ()> { + state.stt_manager.transcribe().await; + Ok(()) +} + +#[tauri::command] +pub async fn stop_stt(state: tauri::State<'_, AppState>) -> Result<(), ()> { + state.stt_manager.stop_transcription().await; + Ok(()) +} diff --git a/src-tauri/src/cartesia/mod.rs b/src-tauri/src/cartesia/mod.rs new file mode 100644 index 0000000..b71c998 --- /dev/null +++ b/src-tauri/src/cartesia/mod.rs @@ -0,0 +1,3 @@ +pub mod client; +pub mod commands; +pub mod stt; diff --git a/src-tauri/src/cartesia/stt.rs b/src-tauri/src/cartesia/stt.rs new file mode 100644 index 0000000..da12e22 --- /dev/null +++ b/src-tauri/src/cartesia/stt.rs @@ -0,0 +1,107 @@ +use super::client::CartesiaClient; +use crate::devices::input::InputDeviceManager; +use dasp::Signal; +use futures_util::{ + future::join, + stream::{SplitSink, SplitStream}, + SinkExt, StreamExt, +}; +use std::sync::Arc; +use tauri::async_runtime::Mutex; +use tokio::net::TcpStream; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; +use tungstenite::{Bytes, Message}; + +pub struct SttManager { + client: Arc, + input: Arc, + is_transcribing: Arc>, +} + +impl SttManager { + pub fn new(client: Arc, input: Arc) -> Self { + Self { + client, + input, + is_transcribing: Arc::new(Mutex::new(false)), + } + } + + async fn handle_messages( + &self, + mut reader: SplitStream>>, + ) { + println!("handling messages"); + while let Some(message) = reader.next().await { + let is_transcribing = self.is_transcribing.lock().await; + + if *is_transcribing { + println!("message from STT: {:?}", message) + } else { + println!("finished transcribing; breaking message handling"); + break; + } + } + } + + async fn send_frames( + &self, + signal: impl Signal, + mut writer: SplitSink>, Message>, + ) { + let mut buffer = Vec::with_capacity(3200); + println!("sending frames"); + for frame in signal.until_exhausted() { + let is_transcribing = self.is_transcribing.lock().await; + + if *is_transcribing { + buffer.push(frame.to_le_bytes()); + + if buffer.len() == buffer.capacity() { + println!("sending {} frames", buffer.len()); + writer + .send(tungstenite::Message::Binary(Bytes::from_iter( + buffer.iter().flat_map(|f| *f), + ))) + .await + .expect("failed to send binary frame message to STT"); + + buffer.clear(); + } + } else { + println!("stopping frame send"); + break; + } + } + } + + /// Begins transcribing text via Cartesia. Blocks until `stop_transcription` is called + pub async fn transcribe(&self) { + let stream = self.client.open_stt_connection().await; + let input = self.input.start_listening().await; + let (writer, reader) = stream.split(); + + { + let mut is_transcribing = self.is_transcribing.lock().await; + *is_transcribing = true; + } + + // Handle incoming messages + let read = self.handle_messages(reader); + let write = self.send_frames(input, writer); + + join(read, write).await; + + stream.close(); + } + + /// Terminates the microphone signal and halts the transcription processes + pub async fn stop_transcription(&self) { + { + let mut is_transcribing = self.is_transcribing.lock().await; + *is_transcribing = false; + } + + self.input.stop_listening().await; + } +} diff --git a/src-tauri/src/devices/input.rs b/src-tauri/src/devices/input.rs index 5e94652..4bfd692 100644 --- a/src-tauri/src/devices/input.rs +++ b/src-tauri/src/devices/input.rs @@ -1,10 +1,26 @@ use crate::devices::types::AudioDeviceError; -use cpal::traits::HostTrait; -use cpal::{default_host, Device, Host}; +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{default_host, Device, SampleFormat, Stream}; +use dasp::interpolate::sinc::Sinc; +use dasp::ring_buffer; +use dasp::signal::{self, Signal}; +use dasp::Frame; +use std::sync::{mpsc::channel, Arc}; +use tauri::async_runtime::Mutex; pub struct InputDeviceManager { - host: Host, device: Device, + stream: Arc>>, +} + +fn resample( + signal: impl Signal, + source_hz: f64, + target_hz: f64, +) -> impl Signal { + let ring_buffer = ring_buffer::Fixed::from([i16::EQUILIBRIUM; 10]); + let sinc = Sinc::new(ring_buffer); + signal.from_hz_to_hz(sinc, source_hz, target_hz) } impl InputDeviceManager { @@ -13,12 +29,56 @@ impl InputDeviceManager { let device = host .default_input_device() .ok_or(AudioDeviceError::NoDevicesFound)?; - let manager = InputDeviceManager { host, device }; + let manager = InputDeviceManager { + device, + stream: Arc::new(Mutex::new(None)), + }; Ok(manager) } - pub fn get_device(&self) -> &Device { - &self.device + pub async fn start_listening(&self) -> impl Signal { + let config = self + .device + .supported_input_configs() + .expect("failed to get supported input configs") + .find(|c| c.sample_format() == SampleFormat::I16) + .expect("failed to find PCM s16 config") + .with_max_sample_rate(); + let channels = config.channels() as usize; + let (tx, rx) = channel::>(); + let stream = self + .device + .build_input_stream( + &config.config(), + move |data: &[i16], _: &cpal::InputCallbackInfo| { + let mono = data + .chunks(channels) + .map(|frame| (frame.iter().sum::() / channels as i16).to_le()); + + let _ = tx + .send(mono.collect()) + .inspect(|_| println!("successfully sent frame")) + .inspect_err(|e| eprintln!("failed to send frame: {e}")); + }, + |err| eprintln!("encountered error streaming input: {}", err), + None, + ) + .expect("failed to create input stream"); + stream.play().expect("failed to start input stream"); + + let samples = rx.into_iter().flat_map(|x| x.into_iter()); + let signal = signal::from_iter(samples); + let signal = resample(signal, config.sample_rate().0 as f64, 16000.0); + + let mut s = self.stream.lock().await; + *s = Some(stream); + + signal + } + + pub async fn stop_listening(&self) { + let mut s = self.stream.lock().await; + *s = None; } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 4aeea6d..d529654 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,8 +1,10 @@ use crate::state::AppState; +use tauri::async_runtime::Mutex; use tauri::{Manager, WebviewUrl, WebviewWindowBuilder}; use tauri_plugin_positioner::{Position, WindowExt}; use tauri_plugin_window_state::{StateFlags, WindowExt as StateWindowExt}; +mod cartesia; mod devices; mod state; @@ -62,7 +64,10 @@ pub fn run() { Ok(()) }) - .invoke_handler(tauri::generate_handler![greet]) + .invoke_handler(tauri::generate_handler![ + cartesia::commands::start_stt, + cartesia::commands::stop_stt, + ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); } diff --git a/src-tauri/src/state.rs b/src-tauri/src/state.rs index c25e793..660e40a 100644 --- a/src-tauri/src/state.rs +++ b/src-tauri/src/state.rs @@ -1,22 +1,33 @@ -use std::sync::{Arc, RwLock}; +use std::sync::Arc; -use crate::devices::{ - input::InputDeviceManager, output::OutputDeviceManager, types::AudioDeviceError, +use crate::{ + cartesia::{client::CartesiaClient, stt::SttManager}, + devices::{input::InputDeviceManager, output::OutputDeviceManager, types::AudioDeviceError}, }; pub struct AppState { - pub input_device_manager: Arc>, - pub output_device_manager: Arc>, + pub cartesia_client: Arc, + pub stt_manager: Arc, + pub input_device_manager: Arc, + pub output_device_manager: Arc, } impl AppState { pub fn new() -> Result { - let input_device_manager = InputDeviceManager::new()?; - let output_device_manager = OutputDeviceManager::new()?; + let input_device_manager = Arc::new(InputDeviceManager::new()?); + let output_device_manager = Arc::new(OutputDeviceManager::new()?); + + let cartesia_client = Arc::new(CartesiaClient::new("TODO".into())); + let stt_manager = Arc::new(SttManager::new( + cartesia_client.clone(), + input_device_manager.clone(), + )); Ok(AppState { - input_device_manager: Arc::new(RwLock::new(input_device_manager)), - output_device_manager: Arc::new(RwLock::new(output_device_manager)), + input_device_manager, + output_device_manager, + cartesia_client, + stt_manager, }) } } diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index b751a09..7758e3d 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -1 +1,8 @@ -
miwiwi
\ No newline at end of file + + +
miwiwi
+ + + -- 2.51.2