diff --git a/apple/whistlegraph/Sources/LiveTranscription.swift b/apple/whistlegraph/Sources/LiveTranscription.swift new file mode 100644 index 0000000000..464c48f79e --- /dev/null +++ b/apple/whistlegraph/Sources/LiveTranscription.swift @@ -0,0 +1,114 @@ +import Foundation + +/// Only a short-lived transcription credential reaches the phone. Microphone +/// chunks are sent during the take; committing ends the turn, not starts it. +@MainActor final class LiveTranscription { + var onText: (String, Bool) -> Void = { _, _ in } + var onFailure: () -> Void = {} + private var socket: URLSessionWebSocketTask? + private var receiver: Task? + private var sender: Task? + private var opening: Task? + private var pending = Data() + private var ready = false, ended = false, committed = false, closed = false + private var text = "", receivedBytes = 0 + + func start(token: @escaping () async throws -> String?) { + opening = Task { [weak self] in + do { + guard let self, let bearer = try await token(), !self.closed else { return } + var request = URLRequest(url: URL(string: "https://help.aesthetic.computer/api/aesel/transcription-session")!) + request.httpMethod = "POST"; request.timeoutInterval = 10 + request.setValue("Bearer " + bearer, forHTTPHeaderField: "Authorization") + let (data, response) = try await URLSession.shared.data(for: request) + guard !Task.isCancelled, !self.closed else { return } + guard (response as? HTTPURLResponse)?.statusCode == 200, + let value = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let secret = value["value"] as? String else { throw URLError(.badServerResponse) } + var connection = URLRequest(url: URL(string: "wss://api.openai.com/v1/realtime?intent=transcription")!) + connection.setValue("Bearer " + secret, forHTTPHeaderField: "Authorization") + let socket = URLSession.shared.webSocketTask(with: connection) + self.socket = socket; socket.resume() + self.receiver = Task { [weak self] in + do { while !Task.isCancelled, let self, !self.closed { try await self.receive(socket.receive()) } } + catch { if !Task.isCancelled { self?.fail() } } + } + } catch { if !Task.isCancelled { self?.fail() } } + } + } + + func append(_ data: Data) { + guard !closed, !ended else { return } + receivedBytes += data.count + guard receivedBytes <= 46 * 24_000 * 2 else { fail(); return } + pending.append(data); drain() + } + func finish() { ended = true; drain() } + private func send(_ value: [String: Any]) async throws { + guard let socket else { throw URLError(.notConnectedToInternet) } + let data = try JSONSerialization.data(withJSONObject: value) + try await socket.send(.string(String(decoding: data, as: UTF8.self))) + } + private func drain() { + guard ready, !closed, sender == nil else { return } + sender = Task { [weak self] in + guard let self else { return } + defer { self.sender = nil } + do { + while !Task.isCancelled, !self.closed { + if self.pending.count >= 4800 || (self.ended && !self.pending.isEmpty) { + let size = min(4800, self.pending.count), chunk = self.pending.prefix(size) + self.pending.removeFirst(size) + try await self.send(["type": "input_audio_buffer.append", "audio": chunk.base64EncodedString()]) + } else { + if self.ended && !self.committed && self.receivedBytes >= 4800 { + self.committed = true + try await self.send(["type": "input_audio_buffer.commit"]) + } + break + } + } + } catch { if !Task.isCancelled { self.fail() } } + } + } + private func receive(_ message: URLSessionWebSocketTask.Message) throws { + let data: Data + switch message { case .data(let d): data = d; case .string(let s): data = Data(s.utf8); @unknown default: return } + guard let event = try JSONSerialization.jsonObject(with: data) as? [String: Any], let type = event["type"] as? String else { return } + switch type { + case "session.created", "session.updated": ready = true; drain() + case "conversation.item.input_audio_transcription.delta": + text += event["delta"] as? String ?? ""; onText(text.trimmingCharacters(in: .whitespacesAndNewlines), false) + case "conversation.item.input_audio_transcription.completed": + text = event["transcript"] as? String ?? text; onText(text.trimmingCharacters(in: .whitespacesAndNewlines), true) + case "error", "conversation.item.input_audio_transcription.failed": fail() + default: break + } + } + private func fail() { guard !closed else { return }; close(); onFailure() } + func close() { + closed = true; opening?.cancel(); receiver?.cancel(); sender?.cancel() + socket?.cancel(with: .normalClosure, reason: nil); socket = nil; pending.removeAll() + } +} + +/// Stateful resampling keeps chunk boundaries off the performance clock. +final class SpeechPCM { + private var position = 0.0, offset = 0, last = 0.0 + func append(_ samples: [Double], rate: Double) -> Data { + guard rate > 0, !samples.isEmpty else { return Data() } + var data = Data(); data.reserveCapacity(samples.count * 2) + let end = offset + samples.count - 1 + while position <= Double(end) { + let lower = Int(floor(position)), fraction = position - Double(lower) + let a = lower < offset ? last : samples[lower - offset] + let b = fraction == 0 ? a : samples[lower + 1 - offset] + let sample = a + (b - a) * fraction + let value = Int16(max(-32768, min(32767, (sample.isFinite ? sample : 0) * 32767))) + let bits = UInt16(bitPattern: value) + data.append(UInt8(bits & 255)); data.append(UInt8(bits >> 8)) + position += rate / 24000 + } + offset += samples.count; last = samples.last!; return data + } +} diff --git a/apple/whistlegraph/Sources/MusicalInput.swift b/apple/whistlegraph/Sources/MusicalInput.swift index 4d90270bb5..93bb7bffa9 100644 --- a/apple/whistlegraph/Sources/MusicalInput.swift +++ b/apple/whistlegraph/Sources/MusicalInput.swift @@ -6,6 +6,8 @@ import AVFoundation final class MusicalInput { private let queue = DispatchQueue(label: "computer.walkieware.sound", qos: .userInitiated) private let recording = UtteranceRecording() + private let speechPCM = SpeechPCM() + var onPCM: ((Data) -> Void)? private var carry: [Double] = [] private var offset = 0 private var decimationSum = 0.0 @@ -26,6 +28,7 @@ final class MusicalInput { queue.async { [self] in guard frames.count < 1500 else { return } recording.append(samples, rate: originalRate) + onPCM?(speechPCM.append(samples, rate: originalRate)) rate = sampleRate // Keep decimation phase across input buffers; average before sampling. for sample in samples { diff --git a/apple/whistlegraph/Sources/RecordedTranscription.swift b/apple/whistlegraph/Sources/RecordedTranscription.swift new file mode 100644 index 0000000000..0b088184b2 --- /dev/null +++ b/apple/whistlegraph/Sources/RecordedTranscription.swift @@ -0,0 +1,52 @@ +import Foundation +import AVFoundation + +struct TimedTranscript: Decodable { + struct Word: Decodable { let text: String; let atMs: Double; let durationMs: Double } + let transcript: String + let words: [Word] + var timeline: [[String: Any]] { words.map { ["text": $0.text, "atMs": $0.atMs, "durationMs": $0.durationMs] } } +} + +enum RecordedTranscription { + static func recover(_ id: String, token: String) async throws -> TimedTranscript { + let audio = try await Task.detached(priority: .utility) { try wav(id) }.value + try Task.checkCancellation() + var request = URLRequest(url: URL(string: "https://help.aesthetic.computer/api/aesel/transcribe")!) + request.httpMethod = "POST"; request.timeoutInterval = 15 + request.setValue("Bearer " + token, forHTTPHeaderField: "Authorization") + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = try JSONSerialization.data(withJSONObject: ["audio": audio.base64EncodedString()]) + let (data, response) = try await URLSession.shared.data(for: request) + guard (response as? HTTPURLResponse)?.statusCode == 200 else { throw URLError(.badServerResponse) } + return try JSONDecoder().decode(TimedTranscript.self, from: data) + } + static func wav(_ id: String) throws -> Data { + guard let url = UtteranceRecording.url(id) else { throw URLError(.badURL) } + return try wav(url: url) + } + static func wav(url: URL) throws -> Data { + let file = try AVAudioFile(forReading: url), rate = file.processingFormat.sampleRate + guard file.length > 0, Double(file.length) / rate <= 46 else { throw URLError(.dataLengthExceedsMaximum) } + let buffer = AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: 16384)! + var samples: [Float] = []; samples.reserveCapacity(Int(file.length)) + while file.framePosition < file.length { + try file.read(into: buffer) + guard buffer.frameLength > 0, let channel = buffer.floatChannelData?[0] else { throw URLError(.cannotDecodeContentData) } + samples.append(contentsOf: UnsafeBufferPointer(start: channel, count: Int(buffer.frameLength))) + } + let count = Int(Double(samples.count) * 16000 / rate) + var data = Data() + func u32(_ n: UInt32) { var value = n.littleEndian; withUnsafeBytes(of: &value) { data.append(contentsOf: $0) } } + func u16(_ n: UInt16) { var value = n.littleEndian; withUnsafeBytes(of: &value) { data.append(contentsOf: $0) } } + data.append(Data("RIFF".utf8)); u32(UInt32(36 + count * 2)); data.append(Data("WAVEfmt ".utf8)) + u32(16); u16(1); u16(1); u32(16000); u32(32000); u16(2); u16(16) + data.append(Data("data".utf8)); u32(UInt32(count * 2)) + for i in 0.. Bool = { false } + var speechToken: () async throws -> String? = { nil } var onEvent: (_ kind: String, _ text: String, _ id: String) -> Void = { _, _, _ in } /// Latest microphone energy, for the live waveform. var onLevel: (_ rms: Double) -> Void = { _ in } @@ -50,6 +51,12 @@ import Speech private var replayTask: Task? private var finishing: Task? private var deadline: Task? + private var live: LiveTranscription? + private var liveText = "", liveFinal = false, recognitionGraceExpired = false + private var finalSound: [String: Any]? + private var alignment: Task? + private var alignmentPending = false + private var aligned: TimedTranscript? private func emit(_ kind: String, text: String = "", id: String? = nil) { if kind == "listening" { listeningSince = Date() } @@ -63,6 +70,27 @@ import Speech if NativeScreenFixture.enabled && NativeScreenFixture.mode == "gestures" { emit("listening"); return } #endif musicalInput = MusicalInput() + if !AudioBenchmark.enabled { + let stream = LiveTranscription(); live = stream + stream.onText = { [weak self] text, final in + guard let self, self.turn == id else { return } + self.liveText = text; self.liveFinal = final + self.emit("partial", text: text) + if final && self.audioEnded { self.deliver() } + } + stream.onFailure = { [weak self] in + guard let self, self.turn == id else { return } + self.liveFinal = true + if !self.latest.isEmpty { self.emit("partial", text: self.latest) } + self.deliver() + } + // This starts before microphone permission/session setup. Initial + // chunks queue until the short-lived socket is ready. + stream.start(token: speechToken) + musicalInput.onPCM = { [weak stream] data in + DispatchQueue.main.async { stream?.append(data) } + } + } musicalInput.onUpdate = { [weak self] pitch, rms in Task { @MainActor in guard let self, self.turn == id, self.held else { return } @@ -124,10 +152,11 @@ import Speech Task { @MainActor in guard let self, self.turn == id else { return } if let segments { self.wordSegments = Array(segments.prefix(256)) } - if let text { self.latest = text; self.emit("partial", text: text) } + if let text { self.latest = text; if self.liveText.isEmpty { self.emit("partial", text: text) } } if isFinal { self.recognitionFinal = true; AudioBenchmark.mark("recognitionFinal") } if self.audioEnded && isFinal { self.deliver() } else if self.audioEnded && message != nil && self.latest.isEmpty { + self.recognitionFinal = true self.deliver() // Sound remains useful when speech finds no words. } } @@ -172,37 +201,59 @@ import Speech self.stopAudio() self.audioEnded = true AudioBenchmark.mark("audioDrained") - if self.recognitionFinal { self.deliver(); return } + let sound = await withCheckedContinuation { continuation in self.musicalInput.finish { continuation.resume(returning: $0) } } + guard !Task.isCancelled, self.turn == id else { return } + // MusicalInput's serial queue has drained all PCM callbacks before + // this continuation, so commit follows the last recorded sample. + await withCheckedContinuation { continuation in + DispatchQueue.main.async { self.live?.finish(); continuation.resume() } + } + guard !Task.isCancelled, self.turn == id else { return } + self.finalSound = sound + if !AudioBenchmark.enabled, let recording = sound["recordingID"] as? String, + self.maxDuration > 8 || self.latest.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + self.alignmentPending = true + self.alignment = Task { [weak self] in + guard let self else { return } + var result: TimedTranscript? + do { if let token = try await self.speechToken() { result = try await RecordedTranscription.recover(recording, token: token) } } + catch { /* Live/on-device words and the original recording survive a network failure. */ } + guard !Task.isCancelled, self.turn == id else { return } + self.aligned = result; self.alignmentPending = false; self.deliver() + } + } + self.deliver() // endAudio lets recognition consume every queued buffer and finalize. // If it never does, the partial transcript is still the person's // words: send what was heard rather than throw the utterance away. try? await Task.sleep(for: .seconds(3)) guard !Task.isCancelled, self.turn == id, !self.delivering else { return } + self.recognitionGraceExpired = true AudioBenchmark.mark("recognitionTimedOut", fields: ["recognizedCharacters": self.latest.count]) self.deliver() } } private func deliver() { - guard !delivering, !turn.isEmpty else { return } + guard !delivering, !turn.isEmpty, audioEnded, let sound = finalSound, !alignmentPending, + aligned != nil || liveFinal || recognitionFinal || recognitionGraceExpired else { return } delivering = true - let text = latest.trimmingCharacters(in: .whitespacesAndNewlines) - let id = turn, words = wordSegments + let recovered = aligned?.transcript.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let text = !recovered.isEmpty ? recovered : (!liveText.isEmpty ? liveText : latest).trimmingCharacters(in: .whitespacesAndNewlines) + // Stream arrival times are not word timestamps. Only use measured + // alignment from the same transcript; never invent timing for deltas. + let words = !recovered.isEmpty ? aligned!.timeline : (text == latest.trimmingCharacters(in: .whitespacesAndNewlines) ? wordSegments : []) + let id = turn stopAudio() - musicalInput.finish { [weak self] sound in - Task { @MainActor in - guard let self, self.turn == id else { return } - guard !text.isEmpty || (sound["audibleMs"] as? Double ?? 0) >= 150 || self.hasVisualInput() else { - self.fail("I didn’t hear words or a sound. Hold to try again."); return - } - let value: [String: Any] = ["transcript":text,"words":words,"sound":sound] - guard let data = try? JSONSerialization.data(withJSONObject:value), let json = String(data:data,encoding:.utf8) else { self.fail("Could not read sound input."); return } - AudioBenchmark.checkTranscript(text) - AudioBenchmark.mark("soundSubmitted", fields: value) - self.cancel() - self.emit("mixedFinal", text:json, id:id) - } + guard !text.isEmpty || (sound["audibleMs"] as? Double ?? 0) >= 150 || hasVisualInput() else { + fail("I didn’t hear words or a sound. Hold to try again."); return } + let value: [String: Any] = ["transcript":text,"words":words,"sound":sound] + guard let data = try? JSONSerialization.data(withJSONObject:value), let json = String(data:data,encoding:.utf8) else { fail("Could not read sound input."); return } + AudioBenchmark.checkTranscript(text) + AudioBenchmark.mark("soundSubmitted", fields: value) + cancel() + emit("mixedFinal", text:json, id:id) } private func fail(_ message: String) { @@ -213,6 +264,8 @@ import Speech } func cancel() { + live?.close(); live = nil; alignment?.cancel(); alignment = nil + liveText = ""; liveFinal = false; finalSound = nil; aligned = nil; alignmentPending = false; recognitionGraceExpired = false turn = ""; held = false; listeningSince = nil; maxDuration = 8 replayTask?.cancel(); replayTask = nil finishing?.cancel(); finishing = nil diff --git a/apple/whistlegraph/Sources/WhistlegraphApp.swift b/apple/whistlegraph/Sources/WhistlegraphApp.swift index 7cfc0ccaca..1da214fc26 100644 --- a/apple/whistlegraph/Sources/WhistlegraphApp.swift +++ b/apple/whistlegraph/Sources/WhistlegraphApp.swift @@ -308,6 +308,10 @@ final class WhistlegraphSession: NSObject, ObservableObject, WKScriptMessageHand override init() { super.init() capture.hasVisualInput = { [weak self] in self?.drawing.hasInk == true } + capture.speechToken = { [weak self] in + guard let self, self.snapshot.handle == "jeffrey" else { return nil } + return try await self.account.token() + } capture.onEvent = { [weak self] kind, text, id in self?.emit(kind, text: text, id: id) } capture.onLevel = { [weak self] rms in guard let self else { return }; self.microphoneLevels = Array(self.microphoneLevels.dropFirst()) + [rms] } capture.onReplayRelease = { [weak self] in self?.webView?.evaluateJavaScript("voiceEnd()", completionHandler: nil) } diff --git a/apple/whistlegraph/Tests/SpeechCaptureCheck.swift b/apple/whistlegraph/Tests/SpeechCaptureCheck.swift new file mode 100644 index 0000000000..8a89aaaf93 --- /dev/null +++ b/apple/whistlegraph/Tests/SpeechCaptureCheck.swift @@ -0,0 +1,32 @@ +import Foundation +import AVFoundation + +@main struct SpeechCaptureCheck { + static func main() throws { + for rate in [16000.0, 24000, 44100, 48000] { + let samples = (0..