From 3df7031e0beca90ccd8fae641e3403dbaf263120 Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Thu, 1 Oct 2026 20:28:44 -0700 Subject: [PATCH] Let Chalk accompany held Talk with an independent finger --- apple/whistlegraph/README.md | 4 +- .../whistlegraph/Sources/TalkHoldInput.swift | 58 +++++++++++++++++ .../Sources/WhistlegraphDrawing.swift | 6 +- .../Sources/WhistlegraphScreen.swift | 65 ++++++++----------- apple/whistlegraph/project.yml | 2 +- 5 files changed, 92 insertions(+), 43 deletions(-) create mode 100644 apple/whistlegraph/Sources/TalkHoldInput.swift diff --git a/apple/whistlegraph/README.md b/apple/whistlegraph/README.md index ddb05952e4..9a2d9be354 100644 --- a/apple/whistlegraph/README.md +++ b/apple/whistlegraph/README.md @@ -55,9 +55,9 @@ node apple/whistlegraph/Tests/native-bridge.test.cjs --native-shell --checked-ed Both experiment browser checks use mock inference. They verify the contract, repair limit, rollback, render identity, and receipt fields without spending braincells. -## Drawing with words and sound +## Chalk with words and sound -**Draw** toggles an ink layer over the preview above the controls. The keyboard and microphone remain available. While Draw is on, Talk uses tap-to-record / tap-to-send (still capped at eight seconds), leaving a finger free to draw. Typed Enter sends text and the current sketch together; Send beside the drawing can submit it alone. Undo removes the last stroke; Clear discards the sketch. Turning Draw off returns touch interaction to the piece while retaining visible unsent marks. +**Chalk** toggles an ink layer over the preview above the controls. The keyboard and microphone remain available. Hold Talk with one finger and chalk over the preview with another, even when Chalk was off. Releasing Talk sends the combined request (still capped at eight seconds). The Chalk toggle keeps the layer available between recordings and while typing. Typed Enter sends text and the current sketch together; Send beside the drawing can submit it alone. Undo removes the last stroke; Clear discards the sketch. Turning Chalk off returns touch interaction to the piece while retaining visible unsent marks. Each successful version retains its combined request: transcript, ordered normalized strokes, elapsed stroke timing, optional measured Pencil pressure, and existing sound/word measurements. Sound time zero is aligned to the drawing timeline. Finger pressure is unknown. Failed requests retain the draft and saved request for retry; a successful commit consumes only the matching draft revision. Unsubmitted native sketches are in memory; accepted attempts use the existing durable recovery journal. New/open piece discards the current sketch. diff --git a/apple/whistlegraph/Sources/TalkHoldInput.swift b/apple/whistlegraph/Sources/TalkHoldInput.swift new file mode 100644 index 0000000000..17c46328e0 --- /dev/null +++ b/apple/whistlegraph/Sources/TalkHoldInput.swift @@ -0,0 +1,58 @@ +import SwiftUI +import UIKit + +// Track only the finger that starts on Talk. A second finger on the preview +// belongs to the chalk view and must not end or cancel this hold. +struct TalkHoldInput: UIViewRepresentable { + var enabled: Bool + var began: () -> Void + var ended: () -> Void + var cancelled: () -> Void + + func makeUIView(context: Context) -> TalkHoldView { + let view = TalkHoldView() + view.backgroundColor = .clear + view.isMultipleTouchEnabled = false + view.isExclusiveTouch = false + return view + } + + func updateUIView(_ view: TalkHoldView, context: Context) { + view.began = began; view.ended = ended; view.cancelled = cancelled + view.isUserInteractionEnabled = enabled + } + + static func dismantleUIView(_ view: TalkHoldView, coordinator: ()) { + view.cancelTracking() + } +} + +final class TalkHoldView: UIView { + var began: (() -> Void)? + var ended: (() -> Void)? + var cancelled: (() -> Void)? + private var finger: UITouch? + + override func touchesBegan(_ touches: Set, with event: UIEvent?) { + guard finger == nil, let touch = touches.first else { return } + finger = touch + began?() + } + + override func touchesEnded(_ touches: Set, with event: UIEvent?) { + guard let finger, touches.contains(finger) else { return } + self.finger = nil + ended?() + } + + override func touchesCancelled(_ touches: Set, with event: UIEvent?) { + guard let finger, touches.contains(finger) else { return } + cancelTracking() + } + + func cancelTracking() { + guard finger != nil else { return } + finger = nil + cancelled?() + } +} diff --git a/apple/whistlegraph/Sources/WhistlegraphDrawing.swift b/apple/whistlegraph/Sources/WhistlegraphDrawing.swift index 7d56f617a3..102f05e51e 100644 --- a/apple/whistlegraph/Sources/WhistlegraphDrawing.swift +++ b/apple/whistlegraph/Sources/WhistlegraphDrawing.swift @@ -48,9 +48,9 @@ struct DrawingPad: UIViewRepresentable { var interactive: Bool func makeUIView(context: Context) -> GestureInkView { let view = GestureInkView(); view.backgroundColor = .clear; view.isOpaque = false - view.isMultipleTouchEnabled = false; view.accessibilityIdentifier = "drawing-pad" - view.accessibilityLabel = "Drawing over the piece" - view.accessibilityHint = "Draw with a finger or Pencil while talking or typing." + view.isMultipleTouchEnabled = false; view.isExclusiveTouch = false; view.accessibilityIdentifier = "drawing-pad" + view.accessibilityLabel = "Chalk over the piece" + view.accessibilityHint = "Add chalk strokes with a finger or Pencil while talking or typing." return view } func updateUIView(_ view: GestureInkView, context: Context) { diff --git a/apple/whistlegraph/Sources/WhistlegraphScreen.swift b/apple/whistlegraph/Sources/WhistlegraphScreen.swift index 8970127001..99c7d183df 100644 --- a/apple/whistlegraph/Sources/WhistlegraphScreen.swift +++ b/apple/whistlegraph/Sources/WhistlegraphScreen.swift @@ -65,7 +65,6 @@ struct WhistlegraphScreen: View { @StateObject private var narrator = VersionNarrator() @Environment(\.scenePhase) private var scenePhase @State private var held = false - @GestureState private var touching = false @State private var showComposer = false @Environment(\.accessibilityReduceMotion) private var reduceMotion @Environment(\.colorScheme) private var colorScheme @@ -80,6 +79,7 @@ struct WhistlegraphScreen: View { private var theme: WhistlegraphTheme { WhistlegraphTheme(phase: themePhase, dark: colorScheme == .dark) } private var paper: Color { theme.foreground } private var accent: Color { theme.accent } + private var chalkActive: Bool { drawing.enabled || held || session.capturePhase == .opening || session.capturePhase == .recording } var body: some View { VStack(spacing: session.layout.spacing) { HStack { @@ -102,16 +102,10 @@ struct WhistlegraphScreen: View { .allowsHitTesting(session.snapshot.hasPreview && session.capturePhase == .idle) .accessibilityHidden(!session.snapshot.hasPreview) if !session.snapshot.hasPreview && !session.engineReady { ProgressView() } - if !drawing.enabled && (session.capturePhase == .recording || session.capturePhase == .opening) { - ScrollView { - Text(session.transcript.isEmpty ? (session.capturePhase == .opening ? "Opening microphone…" : "Listening…") : session.transcript) - .font(.custom("ComicRelief-Regular", size: 30, relativeTo: .title)).frame(maxWidth: .infinity, alignment: .leading).padding(18) - }.background(theme.background.opacity(0.95)) - } - if !narrator.isPlaying && (drawing.enabled || drawing.hasInk) { - DrawingPad(draft: drawing, interactive: drawing.enabled && canTalk) - .background(drawing.enabled ? Color.black.opacity(0.16) : Color.clear) - .allowsHitTesting(drawing.enabled && canTalk) + if !narrator.isPlaying && (chalkActive || drawing.hasInk) { + DrawingPad(draft: drawing, interactive: chalkActive && canTalk) + .background(chalkActive ? Color.black.opacity(0.16) : Color.clear) + .allowsHitTesting(chalkActive && canTalk) } if let failure = session.startupFailure { VStack(spacing: 12) { Text(failure); Button("Reload") { session.reloadWorkspace() } }.padding().background(.black.opacity(0.85)) @@ -128,24 +122,24 @@ struct WhistlegraphScreen: View { if !narrator.isPlaying { HStack(spacing: 18) { Button { ButtonSounds.play(.tick); drawing.enabled.toggle() } label: { - Label("Draw", systemImage: drawing.enabled ? "pencil.tip.crop.circle.fill" : "pencil.tip.crop.circle") + Label("Chalk", systemImage: drawing.enabled ? "pencil.tip.crop.circle.fill" : "pencil.tip.crop.circle") }.accessibilityIdentifier("draw-control").accessibilityValue(drawing.enabled ? "On" : "Off") .disabled(!canTalk) if drawing.hasInk { Button { drawing.undo() } label: { Image(systemName: "arrow.uturn.backward") } .accessibilityLabel("Undo stroke").accessibilityIdentifier("drawing-undo").disabled(!canTalk) Button { drawing.clear() } label: { Image(systemName: "trash") } - .accessibilityLabel("Clear drawing").accessibilityIdentifier("drawing-clear").disabled(!canTalk) + .accessibilityLabel("Clear chalk").accessibilityIdentifier("drawing-clear").disabled(!canTalk) } Spacer(minLength: 0) if drawing.hasInk && !showComposer { Button("Send") { ButtonSounds.play(.sent); session.command("ask", text: "") } - .accessibilityLabel("Send drawing").accessibilityIdentifier("drawing-send") + .accessibilityLabel("Send chalk").accessibilityIdentifier("drawing-send") .disabled(!canTalk || session.capturePhase != .idle) } }.font(.title3).buttonStyle(.plain).frame(minHeight: 44) - if drawing.full { Text("Drawing full · send or undo a stroke").font(.caption) } - if drawing.enabled && (session.capturePhase == .recording || session.capturePhase == .opening) { + if drawing.full { Text("Chalk full · send or undo a stroke").font(.caption) } + if session.capturePhase == .recording || session.capturePhase == .opening { Text(session.transcript.isEmpty ? "Listening…" : session.transcript) .font(.custom("ComicRelief-Regular", size: 20)).lineLimit(2) .frame(maxWidth: .infinity, alignment: .leading).allowsHitTesting(false) @@ -208,10 +202,8 @@ struct WhistlegraphScreen: View { let progress = session.captureStarted.map { min(1, max(0, context.date.timeIntervalSince($0) / 8)) } ?? 0 VStack(spacing: 10) { if session.capturePhase == .recording { - Text(drawing.enabled ? "Tap to send" : "Talk").font(.custom("ComicRelief-Bold", size: 30, relativeTo: .title2)) + Text("Talk").font(.custom("ComicRelief-Bold", size: 30, relativeTo: .title2)) MicrophoneWaveform(levels: session.microphoneLevels).frame(height: 22) - } else if drawing.enabled { - VStack(spacing: 4) { ShoutButtonLabel(); Text("Tap to talk").font(.caption) } } else { ShoutButtonLabel() } if session.capturePhase == .recording { ProgressView(value: progress).tint(theme.buttonInk) @@ -226,26 +218,25 @@ struct WhistlegraphScreen: View { .clipShape(RoundedRectangle(cornerRadius: 34, style: .continuous)) .overlay(RoundedRectangle(cornerRadius: 34, style: .continuous).strokeBorder(paper, lineWidth: 3)) .contentShape(RoundedRectangle(cornerRadius: 34, style: .continuous)) - .gesture(DragGesture(minimumDistance: 0).updating($touching) { _, state, _ in state = true }.onChanged { value in - guard !drawing.enabled else { return } - guard canTalk || held else { return } - guard !held, session.capturePhase == .idle else { return } - held = true; ButtonSounds.play(.press); session.beginHold() - }.onEnded { value in - if drawing.enabled { - if session.capturePhase == .recording { ButtonSounds.play(.release); session.endHold() } - else if canTalk && session.capturePhase == .idle { ButtonSounds.play(.press); session.beginHold() } - return - } - if held { ButtonSounds.play(.release); session.endHold() } - held = false - }) - .onChange(of: touching) { _, active in - if !active && held { held = false; session.cancelHold() } + .overlay { + TalkHoldInput(enabled: canTalk || held, began: { + guard canTalk, !held, session.capturePhase == .idle else { return } + held = true; ButtonSounds.play(.press); session.beginHold() + }, ended: { + guard held else { return } + held = false + if session.capturePhase == .opening || session.capturePhase == .recording { + ButtonSounds.play(.release); session.endHold() + } + }, cancelled: { + guard held else { return } + held = false + if session.capturePhase == .opening || session.capturePhase == .recording { session.cancelHold() } + }).accessibilityHidden(true) } .accessibilityElement(children: .ignore) .accessibilityLabel("Talk") - .accessibilityHint(drawing.enabled ? "Tap to record while drawing. Tap again to send, or send automatically after eight seconds." : "Hold to record, release to send. Up to eight seconds.") + .accessibilityHint("Hold to record and use another finger to chalk above. Release to send. Up to eight seconds.") .accessibilityIdentifier("talk-control") .accessibilityAddTraits(.isButton) .accessibilityAction(named: Text("Type a request")) { if canTalk { session.cancelHold(); ButtonSounds.play(.key); showComposer = true } } @@ -341,7 +332,7 @@ private struct VersionRow: View { var body: some View { HStack(spacing: 12) { ComicTitle(text: "v\(version.id)", size: textSize).frame(width: 48, alignment: .leading) - if version.hasDrawing == true { Image(systemName: "pencil.tip").accessibilityLabel("With drawing") } + if version.hasDrawing == true { Image(systemName: "pencil.tip").accessibilityLabel("With chalk") } Text(version.utterance).lineLimit(1).truncationMode(.tail) .frame(maxWidth: .infinity, alignment: .trailing) } diff --git a/apple/whistlegraph/project.yml b/apple/whistlegraph/project.yml index d5dfc3383d..6f34e9639b 100644 --- a/apple/whistlegraph/project.yml +++ b/apple/whistlegraph/project.yml @@ -9,7 +9,7 @@ settings: CODE_SIGN_STYLE: Automatic SWIFT_VERSION: "5.0" MARKETING_VERSION: "0.1.0" - CURRENT_PROJECT_VERSION: "88" + CURRENT_PROJECT_VERSION: "89" targets: Whistlegraph: type: application -- 2.51.2