From 880096d16834839480eac886474fcecc5439b7ff Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Thu, 1 Oct 2026 20:37:00 -0700 Subject: [PATCH] Interpret Chalk as contextual gestures for additive edits --- apple/whistlegraph/README.md | 2 +- apple/whistlegraph/Resources/Web/drawing-input.mjs | 7 ++++++- apple/whistlegraph/Resources/Web/inference-input.mjs | 2 +- apple/whistlegraph/Tests/drawing-input.test.mjs | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apple/whistlegraph/README.md b/apple/whistlegraph/README.md index 9a2d9be354..40f5d2929b 100644 --- a/apple/whistlegraph/README.md +++ b/apple/whistlegraph/README.md @@ -61,7 +61,7 @@ Both experiment browser checks use mock inference. They verify the contract, rep Each successful version retains its combined request: transcript, ordered normalized strokes, elapsed stroke timing, optional measured Pencil pressure, and existing sound/word measurements. Sound time zero is aligned to the drawing timeline. Finger pressure is unknown. Failed requests retain the draft and saved request for retry; a successful commit consumes only the matching draft revision. Unsubmitted native sketches are in memory; accepted attempts use the existing durable recovery journal. New/open piece discards the current sketch. -The existing model interprets vector coordinates and measured sound cues in one request. This is not image recognition or raw-audio inference, and gesture meaning remains uncertain. Drawing bypasses text-only local shortcuts so the marks cannot silently be ignored. Sampling caps the prompt at 320 points across at most 32 strokes (the native draft retains at most 1,200 points). Timings and endpoints survive sampling; fine detail can be lost. No separate inference endpoint or backend rollout is required. +The existing model interprets vector coordinates and measured sound cues in one request. Chalk is an instruction channel: the prompt asks the model to infer targeting, motion, emphasis, additions, or depicted content from words, timing, placement, and existing source. It favors focused additive edits and preserves the scene rather than automatically literalizing each mark. This is a prompting policy, not a trained gesture classifier or a measured recognition guarantee. The model receives source context, not a captured preview frame, so precise targeting of animated objects remains unresolved. This is not image recognition or raw-audio inference, and gesture meaning remains uncertain. Drawing bypasses text-only local shortcuts so the marks cannot silently be ignored. Sampling caps the prompt at 320 points across at most 32 strokes (the native draft retains at most 1,200 points). Timings and endpoints survive sampling; fine detail can be lost. No separate inference endpoint or backend rollout is required. ```sh node --test apple/whistlegraph/Tests/drawing-input.test.mjs diff --git a/apple/whistlegraph/Resources/Web/drawing-input.mjs b/apple/whistlegraph/Resources/Web/drawing-input.mjs index d4b19710c7..d182cddb05 100644 --- a/apple/whistlegraph/Resources/Web/drawing-input.mjs +++ b/apple/whistlegraph/Resources/Web/drawing-input.mjs @@ -39,5 +39,10 @@ export function withDrawing(request,value) { } export function drawingEvidence(value) { const {id,revision,schema,...drawing}=normalizeDrawing(value); - return '\nDRAWING REFERENCE: ordered strokes over the current preview. Each point is [x,y,elapsedMs,optionalPencilPressure]; x/y span 0..1000, origin top-left, y downward. Preserve aspect ratio. Time starts with the first mark; pen lifts separate strokes. Pressure (0..1000) exists only when measured with a Pencil; finger pressure is unknown. speechStartMs, when present, locates sound/word time zero on this same timeline. Use direction, pauses, speed changes, placement and nearby sound/word emphasis as clues to the intended edit. These are ambiguous gestures, not certain labels or commands. Follow explicit words, preserve existing scene context, and use the drawn form itself when its subject is unclear. Do not print these data or turn every sketch into a chart. With only a drawing, make a simple playable interpretation of its form.\n'+JSON.stringify(drawing); + return ` +DRAWING REFERENCE — CHALK GESTURES: ordered marks over the current preview, supplied as an instruction channel alongside speech, sound and the existing piece source. These are ambiguous gestures, not certain labels or commands. +Infer each mark's role in context: pointing or enclosing can identify a target; a sweep can suggest movement or a transformation; repeated marks can emphasize or multiply; marks in open space can propose an addition; a drawn form can also be content when the request supports that reading. These are possibilities, not fixed gesture bindings. A circle does not automatically mean "add a circle" or "select this". Read related strokes as a phrase, not unrelated objects. +Follow explicit words first. Use the current scene and the mark's location and timing to resolve "this", "here" and "like that". Preserve existing objects, behavior and composition; prefer a focused additive or local change. Do not replace the scene, erase objects or invent an unrelated subject merely because a gesture is ambiguous. Without words, use the strongest contextual evidence and make the smallest coherent change. In an empty scene, let placement, form and motion guide a simple starting composition without assuming every stroke depicts a literal object. +Each point is [x,y,elapsedMs,optionalPencilPressure]; x/y span 0..1000, origin top-left, y downward. Preserve aspect ratio. Time starts with the first mark; pen lifts separate strokes. Pressure (0..1000) exists only when measured with a Pencil; finger pressure is unknown. speechStartMs, when present, locates sound/word time zero on this same timeline. Use direction, pauses, speed changes and nearby sound/word emphasis as evidence, not proof of intent. The source is context, not a captured frame: do not claim a precise object hit or visual recognition you cannot establish, especially in a moving scene. Do not display the coordinate data, add annotation UI, or leave the chalk overlay in the piece unless requested. +`+JSON.stringify(drawing); } diff --git a/apple/whistlegraph/Resources/Web/inference-input.mjs b/apple/whistlegraph/Resources/Web/inference-input.mjs index 2c3e9a7791..7799d87885 100644 --- a/apple/whistlegraph/Resources/Web/inference-input.mjs +++ b/apple/whistlegraph/Resources/Web/inference-input.mjs @@ -8,7 +8,7 @@ export function inferenceRequest(request) { const input=inputData(request); if(!input)return request; const transcript=input.transcript.trim(); - let prompt=transcript|| (input.drawing?'Interpret this drawing.':'Create a playable interpretation of this nonverbal sound.'); + let prompt=transcript|| (input.drawing?'Infer the intended edit from these chalk gestures and the current piece.':'Create a playable interpretation of this nonverbal sound.'); // Drawing makes gesture/sound alignment relevant even for ordinary speech. if(input.sound&&(input.drawing||wantsSoundEvidence(transcript))){ const {recordingID,...sound}=input.sound; diff --git a/apple/whistlegraph/Tests/drawing-input.test.mjs b/apple/whistlegraph/Tests/drawing-input.test.mjs index 05ecd548ba..785dc13a0d 100644 --- a/apple/whistlegraph/Tests/drawing-input.test.mjs +++ b/apple/whistlegraph/Tests/drawing-input.test.mjs @@ -9,7 +9,7 @@ const sound={transcript:'make it move',words:[{text:'move',atMs:1500,durationMs: test('typed and drawing-only requests preserve a bounded vector attachment in one request',()=>{ const stored=withDrawing('make this bounce',sketch),parsed=inputData(stored); assert.equal(parsed.transcript,'make this bounce');assert.deepEqual(parsed.drawing.strokes,sketch.strokes); - assert.match(inferenceRequest(stored),/make this bounce/);assert.match(inferenceRequest(withDrawing('',sketch)),/Interpret this drawing/); + assert.match(inferenceRequest(stored),/make this bounce/);assert.match(inferenceRequest(withDrawing('',sketch)),/Infer the intended edit from these chalk gestures/); assert.doesNotMatch(inferenceRequest(stored),new RegExp(sketch.id)); assert.equal(withDrawing('hello',null),'hello'); }); -- 2.51.2