diff --git a/slab/menuband-win/menuband.c b/slab/menuband-win/menuband.c index 689ecb096f..9a304ee677 100644 --- a/slab/menuband-win/menuband.c +++ b/slab/menuband-win/menuband.c @@ -72,7 +72,7 @@ static CRITICAL_SECTION g_lock; static uint32_t g_seedCounter = 0x9E3779B9u; static double g_sampleRate = 48000.0; static const double g_masterGain = 0.5; -static volatile int g_program = 0; // GM program for new notes (Acoustic Grand) +static volatile int g_program = 78; // GM program for new notes: 79 Whistle (zero-based 78), as on the Mac static volatile LONG g_audioRunning = 0; static double midi_freq(int midi) { return 440.0 * pow(2.0, (midi - 69) / 12.0); } @@ -525,6 +525,7 @@ static HHOOK g_hook = NULL; static HWND g_strip = NULL; #define WM_TYPEKEY (WM_APP + 2) #define WM_TYPEESC (WM_APP + 3) +#define WM_OCTAVE (WM_APP + 4) static void build_keymap(void) { for (int i = 0; i < 256; i++) g_semitoneByVk[i] = -128; @@ -543,6 +544,10 @@ static LRESULT CALLBACK ll_keyboard(int code, WPARAM wp, LPARAM lp) { int down = (wp == WM_KEYDOWN || wp == WM_SYSKEYDOWN); if (!(k->flags & LLKHF_INJECTED) && k->vkCode < 256) { if (k->vkCode == VK_ESCAPE) { if (down) PostMessageW(g_strip, WM_TYPEESC, 0, 0); return 1; } + if (k->vkCode == VK_OEM_COMMA || k->vkCode == VK_OEM_PERIOD) { + if (down) PostMessageW(g_strip, WM_OCTAVE, k->vkCode == VK_OEM_PERIOD ? 1 : (WPARAM)-1, 0); + return 1; + } if (g_semitoneByVk[k->vkCode] != -128 && !(GetAsyncKeyState(VK_CONTROL) & 0x8000) && !(GetAsyncKeyState(VK_LWIN) & 0x8000)) { PostMessageW(g_strip, WM_TYPEKEY, k->vkCode, down); return 1; @@ -614,15 +619,30 @@ static void track(HWND hwnd, int force) { #define IDM_TYPING 101 #define IDM_AUTOSTART 102 #define IDM_PROGRAM_BASE 200 -#define IDM_MIDI_BASE 300 +#define IDM_MIDI_BASE 400 #define IDT_TRACK 1 -static const struct { const wchar_t *name; int program; } PROGRAMS[] = { - { L"Acoustic Grand Piano", 0 }, { L"Electric Piano", 4 }, { L"Harpsichord", 6 }, - { L"Vibraphone", 11 }, { L"Drawbar Organ", 16 }, { L"Nylon Guitar", 24 }, - { L"Electric Bass", 33 }, { L"Strings", 48 }, { L"Choir", 52 }, { L"Flute", 73 }, - { L"Square Lead", 80 }, { L"Warm Pad", 89 }, +// All 128 General MIDI programs, grouped in the 16 GM families for the menu. +static const wchar_t *GM_NAMES[128] = { + L"Acoustic Grand Piano", L"Bright Acoustic Piano", L"Electric Grand Piano", L"Honky-tonk Piano", L"Electric Piano 1", L"Electric Piano 2", L"Harpsichord", L"Clavinet", + L"Celesta", L"Glockenspiel", L"Music Box", L"Vibraphone", L"Marimba", L"Xylophone", L"Tubular Bells", L"Dulcimer", + L"Drawbar Organ", L"Percussive Organ", L"Rock Organ", L"Church Organ", L"Reed Organ", L"Accordion", L"Harmonica", L"Tango Accordion", + L"Nylon Guitar", L"Steel Guitar", L"Jazz Guitar", L"Clean Guitar", L"Muted Guitar", L"Overdriven Guitar", L"Distortion Guitar", L"Guitar Harmonics", + L"Acoustic Bass", L"Fingered Bass", L"Picked Bass", L"Fretless Bass", L"Slap Bass 1", L"Slap Bass 2", L"Synth Bass 1", L"Synth Bass 2", + L"Violin", L"Viola", L"Cello", L"Contrabass", L"Tremolo Strings", L"Pizzicato Strings", L"Orchestral Harp", L"Timpani", + L"String Ensemble 1", L"String Ensemble 2", L"Synth Strings 1", L"Synth Strings 2", L"Choir Aahs", L"Voice Oohs", L"Synth Voice", L"Orchestra Hit", + L"Trumpet", L"Trombone", L"Tuba", L"Muted Trumpet", L"French Horn", L"Brass Section", L"Synth Brass 1", L"Synth Brass 2", + L"Soprano Sax", L"Alto Sax", L"Tenor Sax", L"Baritone Sax", L"Oboe", L"English Horn", L"Bassoon", L"Clarinet", + L"Piccolo", L"Flute", L"Recorder", L"Pan Flute", L"Blown Bottle", L"Shakuhachi", L"Whistle", L"Ocarina", + L"Square Lead", L"Sawtooth Lead", L"Calliope Lead", L"Chiff Lead", L"Charang Lead", L"Voice Lead", L"Fifths Lead", L"Bass + Lead", + L"New Age Pad", L"Warm Pad", L"Polysynth Pad", L"Choir Pad", L"Bowed Pad", L"Metallic Pad", L"Halo Pad", L"Sweep Pad", + L"Rain", L"Soundtrack", L"Crystal", L"Atmosphere", L"Brightness", L"Goblins", L"Echoes", L"Sci-Fi", + L"Sitar", L"Banjo", L"Shamisen", L"Koto", L"Kalimba", L"Bagpipe", L"Fiddle", L"Shanai", + L"Tinkle Bell", L"Agogo", L"Steel Drums", L"Woodblock", L"Taiko Drum", L"Melodic Tom", L"Synth Drum", L"Reverse Cymbal", + L"Guitar Fret Noise", L"Breath Noise", L"Seashore", L"Bird Tweet", L"Telephone Ring", L"Helicopter", L"Applause", L"Gunshot", }; +static const wchar_t *GM_FAMILIES[16] = { L"Piano", L"Chromatic Percussion", L"Organ", L"Guitar", L"Bass", L"Strings", L"Ensemble", L"Brass", L"Reed", L"Pipe", L"Synth Lead", L"Synth Pad", L"Synth Effects", L"Ethnic", L"Percussive", L"Sound Effects" }; + static NOTIFYICONDATAW g_nid; @@ -646,8 +666,15 @@ static void show_menu(HWND hwnd) { HMENU menu = CreatePopupMenu(); AppendMenuW(menu, MF_STRING | (g_typing ? MF_CHECKED : 0), IDM_TYPING, L"Type to play (Esc leaves)"); AppendMenuW(menu, MF_SEPARATOR, 0, NULL); - for (int i = 0; i < (int)(sizeof PROGRAMS / sizeof PROGRAMS[0]); i++) - AppendMenuW(menu, MF_STRING | (PROGRAMS[i].program == g_program ? MF_CHECKED : 0), IDM_PROGRAM_BASE + i, PROGRAMS[i].name); + for (int f = 0; f < 16; f++) { + HMENU fam = CreatePopupMenu(); + for (int i = f * 8; i < f * 8 + 8; i++) { + wchar_t label[64]; swprintf(label, 64, L"%d %s", i + 1, GM_NAMES[i]); + AppendMenuW(fam, MF_STRING | (i == g_program ? MF_CHECKED : 0), IDM_PROGRAM_BASE + i, label); + } + wchar_t famLabel[64]; swprintf(famLabel, 64, L"%s%s", GM_FAMILIES[f], (g_program / 8 == f) ? L" \x2713" : L""); + AppendMenuW(menu, MF_POPUP, (UINT_PTR)fam, famLabel); + } AppendMenuW(menu, MF_SEPARATOR, 0, NULL); HMENU midi = CreatePopupMenu(); AppendMenuW(midi, MF_STRING | (g_midiDevice < 0 ? MF_CHECKED : 0), IDM_MIDI_BASE, L"Built-in synth"); @@ -676,6 +703,14 @@ static void mouse_up(HWND hwnd) { if (g_mouseMidi >= 0) { lit_off(hwnd, g_mouseMidi); g_mouseMidi = -1; } } +static void shift_octave(HWND hwnd, int dir) { + int next = g_firstMidi + 12 * dir; + if (next < 36 || next > 84) return; + mouse_up(hwnd); + release_typed(hwnd); + g_firstMidi = next; build_keys(); push_frame(hwnd); +} + static LRESULT CALLBACK wndproc(HWND hwnd, UINT msg, WPARAM wp, LPARAM lp) { switch (msg) { case WM_CREATE: @@ -716,16 +751,12 @@ static LRESULT CALLBACK wndproc(HWND hwnd, UINT msg, WPARAM wp, LPARAM lp) { case WM_RBUTTONUP: show_menu(hwnd); return 0; - case WM_MOUSEWHEEL: { - int delta = GET_WHEEL_DELTA_WPARAM(wp); - int next = g_firstMidi + (delta > 0 ? 12 : -12); - if (next >= 36 && next <= 84) { - mouse_up(hwnd); - release_typed(hwnd); - g_firstMidi = next; build_keys(); push_frame(hwnd); - } + case WM_MOUSEWHEEL: + shift_octave(hwnd, GET_WHEEL_DELTA_WPARAM(wp) > 0 ? 1 : -1); + return 0; + case WM_OCTAVE: + shift_octave(hwnd, (int)(intptr_t)wp); return 0; - } case WM_TYPEKEY: { int vk = (int)wp, down = (int)lp; if (vk < 0 || vk > 255) return 0; @@ -745,9 +776,7 @@ static LRESULT CALLBACK wndproc(HWND hwnd, UINT msg, WPARAM wp, LPARAM lp) { if (id == IDM_QUIT) { DestroyWindow(hwnd); return 0; } if (id == IDM_TYPING) { typing_set(hwnd, !g_typing); return 0; } if (id == IDM_AUTOSTART) { autostart_set(!autostart_enabled()); return 0; } - if (id >= IDM_PROGRAM_BASE && id < IDM_PROGRAM_BASE + (int)(sizeof PROGRAMS / sizeof PROGRAMS[0])) { - g_program = PROGRAMS[id - IDM_PROGRAM_BASE].program; midi_program(); - } + if (id >= IDM_PROGRAM_BASE && id < IDM_PROGRAM_BASE + 128) { g_program = id - IDM_PROGRAM_BASE; midi_program(); } if (id >= IDM_MIDI_BASE && id < IDM_MIDI_BASE + 40) midi_select(id - IDM_MIDI_BASE - 1); return 0; } diff --git a/slab/windows/README.md b/slab/windows/README.md new file mode 100644 index 0000000000..b5597c9767 --- /dev/null +++ b/slab/windows/README.md @@ -0,0 +1,22 @@ +# slab on Windows + +`slab-agent.ps1` is the `frame` agent for a Windows 11 machine. It speaks +the same file protocol as `FrameCapture.swift` inside the Mac menubar app, +so `frame ` from a Mac works unchanged: the client ships its +request over `ssh host bash -s`, Git bash answers on Windows, and the agent +(resident in the signed-in desktop session) produces the envelope. + +- Observe: GDI screen capture (foreground window by default, `screen` for + all of it, `crop=x,y,w,h`), Windows.Media.Ocr text with click centres, a + UI Automation walk of the foreground window as the `ax` tree, visible + windows, cursor, DPI. JPEG thumb ≤ 1568 px. +- Act: `native-click=` / `native-drag=` through SendInput. +- Coordinates are physical pixels (`coordinateSpace: windows-physical-pixels`). + +Install on the box: copy to `~\src\slab\slab-agent.ps1`, put Git's `bin` +on the machine PATH (for `bash -s`), start it inside the desktop session +(a scheduled task running as the user, or the HKCU Run key `SlabAgent`). +Register the machine in `~/.config/slab/puppet.json` with an `sshHost`. +Windows PowerShell 5.1 only; it is the one with WinRT. + +Timings on a MiniBook X: ~0.7 s warm (OCR dominates), ~4 s cold. diff --git a/slab/windows/slab-agent.ps1 b/slab/windows/slab-agent.ps1 new file mode 100644 index 0000000000..a8a67be52f --- /dev/null +++ b/slab/windows/slab-agent.ps1 @@ -0,0 +1,297 @@ +# slab-agent.ps1 — the slab `frame` agent for a Windows 11 box. +# +# Speaks the same file protocol as FrameCapture.swift inside the Mac +# menubar app, so `frame ` on a Mac works unchanged against a +# Windows machine (the client ships its request over `ssh host bash -s`, +# which Git bash answers here): +# +# ~/.local/share/slab/state/frame.req mode line, written by the client +# ~/.local/share/slab/state/frame.out.json envelope +# ~/.local/share/slab/state/frame.out.jpg thumbnail (≤ 1568 px wide) +# ~/.local/share/slab/state/frame.done touched when both are written +# +# Mode tokens honoured: window (default) | screen, noocr, fast, novisual, +# crop=x,y,w,h, cursor=x,y, session=…, native-click=, +# native-drag=, native-guard=. Unknown tokens are ignored. +# +# Observes with GDI (CopyFromScreen), reads text with Windows.Media.Ocr, walks +# the foreground window with UI Automation, and sends real input with +# SendInput. Must run INSIDE the signed-in desktop session (start it through +# the ac-desk task or the Run key), never from an ssh session: those have no +# desktop to capture. Windows PowerShell 5.1 only — it is the one with WinRT. + +$ErrorActionPreference = 'Continue' +Add-Type -AssemblyName System.Drawing, System.Windows.Forms, System.Runtime.WindowsRuntime, UIAutomationClient, UIAutomationTypes + +Add-Type @" +using System; +using System.Runtime.InteropServices; +using System.Text; +public class SlabNative { + [DllImport("user32.dll")] public static extern bool SetProcessDPIAware(); + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); + [DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r); + [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr h, out uint pid); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] public static extern int GetWindowTextW(IntPtr h, StringBuilder s, int n); + [DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p); + [DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y); + [DllImport("user32.dll")] public static extern uint SendInput(uint n, INPUT[] inputs, int size); + [DllImport("user32.dll")] public static extern bool EnumWindows(EnumProc cb, IntPtr l); + [DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr h); + [DllImport("dwmapi.dll")] public static extern int DwmGetWindowAttribute(IntPtr h, int attr, out RECT r, int size); + public delegate bool EnumProc(IntPtr h, IntPtr l); + [StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left, Top, Right, Bottom; } + [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X, Y; } + [StructLayout(LayoutKind.Sequential)] public struct MOUSEINPUT { public int dx, dy; public uint mouseData, dwFlags, time; public IntPtr dwExtraInfo; } + [StructLayout(LayoutKind.Sequential)] public struct KEYBDINPUT { public ushort wVk, wScan; public uint dwFlags, time; public IntPtr dwExtraInfo; } + [StructLayout(LayoutKind.Explicit)] public struct INPUTUNION { [FieldOffset(0)] public MOUSEINPUT mi; [FieldOffset(0)] public KEYBDINPUT ki; } + [StructLayout(LayoutKind.Sequential)] public struct INPUT { public uint type; public INPUTUNION u; } + public static void Mouse(uint flags, int dx, int dy, uint data) { + INPUT[] i = new INPUT[1]; i[0].type = 0; i[0].u.mi.dwFlags = flags; i[0].u.mi.dx = dx; i[0].u.mi.dy = dy; i[0].u.mi.mouseData = data; + SendInput(1, i, Marshal.SizeOf(typeof(INPUT))); + } + public static void Key(ushort vk, bool up) { + INPUT[] i = new INPUT[1]; i[0].type = 1; i[0].u.ki.wVk = vk; i[0].u.ki.dwFlags = up ? 2u : 0u; + SendInput(1, i, Marshal.SizeOf(typeof(INPUT))); + } +} +"@ +[SlabNative]::SetProcessDPIAware() | Out-Null + +# ── WinRT OCR plumbing (Windows PowerShell 5.1) ────────────────────────── +[void][Windows.Media.Ocr.OcrEngine, Windows.Foundation, ContentType = WindowsRuntime] +[void][Windows.Graphics.Imaging.BitmapDecoder, Windows.Foundation, ContentType = WindowsRuntime] +[void][Windows.Storage.Streams.InMemoryRandomAccessStream, Windows.Foundation, ContentType = WindowsRuntime] +[void][Windows.Globalization.Language, Windows.Foundation, ContentType = WindowsRuntime] +$asTaskGeneric = ([System.WindowsRuntimeSystemExtensions].GetMethods() | Where-Object { + $_.Name -eq 'AsTask' -and $_.GetParameters().Count -eq 1 -and $_.GetParameters()[0].ParameterType.Name -eq 'IAsyncOperation`1' })[0] +function Await($op, $type) { + $t = $asTaskGeneric.MakeGenericMethod($type).Invoke($null, @($op)); $t.Wait(-1) | Out-Null; $t.Result +} +$script:ocr = [Windows.Media.Ocr.OcrEngine]::TryCreateFromUserProfileLanguages() +if (-not $script:ocr) { $script:ocr = [Windows.Media.Ocr.OcrEngine]::TryCreateFromLanguage([Windows.Globalization.Language]::new('en-US')) } + +function Ocr-Bitmap([System.Drawing.Bitmap]$bmp, [int]$ox, [int]$oy) { + if (-not $script:ocr) { return @() } + $ms = New-Object System.IO.MemoryStream + $bmp.Save($ms, [System.Drawing.Imaging.ImageFormat]::Bmp); $ms.Position = 0 + $ras = [System.IO.WindowsRuntimeStreamExtensions]::AsRandomAccessStream($ms) + $dec = Await ([Windows.Graphics.Imaging.BitmapDecoder]::CreateAsync($ras)) ([Windows.Graphics.Imaging.BitmapDecoder]) + $sb = Await ($dec.GetSoftwareBitmapAsync()) ([Windows.Graphics.Imaging.SoftwareBitmap]) + $res = Await ($script:ocr.RecognizeAsync($sb)) ([Windows.Media.Ocr.OcrResult]) + $out = New-Object System.Collections.ArrayList + foreach ($line in $res.Lines) { + $x0 = 1e9; $y0 = 1e9; $x1 = -1e9; $y1 = -1e9 + foreach ($w in $line.Words) { $r = $w.BoundingRect; if ($r.X -lt $x0) { $x0 = $r.X }; if ($r.Y -lt $y0) { $y0 = $r.Y }; if ($r.X + $r.Width -gt $x1) { $x1 = $r.X + $r.Width }; if ($r.Y + $r.Height -gt $y1) { $y1 = $r.Y + $r.Height } } + if ($x1 -le $x0) { continue } + [void]$out.Add([ordered]@{ t = $line.Text; cx = [int]($ox + ($x0 + $x1) / 2); cy = [int]($oy + ($y0 + $y1) / 2); r = @([int]($ox + $x0), [int]($oy + $y0), [int]($x1 - $x0), [int]($y1 - $y0)) }) + } + $ms.Dispose() + ,$out.ToArray() +} + +# ── UI Automation (the AX tree's cousin) ──────────────────────────────── +$roleMap = @{ 'ControlType.Button' = 'AXButton'; 'ControlType.Edit' = 'AXTextField'; 'ControlType.Text' = 'AXStaticText'; 'ControlType.CheckBox' = 'AXCheckBox'; + 'ControlType.RadioButton' = 'AXRadioButton'; 'ControlType.ComboBox' = 'AXPopUpButton'; 'ControlType.Hyperlink' = 'AXLink'; 'ControlType.MenuItem' = 'AXMenuItem'; + 'ControlType.ListItem' = 'AXRow'; 'ControlType.TabItem' = 'AXRadioButton'; 'ControlType.Slider' = 'AXSlider'; 'ControlType.Image' = 'AXImage'; 'ControlType.Window' = 'AXWindow' } +function Walk-Uia([IntPtr]$hwnd, [int]$cap, [int]$budgetMs) { + $els = New-Object System.Collections.ArrayList + try { + $root = [System.Windows.Automation.AutomationElement]::FromHandle($hwnd) + $walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker + $queue = New-Object System.Collections.Generic.Queue[System.Windows.Automation.AutomationElement] + $queue.Enqueue($root) + $sw = [System.Diagnostics.Stopwatch]::StartNew() + while ($queue.Count -gt 0 -and $els.Count -lt $cap -and $sw.ElapsedMilliseconds -lt $budgetMs) { + $el = $queue.Dequeue() + try { + $c = $el.Current + if (-not $c.IsOffscreen -and $c.BoundingRectangle.Width -gt 0) { + $ct = $c.ControlType.ProgrammaticName + $role = if ($roleMap.ContainsKey($ct)) { $roleMap[$ct] } else { 'UIA' + ($ct -replace 'ControlType\.', '') } + $acts = New-Object System.Collections.ArrayList + if ($el.GetCurrentPropertyValue([System.Windows.Automation.AutomationElement]::IsInvokePatternAvailableProperty)) { [void]$acts.Add('AXPress') } + if ($el.GetCurrentPropertyValue([System.Windows.Automation.AutomationElement]::IsTogglePatternAvailableProperty)) { [void]$acts.Add('AXPress') } + if ($el.GetCurrentPropertyValue([System.Windows.Automation.AutomationElement]::IsValuePatternAvailableProperty)) { [void]$acts.Add('AXSetValue') } + if ($c.Name -or $acts.Count -gt 0) { + $b = $c.BoundingRectangle + [void]$els.Add([ordered]@{ role = $role; title = [string]$c.Name; cx = [int]($b.X + $b.Width / 2); cy = [int]($b.Y + $b.Height / 2); + r = @([int]$b.X, [int]$b.Y, [int]$b.Width, [int]$b.Height); actions = @($acts | Select-Object -Unique) }) + } + } + } catch {} + try { $child = $walker.GetFirstChild($el); while ($child -ne $null) { $queue.Enqueue($child); $child = $walker.GetNextSibling($child) } } catch {} + } + } catch {} + ,$els.ToArray() +} + +# ── helpers ───────────────────────────────────────────────────────────── +function Window-Title([IntPtr]$h) { $sb = New-Object System.Text.StringBuilder 512; [void][SlabNative]::GetWindowTextW($h, $sb, 512); $sb.ToString() } +function Window-Bounds([IntPtr]$h) { + $r = New-Object SlabNative+RECT + if ([SlabNative]::DwmGetWindowAttribute($h, 9, [ref]$r, 16) -ne 0) { [void][SlabNative]::GetWindowRect($h, [ref]$r) } + $r +} +function Visible-Windows([int]$cap) { + $list = New-Object System.Collections.ArrayList + $cb = [SlabNative+EnumProc]{ param($h, $l) + if ($list.Count -ge $cap) { return $false } + if ([SlabNative]::IsWindowVisible($h)) { $t = Window-Title $h; if ($t) { $b = Window-Bounds $h; $wpid = 0; [void][SlabNative]::GetWindowThreadProcessId($h, [ref]$wpid) + $app = try { (Get-Process -Id $wpid -ErrorAction Stop).ProcessName } catch { '?' } + [void]$list.Add([ordered]@{ app = $app; title = $t; pid = [int]$wpid; x = $b.Left; y = $b.Top; w = $b.Right - $b.Left; h = $b.Bottom - $b.Top }) } } + $true } + [void][SlabNative]::EnumWindows($cb, [IntPtr]::Zero) + ,$list.ToArray() +} +function Save-Jpeg([System.Drawing.Bitmap]$bmp, [string]$path, [int]$maxW) { + $img = $bmp + if ($bmp.Width -gt $maxW) { $h = [int]($bmp.Height * $maxW / $bmp.Width); $img = New-Object System.Drawing.Bitmap $bmp, $maxW, $h } + $codec = [System.Drawing.Imaging.ImageCodecInfo]::GetImageEncoders() | Where-Object { $_.MimeType -eq 'image/jpeg' } + $ep = New-Object System.Drawing.Imaging.EncoderParameters 1 + $ep.Param[0] = New-Object System.Drawing.Imaging.EncoderParameter ([System.Drawing.Imaging.Encoder]::Quality), 72L + $img.Save($path, $codec, $ep) + if ($img -ne $bmp) { $img.Dispose() } +} +function Decode-B64Json($s) { try { [System.Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($s)) | ConvertFrom-Json } catch { $null } } + +# ── native input ──────────────────────────────────────────────────────── +function Move-To([int]$x, [int]$y) { + $sw = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds + $nx = [int]($x * 65535 / ($sw.Width - 1)); $ny = [int]($y * 65535 / ($sw.Height - 1)) + [SlabNative]::Mouse(0x8001, $nx, $ny, 0) # MOVE | ABSOLUTE +} +function Click-At($req) { + $x = [int]$req.x; $y = [int]$req.y + $button = if ($req.button) { [string]$req.button } else { 'left' } + $count = if ($req.count) { [int]$req.count } else { 1 } + $down = 0x0002; $up = 0x0004 + if ($button -eq 'right') { $down = 0x0008; $up = 0x0010 } + Move-To $x $y; Start-Sleep -Milliseconds 20 + if ($req.drag) { + [SlabNative]::Mouse($down, 0, 0, 0) + $steps = [Math]::Max(2, [int]([double]($(if ($req.drag.durationMs) { $req.drag.durationMs } else { 160 })) / 8)) + for ($i = 1; $i -le $steps; $i++) { Move-To ([int]($x + ($req.drag.x - $x) * $i / $steps)) ([int]($y + ($req.drag.y - $y) * $i / $steps)); Start-Sleep -Milliseconds 8 } + [SlabNative]::Mouse($up, 0, 0, 0) + return [ordered]@{ sent = $true; kind = 'drag'; from = @($x, $y); to = @([int]$req.drag.x, [int]$req.drag.y) } + } + for ($i = 0; $i -lt $count; $i++) { + [SlabNative]::Mouse($down, 0, 0, 0) + if ($req.holdMs) { Start-Sleep -Milliseconds ([int]$req.holdMs) } else { Start-Sleep -Milliseconds 12 } + [SlabNative]::Mouse($up, 0, 0, 0) + if ($i -lt $count - 1) { Start-Sleep -Milliseconds 60 } + } + [ordered]@{ sent = $true; kind = 'click'; x = $x; y = $y; button = $button; count = $count } +} + +# ── one observation ───────────────────────────────────────────────────── +$state = "$HOME\.local\share\slab\state" +New-Item -ItemType Directory -Force $state | Out-Null +$utf8 = New-Object System.Text.UTF8Encoding $false + +function Trace($m) { try { Add-Content -Path "$state\agent.trace" -Value ("{0:HH:mm:ss.fff} {1}" -f (Get-Date), $m) } catch {} } +function Produce([string]$mode) { + Trace "produce: $mode" + $t0 = [System.Diagnostics.Stopwatch]::StartNew() + $tm = [ordered]@{} + $tokens = $mode -split ' ' | Where-Object { $_ } + $flags = @{}; foreach ($tk in $tokens) { $flags[$tk] = $true } + $session = ($tokens | Where-Object { $_ -like 'session=*' } | Select-Object -First 1); if ($session) { $session = $session.Substring(8) } else { $session = 'legacy' } + $fr = [ordered]@{ capture = 'ok'; capture_scope = 'window'; coordinateSpace = 'windows-physical-pixels' } + + # Native input first, then observe what it did. + $nativeToken = $tokens | Where-Object { $_ -like 'native-click=*' -or $_ -like 'native-drag=*' -or $_ -like 'native-guard=*' } | Select-Object -First 1 + if ($nativeToken) { + $kind, $b64 = $nativeToken -split '=', 2 + $req = Decode-B64Json $b64 + if (-not $req) { $fr.nativeInput = @{ error = 'Invalid native request; no action sent' } } + elseif ($kind -eq 'native-guard') { $fr.nativeInput = @{ guard = 'ok' } } + else { $ti = [System.Diagnostics.Stopwatch]::StartNew(); $fr.nativeInput = Click-At $req; $fr.nativeInput.ms = $ti.ElapsedMilliseconds; Start-Sleep -Milliseconds 120 } + $fr.nativeCapabilities = @('guarded-click-v1', 'guarded-drag-v1', 'click-hold-v1') + } + + Trace "geometry" + # Geometry. + $screen = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds + $scale = [Math]::Round(([System.Drawing.Graphics]::FromHwnd([IntPtr]::Zero)).DpiX / 96, 2) + $fg = [SlabNative]::GetForegroundWindow() + $fgPid = 0; [void][SlabNative]::GetWindowThreadProcessId($fg, [ref]$fgPid) + $fgApp = try { (Get-Process -Id $fgPid -ErrorAction Stop).ProcessName } catch { '?' } + $fgTitle = Window-Title $fg + $wb = Window-Bounds $fg + $region = $screen + if ($flags['screen'] -or $fg -eq [IntPtr]::Zero -or ($wb.Right - $wb.Left) -le 0) { $fr.capture_scope = 'screen' } + else { $region = New-Object System.Drawing.Rectangle ([Math]::Max(0, $wb.Left)), ([Math]::Max(0, $wb.Top)), ([Math]::Min($screen.Width, $wb.Right) - [Math]::Max(0, $wb.Left)), ([Math]::Min($screen.Height, $wb.Bottom) - [Math]::Max(0, $wb.Top)) } + $cropTok = $tokens | Where-Object { $_ -like 'crop=*' } | Select-Object -First 1 + if ($cropTok) { $c = $cropTok.Substring(5) -split ','; if ($c.Count -eq 4) { $region = New-Object System.Drawing.Rectangle ([int]$c[0]), ([int]$c[1]), ([int]$c[2]), ([int]$c[3]); $fr.capture_scope = 'crop' } } + if ($region.Width -le 0 -or $region.Height -le 0) { $region = $screen; $fr.capture_scope = 'screen' } + if ($fr.capture_scope -ne 'screen') { $fr.crop = [ordered]@{ x = $region.X; y = $region.Y; w = $region.Width; h = $region.Height } } + + Trace "pixels" + # Pixels. + $t = [System.Diagnostics.Stopwatch]::StartNew() + $bmp = New-Object System.Drawing.Bitmap $region.Width, $region.Height + $g = [System.Drawing.Graphics]::FromImage($bmp) + $g.CopyFromScreen($region.Location, [System.Drawing.Point]::Empty, $region.Size); $g.Dispose() + $tm.capture = $t.ElapsedMilliseconds + + Trace "meta" + # Meta. + $t.Restart() + $cur = New-Object SlabNative+POINT; [void][SlabNative]::GetCursorPos([ref]$cur) + $fr.meta = [ordered]@{ + host = $env:COMPUTERNAME; os = 'windows' + screen = [ordered]@{ w = $screen.Width; h = $screen.Height; scale = $scale } + cursor = [ordered]@{ x = $cur.X; y = $cur.Y } + frontmost = [ordered]@{ app = $fgApp; pid = [int]$fgPid; title = $fgTitle; bundle = $fgApp } + windows = [object[]](Visible-Windows 24) + } + $tm.meta = $t.ElapsedMilliseconds + + Trace "ocr" + # OCR. + $t.Restart() + if ($flags['noocr']) { $fr.ocr = @() } else { $fr.ocr = [object[]](Ocr-Bitmap $bmp $region.X $region.Y) } + $tm.ocr = $t.ElapsedMilliseconds + + Trace "uia" + # UIA. + $t.Restart() + $cap = if ($flags['fast']) { 80 } else { 300 } + $els = if ($fg -ne [IntPtr]::Zero) { [object[]](Walk-Uia $fg $cap 600) } else { @() } + $fr.ax = [ordered]@{ trusted = $true; app = $fgApp; scope = $(if ($fr.capture_scope -eq 'screen') { 'application' } else { 'window' }); elements = $els } + $tm.ax = $t.ElapsedMilliseconds + + $fr.visual = @() + $fr.observation = [ordered]@{ id = [guid]::NewGuid().ToString(); session = $session; windowId = [int64]$fg; kind = 'frame'; coordinateSpace = 'windows-physical-pixels' } + + Trace "thumb" + # Thumb + envelope. + $t.Restart() + $jpg = "$state\frame.out.jpg"; $json = "$state\frame.out.json" + Save-Jpeg $bmp $jpg 1568 + $bmp.Dispose() + $fr.thumb_bytes = (Get-Item $jpg).Length + $tm.thumb = $t.ElapsedMilliseconds + $tm.wall = $t0.ElapsedMilliseconds + $fr.timings_ms = $tm + [IO.File]::WriteAllText($json, ($fr | ConvertTo-Json -Depth 8 -Compress), $utf8) + Trace "done $($tm.wall)ms" +} + +# ── request loop ──────────────────────────────────────────────────────── +$req = "$state\frame.req"; $done = "$state\frame.done" +"slab-agent up in session $([System.Diagnostics.Process]::GetCurrentProcess().SessionId) · ocr=$([bool]$script:ocr) · $state" +while ($true) { + if (Test-Path $req) { + $mode = '' + try { $mode = [IO.File]::ReadAllText($req) } catch {} + Remove-Item $req -Force -ErrorAction SilentlyContinue + try { Produce $mode } catch { + [IO.File]::WriteAllText("$state\frame.out.json", (@{ capture = 'error'; reason = "$($_.Exception.Message)" } | ConvertTo-Json -Compress), $utf8) + Remove-Item "$state\frame.out.jpg" -Force -ErrorAction SilentlyContinue + } + [IO.File]::WriteAllText($done, 'ok', $utf8) + } + Start-Sleep -Milliseconds 15 +}