From a29fa68d3b3189cac5c4e4135719011b8985c7a3 Mon Sep 17 00:00:00 2001 From: Kieran Klukas Date: Fri, 5 Jun 2026 15:29:52 -0400 Subject: [PATCH] feat: working version babbbyyyyy --- .env | 2 + .gitignore | 2 + grimoire.py | 73 ++- grimoire_loop.py | 534 ++++++++++++++++++ inject.sh | 14 + ocr_pipeline.py | 133 +++++ .../grimoire-injector/GrimoireInjector.cpp | 137 ++++- .../grimoire-injector/GrimoireInjector.hpp | 30 + xovi-ext/grimoire-injector/entry.c | 12 + 9 files changed, 930 insertions(+), 7 deletions(-) create mode 100644 .env create mode 100644 grimoire_loop.py create mode 100755 inject.sh create mode 100644 ocr_pipeline.py diff --git a/.env b/.env new file mode 100644 index 0000000..527f11d --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +POTLUCK_API_KEY=pot_slate_OSZqyYeJ1363l6ZKdS_1641K +GEMINI_API_KEY=AIzaSyC9WVz9kAEDWozFrk3OBgWbYplZyIglG-c diff --git a/.gitignore b/.gitignore index cacf5c5..f10eaa1 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ analysis/hashtab # Frida scripts (temporary) frida-explore.js + +.env diff --git a/grimoire.py b/grimoire.py index d4865c1..f020e10 100644 --- a/grimoire.py +++ b/grimoire.py @@ -91,6 +91,28 @@ def _parse_d(d: str) -> list[list[tuple[float, float]]]: return polylines +def _densify( + pts: list[tuple[float, float]], min_seg_len: float = 120.0 +) -> list[tuple[float, float]]: + """Subdivide a polyline so no segment is longer than min_seg_len. + + Short subpaths (single line segments) render unreliably on device — + the renderer needs several samples to draw a visible stroke. This walks + each segment in glyph-space (pre-scale) and inserts evenly spaced + midpoints, guaranteeing at least one interior point per segment. + """ + if len(pts) < 2: + return pts + out = [pts[0]] + for (x0, y0), (x1, y1) in zip(pts, pts[1:]): + dist = math.hypot(x1 - x0, y1 - y0) + steps = max(2, int(math.ceil(dist / min_seg_len))) + for s in range(1, steps + 1): + t = s / steps + out.append((x0 + (x1 - x0) * t, y0 + (y1 - y0) * t)) + return out + + _FONT_CACHE: Optional[tuple[dict, dict[str, float]]] = None DEFAULT_SCALE = 0.07 DEFAULT_X = -550.0 @@ -144,8 +166,14 @@ def text_to_strokes( for polyline in glyph: if len(polyline) < 2: continue + # Densify: subdivide each segment so even short subpaths + # (e.g. the arms of '*', the bowl of 'p') carry enough + # points for the device renderer, which collapses or drops + # 2-point strokes. Insert midpoints until every stroke has + # at least a few samples per segment. + dense = _densify(polyline, min_seg_len=120.0) points = [] - for j, (gx, gy) in enumerate(polyline): + for j, (gx, gy) in enumerate(dense): sx = gx * scale sy = -gy * scale sx += sy * slant @@ -173,6 +201,32 @@ def text_to_strokes( return result +def _find_content_bottom(data: bytes) -> float: + """Scan existing strokes and return the max Y coordinate + margin. + + Used to position reply text below the user's handwriting instead of + overwriting it. Returns DEFAULT_Y if no strokes are found. + """ + max_y = 0.0 + found = False + try: + with BytesIO(data) as f: + for block in read_blocks(f): + if isinstance(block, SceneLineItemBlock): + line = block.item.value + if hasattr(line, 'points'): + for pt in line.points: + if pt.y > max_y: + max_y = pt.y + found = True + except Exception: + pass + if not found: + return DEFAULT_Y + # Place reply below existing content with generous spacing + return max_y + 120.0 + + def _find_last_root_child(data: bytes) -> CrdtId: """Find the last SceneGroupItemBlock value under root for left_id chaining.""" from io import BytesIO @@ -200,8 +254,9 @@ def splice_reply_new_layer( with open(input_path, "rb") as f: original_data = f.read() - # Build reply as strokes - strokes = text_to_strokes(reply_text, DEFAULT_X, DEFAULT_Y) + # Build reply as strokes, positioned below existing content + reply_y = _find_content_bottom(original_data) + strokes = text_to_strokes(reply_text, DEFAULT_X, reply_y) # CRDT IDs — author 2 for programmatic content, starting at seq 500 # to avoid collision with existing data (real Layer 2 uses ~422-427) @@ -300,16 +355,22 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("input", nargs="?", default=None) parser.add_argument("output", nargs="?", default=None) - parser.add_argument("text", nargs="?", default="Grimoire says hello") + parser.add_argument("text", nargs="?", default=None) parser.add_argument("--json", dest="json_output", default=None, help="Export strokes as JSON for xovi injector") + parser.add_argument("--y", type=float, default=None, + help="Y coordinate to start rendering at (overrides default)") args = parser.parse_args() if args.json_output: - strokes = text_to_strokes(args.text, DEFAULT_X, DEFAULT_Y) + # --json [text] + text = args.input or "Grimoire says hello" + y = args.y if args.y is not None else DEFAULT_Y + strokes = text_to_strokes(text, DEFAULT_X, y) strokes_to_json(strokes, args.json_output) elif args.input: + text = args.text or "Grimoire says hello" output = args.output or args.input - splice_reply_new_layer(args.input, output, args.text) + splice_reply_new_layer(args.input, output, text) else: parser.error("Provide input.rm or --json output.json") diff --git a/grimoire_loop.py b/grimoire_loop.py new file mode 100644 index 0000000..f78a8a7 --- /dev/null +++ b/grimoire_loop.py @@ -0,0 +1,534 @@ +#!/usr/bin/env python3 +""" +grimoire_loop.py — Continuous closed loop daemon + +Watches for pen-idle signal from xovi extension, then: + capture → OCR → LLM → render → inject → repeat + +Usage: + python grimoire_loop.py [--model MODEL] [--debounce SECONDS] + +The xovi extension writes /tmp/grimoire_idle when the pen has been +up for 2 seconds. This daemon watches that file via a persistent SSH +connection, pulls the framebuffer only when idle is detected, and +runs the full pipeline. +""" + +import argparse +import base64 +import hashlib +import io +import os +import re +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import requests +from dotenv import load_dotenv + +load_dotenv() + + +# ─── Persistent SSH session ──────────────────────────────────────── + +class SSHSession: + """Persistent SSH connection using ControlMaster multiplexing. + + Opens one SSH connection and reuses it for all subsequent commands, + eliminating the ~1s handshake overhead per call. + """ + + def __init__(self, host="remarkable"): + self.host = host + self.socket = f"/tmp/grimoire_ssh_{os.getpid()}" + self._open() + + def _open(self): + """Establish master connection.""" + cmd = [ + "ssh", "-fN", + "-o", "ControlMaster=yes", + "-o", f"ControlPath={self.socket}", + "-o", "ControlPersist=600", + "-o", "ServerAliveInterval=30", + "-o", "ServerAliveCountMax=3", + self.host, + ] + result = subprocess.run(cmd, capture_output=True, timeout=15) + if result.returncode != 0: + raise RuntimeError( + f"SSH master failed: {result.stderr.decode().strip()}" + ) + print(f"[ssh] Master connection open ({self.socket})") + + def run(self, cmd, timeout=30, capture=True): + """Run command on remote via multiplexed connection.""" + full_cmd = [ + "ssh", + "-o", f"ControlPath={self.socket}", + self.host, + cmd, + ] + return subprocess.run( + full_cmd, + capture_output=capture, + text=True, + timeout=timeout, + ) + + def scp_from(self, remote_path, local_path, timeout=15): + """Pull file from device via multiplexed connection.""" + cmd = [ + "scp", + "-o", f"ControlPath={self.socket}", + f"{self.host}:{remote_path}", + local_path, + ] + return subprocess.run(cmd, capture_output=True, timeout=timeout) + + def scp_to(self, local_path, remote_path, timeout=15): + """Push file to device via multiplexed connection.""" + cmd = [ + "scp", + "-o", f"ControlPath={self.socket}", + local_path, + f"{self.host}:{remote_path}", + ] + return subprocess.run(cmd, capture_output=True, timeout=timeout) + + def close(self): + """Tear down master connection.""" + subprocess.run( + ["ssh", "-o", f"ControlPath={self.socket}", "-O", "exit", self.host], + capture_output=True, timeout=5, + ) + print("[ssh] Master connection closed") + + +def _ts(): + """Timestamp with millisecond precision for profiling.""" + return time.strftime('%H:%M:%S.') + f'{int(time.time() * 1000) % 1000:03d}' + + +def _elapsed(t0): + """Seconds since t0, formatted for display.""" + return f'{time.time() - t0:.1f}s' + + +# ─── Framebuffer capture ────────────────────────────────────────── + +def capture_framebuffer(ssh): + """Trigger screenshot on device and pull raw framebuffer.""" + # Trigger via xovi watch thread + ssh.run("touch /tmp/grimoire_screenshot", timeout=5) + time.sleep(1) # watch thread dumps FB nearly instantly + + result = ssh.scp_from("/tmp/grimoire_fb.raw", "/tmp/grimoire_fb.raw") + if result.returncode != 0: + raise RuntimeError(f"FB pull failed: {result.stderr.decode()}") + + from PIL import Image + + raw = open("/tmp/grimoire_fb.raw", "rb").read() + img = Image.frombytes("RGBA", (1404, 1872), raw) + img_rgb = img.convert("RGB") + + # Crop toolbar (top 80px) + img_rgb = img_rgb.crop((0, 80, img_rgb.width, img_rgb.height)) + return img_rgb + + +# ─── Gemini Vision (OCR + LLM in one call) ──────────────────────── + +SYSTEM_PROMPT = ( + "You are an ink familiar bound to a paper notebook. The user writes " + "questions by hand on a reMarkable tablet. You receive a screenshot of " + "the current page.\n\n" + "IMPORTANT CONTEXT:\n" + "- Your previous replies appear as neat, uniform handwriting near the " + "bottom of the page. IGNORE these completely.\n" + "- The toolbar at the very top has printed icons. IGNORE these.\n" + "- Background speckles and faint dots are e-ink artifacts. IGNORE these.\n" + "- Only respond to NEW handwritten text that looks like a question or " + "prompt from the user.\n\n" + "RULES:\n" + "- If there is no new user question, respond with exactly: [NO_NEW_TEXT]\n" + "- If there IS a new question, answer briefly in plain prose. One or two " + "short sentences max. No markdown, no emoji, no formatting, no bullet " + "points. Just write your answer as if writing by hand.\n" + "- Do not greet the user unless they greeted you first.\n" + "- Do not narrate your reasoning." +) + + +def _image_to_base64(image): + """Encode PIL Image as base64 PNG for Gemini API.""" + buf = io.BytesIO() + image.save(buf, format="PNG") + return base64.b64encode(buf.getvalue()).decode("utf-8") + + +def ask_gemini(image, conversation_history=None, model="gemini-2.5-flash-lite"): + """Send cropped handwriting image to Gemini, return response text. + + Combines OCR and LLM into a single API call — Gemini reads the + handwriting directly from the image. Includes conversation history + for context across multiple exchanges. + """ + api_key = os.environ.get("GEMINI_API_KEY") + if not api_key: + raise RuntimeError("GEMINI_API_KEY not set in .env") + + img_b64 = _image_to_base64(image) + + # Build contents array with history + current turn + contents = [] + if conversation_history: + for role, text in conversation_history: + contents.append({ + "role": role, + "parts": [{"text": text}], + }) + + # Current turn: full page image + contents.append({ + "role": "user", + "parts": [ + {"text": "Here is the current page:"}, + { + "inline_data": { + "mime_type": "image/png", + "data": img_b64, + } + }, + ], + }) + + r = requests.post( + f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent", + headers={"Content-Type": "application/json"}, + params={"key": api_key}, + json={ + "system_instruction": { + "parts": [{"text": SYSTEM_PROMPT}], + }, + "contents": contents, + }, + timeout=60, + ) + + if r.status_code != 200: + raise RuntimeError(f"Gemini API error {r.status_code}: {r.text[:200]}") + + data = r.json() + try: + content = data["candidates"][0]["content"]["parts"][0]["text"].strip() + except (KeyError, IndexError): + content = "..." + + # Strip any markdown/emoji that slipped through + content = _strip_formatting(content) + return content if content else "..." + + +def _framebuffer_hash(img): + """Quick hash of downsampled image for page-change detection.""" + # Resize to small thumbnail for fast comparison + thumb = img.resize((64, 85)) + import io + buf = io.BytesIO() + thumb.save(buf, format="PNG") + return hashlib.md5(buf.getvalue()).hexdigest() + + +def _strip_formatting(text): + """Remove markdown syntax and emoji from LLM output.""" + # Strip common markdown + text = re.sub(r'\*\*(.+?)\*\*', r'\1', text) # bold + text = re.sub(r'\*(.+?)\*', r'\1', text) # italic + text = re.sub(r'`(.+?)`', r'\1', text) # inline code + text = re.sub(r'^#+\s*', '', text, flags=re.MULTILINE) # headings + text = re.sub(r'^\s*[-*+]\s+', '', text, flags=re.MULTILINE) # list items + # Strip emoji (broad unicode ranges) + text = re.sub( + r'[\U0001F300-\U0001FAFF\U00002702-\U000027B0' + r'\U0001F600-\U0001F64F\U0001F680-\U0001F6FF' + r'\U00002600-\U000026FF\U0000FE00-\U0000FE0F]+', + '', text, + ) + return text.strip() + + +# ─── Render + Inject ────────────────────────────────────────────── + +def render_and_inject(ssh, text, reply_y=None): + """Render text as strokes and inject via uinject.""" + json_path = "/tmp/grimoire_reply.json" + + cmd = [".venv/bin/python3", "grimoire.py", "--json", json_path] + if reply_y is not None: + cmd.extend(["--y", str(reply_y)]) + cmd.append(text) + + result = subprocess.run(cmd, capture_output=True, text=True, timeout=10) + if result.returncode != 0: + raise RuntimeError(f"Render failed: {result.stderr.strip()}") + print(f" {result.stdout.strip()}") + + r = ssh.scp_to(json_path, "/tmp/grimoire_strokes.json") + if r.returncode != 0: + raise RuntimeError(f"SCP failed: {r.stderr.decode().strip()}") + + result = ssh.run("/home/root/uinject /tmp/grimoire_strokes.json", timeout=60) + out = result.stdout.strip() if result.stdout else "" + err = result.stderr.strip() if result.stderr else "" + if result.returncode != 0: + raise RuntimeError(f"Inject failed: {err}") + last_line = out.split('\n')[-1] if out else "(no output)" + print(f" Inject: {last_line}") + + +# ─── Idle watching ──────────────────────────────────────────────── + +def watch_for_idle(ssh, last_hash): + """Block until /tmp/grimoire_idle changes on device. + + Returns the new file hash, or None on error. + Polls every 500ms via the persistent SSH connection. + """ + while True: + result = ssh.run("cat /tmp/grimoire_idle 2>/dev/null", timeout=5) + if result.returncode == 0: + content = result.stdout.strip() + h = hashlib.md5(content.encode()).hexdigest() + if h != last_hash: + return h + time.sleep(0.5) + + +# ─── Pixel diff for new content detection ────────────────────────── + +def _find_new_region(current_img, last_img, ignore_below_y=None): + """Compare two framebuffer images, return crop of changed region. + + Returns (cropped_image, content_bottom_y) or (None, 0) if no + significant change. Uses simple pixel difference to find where + new handwriting appeared. + + ignore_below_y: if set, ignore all changes below this Y coordinate + in the cropped image. Used to skip grimoire's own injected strokes. + """ + import numpy as np + + curr = np.array(current_img.convert('L')) + + if last_img is None: + # First capture — find bounding box of existing handwriting. + # Use a strict threshold to ignore background noise/dots. + dark = curr < 128 + if not dark.any(): + return None, 0 + rows = np.any(dark, axis=1) + cols = np.any(dark, axis=0) + y1, y2 = np.where(rows)[0][[0, -1]] + x1, x2 = np.where(cols)[0][[0, -1]] + pad = 20 + y1 = max(0, y1 - pad) + y2 = min(curr.shape[0] - 1, y2 + pad) + x1 = max(0, x1 - pad) + x2 = min(curr.shape[1] - 1, x2 + pad) + # Sanity check: if the bounding box covers >80% of the screen, + # it's probably noise, not real content. + if (y2 - y1) > curr.shape[0] * 0.8 and (x2 - x1) > curr.shape[1] * 0.8: + return None, 0 + crop = current_img.crop((x1, y1, x2 + 1, y2 + 1)) + return crop, y2 + + prev = np.array(last_img.convert('L')) + + # Build mask: ignore edges (e-ink artifacts) and previously injected area + mask = np.zeros_like(curr, dtype=bool) + top_margin = 10 + bottom_margin = 20 + mask[top_margin:curr.shape[0] - bottom_margin, :] = True + + if ignore_below_y is not None: + # Convert device Y back to cropped-image Y (subtract toolbar offset) + cutoff = ignore_below_y - 80 + if 0 < cutoff < curr.shape[0]: + mask[cutoff:, :] = False + + # Find pixels that changed significantly + diff = (np.abs(curr.astype(int) - prev.astype(int)) > 50) & mask + + if not diff.any(): + return None, 0 + + # Find bounding box of changed pixels + rows = np.any(diff, axis=1) + cols = np.any(diff, axis=0) + y1, y2 = np.where(rows)[0][[0, -1]] + x1, x2 = np.where(cols)[0][[0, -1]] + + pad = 30 + y1 = max(0, y1 - pad) + y2 = min(curr.shape[0] - 1, y2 + pad) + x1 = max(0, x1 - pad) + x2 = min(curr.shape[1] - 1, x2 + pad) + + # Minimum size check + if (y2 - y1) < 10 or (x2 - x1) < 10: + return None, 0 + + crop = current_img.crop((x1, y1, x2 + 1, y2 + 1)) + return crop, y2 + + +# ─── Main loop ──────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser(description="Grimoire continuous loop") + parser.add_argument( + "--model", type=str, default="gemini-2.5-flash-lite", + help="Gemini model", + ) + parser.add_argument( + "--once", action="store_true", + help="Run one cycle then exit (for testing)", + ) + args = parser.parse_args() + + print("=== Grimoire Loop ===") + print(f"Model: {args.model}") + print("Waiting for pen idle signal...") + print() + + ssh = SSHSession("remarkable") + last_idle_hash = "" + last_page_hash = None # hash for page-change detection + last_fb_hash = None # quick hash to skip unchanged frames + conversation = [] # list of (role, text) tuples for context + last_inject_y = None # device Y where we last injected + + try: + while True: + # Wait for idle signal + new_hash = watch_for_idle(ssh, last_idle_hash) + if new_hash is None: + continue + last_idle_hash = new_hash + + t0 = time.time() + print(f"[{_ts()}] Pen idle detected!") + + try: + # Capture + print(f" [{_ts()}] Capturing...") + img = capture_framebuffer(ssh) + print(f" [{_ts()}] Captured ({_elapsed(t0)})") + + # Quick hash check — skip if framebuffer unchanged + fb_hash = _framebuffer_hash(img) + if fb_hash == last_fb_hash: + print(f" [{_ts()}] Framebuffer unchanged, skipping.") + print() + continue + last_fb_hash = fb_hash + + # Page change detection — only reset if we haven't just + # injected (our own strokes change the hash too). + is_own_injection = (last_inject_y is not None and + last_page_hash is not None and + fb_hash != last_page_hash) + if last_page_hash is not None and fb_hash != last_page_hash: + if is_own_injection: + print(f" [{_ts()}] Hash changed (own injection), keeping context.") + else: + print(f" [{_ts()}] Page changed, resetting context.") + conversation = [] + last_inject_y = None + last_page_hash = fb_hash + + # Send full page to Gemini — it handles OCR + relevance filtering + print(f" [{_ts()}] Asking Gemini ({len(conversation)} history turns)...") + answer = ask_gemini(img, conversation, args.model) + print(f" [{_ts()}] Reply ({_elapsed(t0)}): {answer[:80]}{'...' if len(answer) > 80 else ''}") + + # Skip if Gemini says no new text + if "[NO_NEW_TEXT]" in answer: + print(f" [{_ts()}] No new text detected by Gemini, skipping.") + last_img = img + print() + continue + + # Add to conversation history + conversation.append(("user", f"[handwritten question]")) + conversation.append(("model", answer)) + + # Position reply below existing content. + # The reMarkable display has a uniform grid artifact: + # isolated 2-row pairs with exactly 68 dark pixels each. + # Real handwriting has variable pixel counts across many + # consecutive rows. Filter by requiring non-68 counts or + # clusters taller than 4 rows. + import numpy as np + gray = np.array(img.convert('L')) + dark_per_row = np.sum(gray < 128, axis=1) + + # Mark rows as "real content" if they have dark pixels + # AND the count isn't exactly 68 (grid artifact), OR + # they're part of a cluster > 4 rows tall. + has_dark = dark_per_row > 5 + not_grid = dark_per_row != 68 + candidate_rows = np.where(has_dark & not_grid)[0] + + content_bottom = None + if len(candidate_rows) > 0: + # Group into contiguous clusters + gaps = np.diff(candidate_rows) + clusters = [] + start = candidate_rows[0] + for i, g in enumerate(gaps): + if g > 10: + clusters.append((start, candidate_rows[i])) + start = candidate_rows[i + 1] + clusters.append((start, candidate_rows[-1])) + + # Use bottom of last real cluster + if clusters: + content_bottom = clusters[-1][1] + + if content_bottom is not None: + reply_y = min(content_bottom + 80 + 150, 1750) + else: + reply_y = 300 + print(f" [{_ts()}] Content bottom: {content_bottom}, reply_y={reply_y}") + + # Render + Inject + print(f" [{_ts()}] Rendering + injecting (reply_y={reply_y})...") + render_and_inject(ssh, answer, reply_y=reply_y) + last_inject_y = reply_y + print(f" [{_ts()}] Injected ({_elapsed(t0)})") + + elapsed = time.time() - t0 + print(f" [{_ts()}] Done in {elapsed:.1f}s") + + except Exception as e: + print(f" ERROR: {e}") + + print() + + if args.once: + break + + except KeyboardInterrupt: + print("\nStopping.") + finally: + ssh.close() + + +if __name__ == "__main__": + main() diff --git a/inject.sh b/inject.sh new file mode 100755 index 0000000..e16bd0f --- /dev/null +++ b/inject.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ $# -eq 0 ]; then + echo "usage: $(basename "$0") " >&2 + exit 1 +fi + +TEXT="$*" +TMP="/tmp/grimoire_test.json" + +.venv/bin/python3 grimoire.py --json "$TMP" "$TEXT" +scp "$TMP" remarkable:/tmp/grimoire_strokes.json +ssh -o ConnectTimeout=10 remarkable '/home/root/uinject /tmp/grimoire_strokes.json' diff --git a/ocr_pipeline.py b/ocr_pipeline.py new file mode 100644 index 0000000..727acfc --- /dev/null +++ b/ocr_pipeline.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +""" +ocr_pipeline.py — Capture framebuffer region → OCR → return text + +Usage: + python ocr_pipeline.py [--region x,y,w,h] [--output text.txt] + +Captures the reMarkable framebuffer via SSH, crops to the specified region, +runs PaddleOCR, and prints/saves the recognized text. +""" + +import argparse +import subprocess +import tempfile +import struct +from pathlib import Path + +# Framebuffer constants for rM2 +FB_WIDTH = 1404 +FB_HEIGHT = 1872 +FB_BPP = 4 # RGBA +FB_DEVICE = "/dev/fb0" + + +def capture_framebuffer(region=None): + """Capture framebuffer from device via SSH, return as raw bytes.""" + if region: + x, y, w, h = region + # Use dd to capture only the needed rows, then crop columns + cmd = f"dd if={FB_DEVICE} bs={FB_WIDTH * FB_BPP} skip={y} count={h} 2>/dev/null" + else: + cmd = f"cat {FB_DEVICE}" + + result = subprocess.run( + ["ssh", "remarkable", cmd], + capture_output=True, timeout=10 + ) + + if result.returncode != 0: + raise RuntimeError(f"Failed to capture framebuffer: {result.stderr.decode()}") + + raw = result.stdout + + if region: + x, y, w, h = region + # Crop columns from each row + cropped = b"" + row_size = FB_WIDTH * FB_BPP + for row_y in range(h): + row_start = row_y * row_size + x * FB_BPP + cropped += raw[row_start:row_start + w * FB_BPP] + return cropped, w, h + else: + return raw, FB_WIDTH, FB_HEIGHT + + +def raw_to_image(raw_bytes, width, height): + """Convert raw framebuffer bytes to PIL Image.""" + from PIL import Image + # rM2 framebuffer is BGR (not RGBA), 32bpp but only 24 bits used + # Try BGR first, fall back to RGBA + try: + img = Image.frombytes("RGB", (width, height), raw_bytes, "raw", "BGR") + except Exception: + img = Image.frombytes("RGBA", (width, height), raw_bytes) + img = img.convert("RGB") + + # E-ink displays often have inverted or low-contrast content + # Auto-enhance contrast for better OCR + from PIL import ImageEnhance + enhancer = ImageEnhance.Contrast(img) + img = enhancer.enhance(3.0) + + return img + + +def ocr_image(image): + """Run PaddleOCR on a PIL Image, return recognized text.""" + from paddleocr import PaddleOCR + + ocr = PaddleOCR(use_textline_orientation=True, lang="en") + + # Save to temp file for PaddleOCR + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f: + image.save(f.name) + result = ocr.predict(f.name) + + # Extract text from results + texts = [] + for item in result: + if hasattr(item, 'rec_texts'): + texts.extend(item.rec_texts) + elif isinstance(item, dict) and 'rec_texts' in item: + texts.extend(item['rec_texts']) + + return " ".join(texts) + + +def main(): + parser = argparse.ArgumentParser(description="OCR pipeline for reMarkable") + parser.add_argument("--region", type=str, default=None, + help="Crop region as x,y,w,h (pixels)") + parser.add_argument("--output", type=str, default=None, + help="Save recognized text to file") + parser.add_argument("--image", type=str, default=None, + help="Use local image instead of capturing framebuffer") + args = parser.parse_args() + + if args.image: + from PIL import Image + img = Image.open(args.image).convert("RGB") + text = ocr_image(img) + else: + region = None + if args.region: + region = tuple(int(x) for x in args.region.split(",")) + + print(f"Capturing framebuffer (region={region})...") + raw, w, h = capture_framebuffer(region) + print(f"Captured {w}x{h} ({len(raw)} bytes)") + + img = raw_to_image(raw, w, h) + text = ocr_image(img) + + print(f"Recognized text: {text}") + + if args.output: + Path(args.output).write_text(text) + print(f"Saved to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/xovi-ext/grimoire-injector/GrimoireInjector.cpp b/xovi-ext/grimoire-injector/GrimoireInjector.cpp index d0623e0..05813bc 100644 --- a/xovi-ext/grimoire-injector/GrimoireInjector.cpp +++ b/xovi-ext/grimoire-injector/GrimoireInjector.cpp @@ -26,8 +26,24 @@ #include #include #include +#include +#include +#include + +static void grimoire_log(const char *fmt, ...) { + FILE *fp = fopen("/tmp/grimoire_ext.log", "a"); + if (fp) { + va_list ap; + va_start(ap, fmt); + vfprintf(fp, fmt, ap); + va_end(ap); + fprintf(fp, "\n"); + fclose(fp); + } +} static GrimoireInjector *g_instance = nullptr; +static PenIdleWatcher *g_idleWatcher = nullptr; /* Provided by entry.c hooks */ extern "C" { @@ -38,9 +54,28 @@ extern "C" { } static void *watchThreadFunc(void *) { - fprintf(stderr, "[grimoire] Watch thread started\n"); + grimoire_log("Watch thread started"); sleep(5); // Wait for xochitl to fully initialize + /* Start the pen idle watcher now that QGuiApplication should exist. + * Must invoke on main thread since installEventFilter is not thread-safe. + * Retry a few times in case startup is slow. */ + bool idleWatcherStarted = false; + for (int attempt = 0; attempt < 10 && !idleWatcherStarted; attempt++) { + if (g_idleWatcher && qobject_cast(QCoreApplication::instance())) { + QMetaObject::invokeMethod(g_idleWatcher, "activate", Qt::QueuedConnection); + idleWatcherStarted = true; + grimoire_log("PenIdleWatcher activate queued (attempt %d)", attempt + 1); + } else { + grimoire_log("Waiting for QGuiApplication (attempt %d, g_idleWatcher=%p, app=%p)...", + attempt + 1, (void*)g_idleWatcher, (void*)QGuiApplication::instance()); + sleep(2); + } + } + if (!idleWatcherStarted) { + grimoire_log("WARNING: PenIdleWatcher failed to start!"); + } + const char *path = "/tmp/grimoire_strokes.json"; long long lastMod = 0; @@ -51,6 +86,26 @@ static void *watchThreadFunc(void *) { lastMod = 0; // Force re-read of stroke file } + /* Check for screenshot trigger */ + if (access("/tmp/grimoire_screenshot", F_OK) == 0) { + unlink("/tmp/grimoire_screenshot"); + int sw, sh, sbpl, sfmt; + void *fb = grimoire_getFramebuffer(&sw, &sh, &sbpl, &sfmt); + if (fb && sw > 0 && sh > 0) { + const char *outpath = "/tmp/grimoire_fb.raw"; + FILE *fp = fopen(outpath, "wb"); + if (fp) { + size_t size = (size_t)sbpl * sh; + fwrite(fb, 1, size, fp); + fclose(fp); + fprintf(stderr, "[grimoire] Screenshot saved: %dx%d bpl=%d (%zu bytes)\n", + sw, sh, sbpl, size); + } + } else { + fprintf(stderr, "[grimoire] Screenshot requested but no framebuffer available\n"); + } + } + struct stat st; if (stat(path, &st) == 0 && st.st_mtime != lastMod) { lastMod = st.st_mtime; @@ -70,6 +125,13 @@ GrimoireInjector::GrimoireInjector(QObject *parent) : QObject(parent) { g_instance = this; + + /* Start pen idle watcher — detects when user stops writing. + * QGuiApplication may not exist yet during _xovi_construct, so + * we defer event filter installation to the watch thread which + * retries until the app is ready. */ + g_idleWatcher = new PenIdleWatcher(this); + pthread_t tid; pthread_create(&tid, nullptr, watchThreadFunc, nullptr); pthread_detach(tid); @@ -306,3 +368,76 @@ bool GrimoireInjector::setupVtable() { fprintf(stderr, "[grimoire] Vtable ready\n"); return true; } + +/* ─── PenIdleWatcher ─────────────────────────────────────────────── */ + +static long long nowMs() { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (long long)ts.tv_sec * 1000 + ts.tv_nsec / 1000000; +} + +PenIdleWatcher::PenIdleWatcher(QObject *parent) + : QObject(parent), m_debounceThread(0), m_running(false), m_lastLiftMs(0) +{ +} + +void PenIdleWatcher::activate() { + if (m_running.exchange(true)) return; + + auto *app = qobject_cast(QCoreApplication::instance()); + if (!app) { + grimoire_log("PenIdleWatcher::activate: no QGuiApplication!"); + m_running = false; + return; + } + + app->installEventFilter(this); + grimoire_log("PenIdleWatcher: event filter installed"); + + pthread_create(&m_debounceThread, nullptr, debounceThreadFunc, this); + pthread_detach(m_debounceThread); + grimoire_log("PenIdleWatcher: debounce thread started (debounce=%dms)", DEBOUNCE_MS); +} + +bool PenIdleWatcher::eventFilter(QObject * /*obj*/, QEvent *event) { + // reMarkable delivers pen input as touch + mouse events + if (event->type() == QEvent::MouseButtonPress || + event->type() == QEvent::TouchBegin || + event->type() == QEvent::TouchUpdate) { + m_penDown = true; + } else if (event->type() == QEvent::MouseButtonRelease || + event->type() == QEvent::TouchEnd) { + m_penDown = false; + m_lastLiftMs = nowMs(); + } + return false; +} + +void *PenIdleWatcher::debounceThreadFunc(void *arg) { + auto *self = static_cast(arg); + fprintf(stderr, "[grimoire] PenIdleWatcher debounce thread running\n"); + + while (self->m_running) { + long long lift = self->m_lastLiftMs.load(); + if (lift > 0 && !self->m_penDown.load()) { + long long elapsed = nowMs() - lift; + if (elapsed >= DEBOUNCE_MS) { + self->writeIdleSignal(); + self->m_lastLiftMs = 0; // reset so we don't re-fire + } + } + usleep(250000); // check every 250ms + } + return nullptr; +} + +void PenIdleWatcher::writeIdleSignal() { + FILE *fp = fopen(IDLE_PATH, "w"); + if (fp) { + long long ts = nowMs(); + fprintf(fp, "%lld\n", ts); + fclose(fp); + fprintf(stderr, "[grimoire] Idle signal written (pen up for %dms)\n", DEBOUNCE_MS); + } +} diff --git a/xovi-ext/grimoire-injector/GrimoireInjector.hpp b/xovi-ext/grimoire-injector/GrimoireInjector.hpp index c1a8728..dafd705 100644 --- a/xovi-ext/grimoire-injector/GrimoireInjector.hpp +++ b/xovi-ext/grimoire-injector/GrimoireInjector.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "rm_SceneItem.hpp" #include "rm_Line.hpp" @@ -26,3 +27,32 @@ private: QString m_watchPath = "/tmp/grimoire_strokes.json"; qint64 m_lastModTime = 0; }; + +/** + * Event filter that detects pen-lift (TabletRelease) and writes an idle + * signal file after a debounce period. Installed on QGuiApplication. + */ +class PenIdleWatcher : public QObject { + Q_OBJECT +public: + explicit PenIdleWatcher(QObject *parent = nullptr); + +public slots: + /** Must be called on the main thread. Installs event filter on QGuiApplication. */ + void activate(); + +protected: + bool eventFilter(QObject *obj, QEvent *event) override; + +private: + std::atomic m_penDown{false}; + pthread_t m_debounceThread; + std::atomic m_running{false}; + std::atomic m_lastLiftMs{0}; + + static constexpr int DEBOUNCE_MS = 4000; + static constexpr const char *IDLE_PATH = "/tmp/grimoire_idle"; + + static void *debounceThreadFunc(void *arg); + void writeIdleSignal(); +}; diff --git a/xovi-ext/grimoire-injector/entry.c b/xovi-ext/grimoire-injector/entry.c index 39e2e39..5c1a36e 100644 --- a/xovi-ext/grimoire-injector/entry.c +++ b/xovi-ext/grimoire-injector/entry.c @@ -57,6 +57,9 @@ void override$_ZN6QImageC1EPhiiiNS_6FormatEPFvPvES2_( fprintf(stderr, "[grimoire] Framebuffer captured: %p %dx%d bpl=%d fmt=%d\n", data, w, h, bpl, fmt); } + + /* Screenshot trigger is now handled by the watch thread in GrimoireInjector */ + $_ZN6QImageC1EPhiiiNS_6FormatEPFvPvES2_(self, data, w, h, bpl, fmt, cleanup, info); } @@ -91,6 +94,15 @@ void _xovi_construct() { if (strstr(program_invocation_short_name, "worker") != NULL) { return; } + + /* Write proof-of-life to a file since stderr may be swallowed */ + FILE *log = fopen("/tmp/grimoire_ext.log", "a"); + if (log) { + fprintf(log, "[%d] _xovi_construct called for %s\n", + getpid(), program_invocation_short_name); + fclose(log); + } + printf("[grimoire] Main process (%s), registering hooks + singleton\n", program_invocation_short_name); -- 2.51.2