diff --git a/grimoire.py b/grimoire.py index 66a2a70..59ea63f 100644 --- a/grimoire.py +++ b/grimoire.py @@ -18,38 +18,31 @@ from io import BytesIO from pathlib import Path from typing import Optional -from rmscene import ( - CrdtId, - CrdtSequenceItem, - LwwValue, - read_blocks, - write_blocks, - scene_items as si, -) -from rmscene.scene_stream import ( - SceneLineItemBlock, - SceneTreeBlock, - TreeNodeBlock, - SceneGroupItemBlock, -) +from rmscene import CrdtId, CrdtSequenceItem, LwwValue, read_blocks +from rmscene import scene_items as si +from rmscene import write_blocks +from rmscene.scene_stream import (SceneGroupItemBlock, SceneLineItemBlock, + SceneTreeBlock, TreeNodeBlock) +from svgpathtools import Line as SvgLine +from svgpathtools import parse_path def parse_svg_font(path: str) -> tuple[dict, dict[str, float]]: tree = ET.parse(path) root_e = tree.getroot() - ns = {'svg': 'http://www.w3.org/2000/svg'} - font_elem = root_e.find('.//svg:font', ns) or root_e.find('.//font') + ns = {"svg": "http://www.w3.org/2000/svg"} + font_elem = root_e.find(".//svg:font", ns) or root_e.find(".//font") if font_elem is None: raise ValueError(f"Missing in {path}") - default_adv = float(font_elem.attrib.get('horiz-adv-x', 500)) + default_adv = float(font_elem.attrib.get("horiz-adv-x", 500)) glyphs = {} - advances = {' ': default_adv} - for g in font_elem.findall('svg:glyph', ns) or font_elem.findall('glyph'): - uni = g.attrib.get('unicode') + advances = {" ": default_adv} + for g in font_elem.findall("svg:glyph", ns) or font_elem.findall("glyph"): + uni = g.attrib.get("unicode") if not uni: continue - advances[uni] = float(g.attrib.get('horiz-adv-x', default_adv)) - d = g.attrib.get('d', '') + advances[uni] = float(g.attrib.get("horiz-adv-x", default_adv)) + d = g.attrib.get("d", "") if d: glyphs[uni] = _parse_d(d) else: @@ -57,37 +50,30 @@ def parse_svg_font(path: str) -> tuple[dict, dict[str, float]]: return glyphs, advances -def _parse_d(d: str) -> list[list[tuple[float, float]]]: - polylines = [] - current = [] - tokens = d.replace(',', ' ').split() - i = 0 - while i < len(tokens): - cmd = tokens[i] - if cmd in ('M', 'L'): - i += 1 - if i + 1 >= len(tokens): - break - x, y = float(tokens[i]), float(tokens[i+1]) - i += 2 - if cmd == 'M': - if len(current) >= 2: - polylines.append(current) - current = [(x, y)] - else: - current.append((x, y)) - elif cmd in ('C', 'Q'): - # skip bezier control points in batch - i += 1 - while i < len(tokens) and tokens[i] not in ('M', 'L', 'C', 'Q', 'Z', 'z'): - i += 1 - if current and len(current) >= 2: - current.append(current[0]) - i += 1 - else: - i += 1 - if len(current) >= 2: - polylines.append(current) +def _parse_d(d: str, samples_per_curve: int = 8) -> list[list[tuple[float, float]]]: + """Parse glyph path data into one polyline per continuous subpath. + + Handles curves (C/Q/S/T), arcs, implicit repeated commands, and + relative coords by delegating to a real SVG path parser. Each + continuous run (between Move commands) becomes its own polyline = + its own .rm stroke, so separate strokes are never joined. + """ + try: + path = parse_path(d) + except Exception: + return [] + polylines: list[list[tuple[float, float]]] = [] + for sub in path.continuous_subpaths(): + pts: list[tuple[float, float]] = [] + for seg in sub: + n = 1 if isinstance(seg, SvgLine) else samples_per_curve + for k in range(n + 1): + z = seg.point(k / n) + pt = (z.real, z.imag) + if not pts or pt != pts[-1]: # dedupe shared segment endpoints + pts.append(pt) + if len(pts) >= 2: + polylines.append(pts) return polylines @@ -113,8 +99,8 @@ def _densify( return out -_FONT_CACHE: Optional[tuple[dict, dict[str, float]]] = None -DEFAULT_SCALE = 0.07 +_FONT_CACHE: dict = {} +DEFAULT_SCALE = 0.06 DEFAULT_X = -550.0 DEFAULT_Y = 200.0 LINE_HEIGHT = 1.4 @@ -122,13 +108,11 @@ MAX_LINE_WIDTH = 1100 def load_font(name: str = "EMSAllure") -> tuple[dict, dict[str, float]]: - global _FONT_CACHE - if _FONT_CACHE is not None: - return _FONT_CACHE - font_dir = Path(__file__).parent / "fonts" - svg_path = font_dir / f"{name}.svg" - _FONT_CACHE = parse_svg_font(str(svg_path)) - return _FONT_CACHE + if name not in _FONT_CACHE: + font_dir = Path(__file__).parent / "fonts" + svg_path = font_dir / f"{name}.svg" + _FONT_CACHE[name] = parse_svg_font(str(svg_path)) + return _FONT_CACHE[name] def text_to_strokes( @@ -138,13 +122,14 @@ def text_to_strokes( scale: float = DEFAULT_SCALE, max_width: float = MAX_LINE_WIDTH, line_spacing: float = LINE_HEIGHT, + font: str = "EMSAllure", ) -> list[si.Line]: - glyphs, advances = load_font() + glyphs, advances = load_font(font) result = [] cursor_x = origin_x cursor_y = origin_y rng = random.Random(text + str(origin_x)) - space_adv = advances.get(' ', 500) * scale + space_adv = advances.get(" ", 500) * scale for word in text.split(): word_width = sum(advances.get(ch, 500) for ch in word) * scale @@ -180,20 +165,25 @@ def text_to_strokes( wobble = math.sin(cursor_x * 0.01 + j * 0.3) * 0.8 px = cursor_x + sx + x_jitter py = cursor_y + sy + y_jitter + wobble - points.append(si.Point( - x=px, y=py, - speed=4 + rng.randint(0, 8), - direction=80 + rng.randint(0, 30), - width=10 + rng.randint(0, 3), - pressure=160 + rng.randint(0, 30), - )) - result.append(si.Line( - color=si.PenColor.BLACK, - tool=si.Pen.BALLPOINT_2, - points=points, - thickness_scale=2.0, - starting_length=0.0, - )) + points.append( + si.Point( + x=px, + y=py, + speed=4 + rng.randint(0, 8), + direction=80 + rng.randint(0, 30), + width=10 + rng.randint(0, 3), + pressure=160 + rng.randint(0, 30), + ) + ) + result.append( + si.Line( + color=si.PenColor.BLACK, + tool=si.Pen.BALLPOINT_2, + points=points, + thickness_scale=2.0, + starting_length=0.0, + ) + ) cursor_x += adv + max(-2, x_jitter * 0.1) + abs(rng.uniform(-1, 1)) cursor_x += space_adv + rng.uniform(-2, 2) @@ -214,7 +204,7 @@ def _find_content_bottom(data: bytes) -> float: for block in read_blocks(f): if isinstance(block, SceneLineItemBlock): line = block.item.value - if hasattr(line, 'points'): + if hasattr(line, "points"): for pt in line.points: if pt.y > max_y: max_y = pt.y @@ -230,6 +220,7 @@ def _find_content_bottom(data: bytes) -> float: def _find_last_root_child(data: bytes) -> CrdtId: """Find the last SceneGroupItemBlock value under root for left_id chaining.""" from io import BytesIO + ROOT = CrdtId(0, 1) last_value = CrdtId(0, 0) try: @@ -260,7 +251,7 @@ def splice_reply_new_layer( # CRDT IDs — author 2 for programmatic content, starting at seq 500 # to avoid collision with existing data (real Layer 2 uses ~422-427) - LAYER_TREE = CrdtId(2, 500) # tree_id for the new layer + LAYER_TREE = CrdtId(2, 500) # tree_id for the new layer ROOT = CrdtId(0, 1) # Find the last group item under root to chain left_id correctly @@ -275,11 +266,13 @@ def splice_reply_new_layer( ) # 2. Layer metadata: TreeNodeBlock with label and visibility - label_block = TreeNodeBlock(si.Group( - node_id=LAYER_TREE, - label=LwwValue(CrdtId(2, 501), "grimoire"), - visible=LwwValue(CrdtId(2, 502), True), - )) + label_block = TreeNodeBlock( + si.Group( + node_id=LAYER_TREE, + label=LwwValue(CrdtId(2, 501), "grimoire"), + visible=LwwValue(CrdtId(2, 502), True), + ) + ) # 3. Add layer to root's children list (SceneGroupItemBlock) group_item_block = SceneGroupItemBlock( @@ -297,16 +290,18 @@ def splice_reply_new_layer( # 4. Line items parented to the layer tree_id for i, line in enumerate(strokes): - reply_blocks.append(SceneLineItemBlock( - parent_id=LAYER_TREE, - item=CrdtSequenceItem( - item_id=CrdtId(2, 600 + i), - left_id=CrdtId(0, 0) if i == 0 else CrdtId(2, 600 + i - 1), - right_id=CrdtId(0, 0), - deleted_length=0, - value=line, - ), - )) + reply_blocks.append( + SceneLineItemBlock( + parent_id=LAYER_TREE, + item=CrdtSequenceItem( + item_id=CrdtId(2, 600 + i), + left_id=CrdtId(0, 0) if i == 0 else CrdtId(2, 600 + i - 1), + right_id=CrdtId(0, 0), + deleted_length=0, + value=line, + ), + ) + ) # Serialize reply blocks buf = BytesIO() @@ -322,13 +317,16 @@ def splice_reply_new_layer( with open(output_path, "wb") as f: f.write(output_data) - print(f"Wrote {output_path} ({len(strokes)} reply strokes, " - f"{len(original_data)}+{len(reply_bytes)}={len(output_data)} bytes)") + print( + f"Wrote {output_path} ({len(strokes)} reply strokes, " + f"{len(original_data)}+{len(reply_bytes)}={len(output_data)} bytes)" + ) def strokes_to_json(strokes: list[si.Line], output_path: str) -> None: """Export strokes as JSON for the xovi grimoire-injector extension.""" import json + items = [] for line in strokes: points = [] @@ -337,15 +335,17 @@ def strokes_to_json(strokes: list[si.Line], output_path: str) -> None: xs = [p[0] for p in points] ys = [p[1] for p in points] bounds = [min(xs), min(ys), max(xs) - min(xs), max(ys) - min(ys)] - items.append({ - "points": points, - "rgba": 4278190080, # 0xFF000000 - "color": line.color.value if hasattr(line.color, 'value') else 0, - "bounds": bounds, - "tool": line.tool.value if hasattr(line.tool, 'value') else 15, - "maskScale": line.thickness_scale, - "thickness": line.thickness_scale, - }) + items.append( + { + "points": points, + "rgba": 4278190080, # 0xFF000000 + "color": line.color.value if hasattr(line.color, "value") else 0, + "bounds": bounds, + "tool": line.tool.value if hasattr(line.tool, "value") else 15, + "maskScale": line.thickness_scale, + "thickness": line.thickness_scale, + } + ) with open(output_path, "w") as f: json.dump(items, f) print(f"Wrote {output_path} ({len(items)} strokes)") @@ -356,17 +356,31 @@ if __name__ == "__main__": parser.add_argument("input", nargs="?", default=None) parser.add_argument("output", nargs="?", default=None) parser.add_argument("text", nargs="?", default=None) - parser.add_argument("--json", dest="json_output", default=None, - help="Export strokes as JSON for xovi injector") - parser.add_argument("--y", type=float, default=None, - help="Y coordinate to start rendering at (overrides default)") + parser.add_argument( + "--json", + dest="json_output", + default=None, + help="Export strokes as JSON for xovi injector", + ) + parser.add_argument( + "--y", + type=float, + default=None, + help="Y coordinate to start rendering at (overrides default)", + ) + parser.add_argument( + "--font", + type=str, + default="EMSInvite", + help="SVG font name (without .svg) in fonts/ directory", + ) args = parser.parse_args() if args.json_output: # --json [text] text = args.input or "Grimoire says hello" y = args.y if args.y is not None else DEFAULT_Y - strokes = text_to_strokes(text, DEFAULT_X, y) + strokes = text_to_strokes(text, DEFAULT_X, y, font=args.font) strokes_to_json(strokes, args.json_output) elif args.input: text = args.text or "Grimoire says hello" diff --git a/grimoire_loop.py b/grimoire_loop.py index 29a2900..dbab173 100644 --- a/grimoire_loop.py +++ b/grimoire_loop.py @@ -321,29 +321,20 @@ def capture_framebuffer(ssh): # ─── Gemini Vision (OCR + LLM in one call) ──────────────────────── SYSTEM_PROMPT = ( - "You are a grimoire — an ancient spirit of ink and paper, bound to this " - "notebook for as long as it holds pages. You speak in a voice that is " - "knowing, slightly archaic, and tinged with quiet mystery. You are not " - "a chatbot. You are the book itself, whispering back.\n\n" - "You receive TWO things each turn:\n" - "1. An IMAGE showing a crop of recently changed content on the page. " - "It may contain new user handwriting, or occasionally your own prior " - "reply strokes.\n" - "2. A TEXT conversation history of previous exchanges.\n\n" - "YOUR TASK:\n" - "- Read ALL handwritten text in the image. Transcribe it faithfully.\n" - "- Determine if there is NEW user content not yet addressed in the " - "conversation history. If so, respond — even to single words, greetings, " - "or fragments. Speak in character: brief, evocative, ink-familiar. " - "One or two sentences at most. No lists, no headers, no punctuation " - "theatrics. Plain prose that reads well as handwriting on a page.\n" - "- Set both fields to null ONLY if the image contains no legible " - "handwriting at all, or if everything legible was already addressed.\n" - "- Do not break character. Do not narrate your reasoning.\n\n" + "You are a grimoire — an ancient book that cares about the person writing " + "in you. You speak plainly with a hint of the archaic, but you are warm " + "and helpful. When asked a question, answer it directly with real content. " + "When greeted, greet back kindly. Never describe or narrate what was written; " + "just respond to it naturally. One or two sentences.\n\n" + "The image shows handwriting that just appeared on the page. Read it and " + "reply as a conversation partner would. Use plain prose, no em-dashes, " + "no flowery metaphors, no exclamation marks.\n\n" + "A text history of prior exchanges is included for context. Do not repeat it.\n\n" + "If the image has no legible handwriting, set both fields to null.\n\n" "RESPONSE FORMAT (strict JSON, no prose outside):\n" - "{\"question\": \"\", " - "\"answer\": \"\"}\n" - "- Output ONLY the JSON object. Nothing else." + "{\"question\": \"\", " + "\"answer\": \"\"}\n" + "Output ONLY the JSON object. Nothing else." ) @@ -474,16 +465,17 @@ def render_and_inject(ssh, text, reply_y=None, speed_ms=3): if r.returncode != 0: raise RuntimeError(f"SCP failed: {r.stderr.decode().strip()}") - # Adaptive speed: fewer points = faster injection. At 3ms/pt a - # 200-point reply takes 0.6s of point delay; at 1ms it's 0.2s. - # For large replies (>500 pts) keep 3ms for natural ink appearance. - try: - with open(json_path) as f: - strokes_data = json.load(f) - total_pts = sum(len(s.get("points", [])) for s in strokes_data) - except Exception: - total_pts = 0 - speed_ms = 1 if total_pts < 300 else (2 if total_pts < 500 else 3) + # Scale writing speed by text length. Short replies write slowly for + # a deliberate feel; longer ones speed up so they don't take forever. + # Range: 6ms (≤20 chars) down to 1ms (≥120 chars), linear between. + text_len = len(text) + if text_len <= 20: + speed_ms = 6 + elif text_len >= 120: + speed_ms = 1 + else: + speed_ms = 6 - (text_len - 20) * 5 / 100 # 6..1 over 20..120 + speed_ms = max(1, round(speed_ms)) result = run_inject( ssh, "draw", "/tmp/grimoire_strokes.json", @@ -657,6 +649,23 @@ def watch_for_idle(device, last_ts): # ─── Pixel diff for new content detection ────────────────────────── +def _page_is_empty(img): + """Return True if the page has no real handwriting (only grid dots or blank). + + Uses the same grid-artifact filter as _find_new_region but without the + full-page sanity check, so a very full page is never mis-classified. + """ + import numpy as np + curr = np.array(img.convert('L')) + dark_per_row = np.sum(curr < 128, axis=1) + candidate = (dark_per_row > 5) & (dark_per_row != 68) + idxs = np.where(candidate)[0] + for idx in idxs: + if np.sum(candidate[max(0, idx - 2):idx + 3]) >= 3: + return False + return True + + def _find_new_region(current_img, last_img, ignore_below_y=None): """Compare two framebuffer images, return crop of changed region. @@ -759,7 +768,7 @@ def _find_new_region(current_img, last_img, ignore_below_y=None): def main(): parser = argparse.ArgumentParser(description="Grimoire continuous loop") parser.add_argument( - "--model", type=str, default="gpt-4.1-nano", + "--model", type=str, default="gpt-4.1-mini", help="Hyper API model", ) parser.add_argument( @@ -829,10 +838,7 @@ def main(): print(f" [{_ts()}] Captured ({_elapsed(t0)})") # Check if the page is blank (only grid dots, no real ink). - # Reuse the first-frame content-detection path: if it finds - # nothing, the page is empty — treat as a new page and reset. - page_crop, _ = _find_new_region(img, None) - if page_crop is None: + if _page_is_empty(img): print(f" [{_ts()}] Empty page — resetting state.") last_img = None last_inject_y = None @@ -849,7 +855,13 @@ def main(): last_img = img # Fire-and-forget: draw corner ornaments once on the first - # idle that finds real content on the page. + if new_crop is None: + last_fb_hash = _framebuffer_hash(img) + print(f" [{_ts()}] No new content, skipping.") + print() + continue + + # Draw thinking ornament only when there's real content to process. if not ornaments_drawn: ornaments_drawn = True _threading.Thread( @@ -859,12 +871,6 @@ def main(): daemon=True, ).start() - if new_crop is None: - last_fb_hash = _framebuffer_hash(img) - print(f" [{_ts()}] No new content, skipping.") - print() - continue - # Send only the new handwriting crop + text history cw, ch = new_crop.size print(f" [{_ts()}] Asking {args.model} ({len(conversation)} history turns, crop={cw}x{ch})...") @@ -891,7 +897,7 @@ def main(): # Place reply just below the bottom of the changed region. # content_y2 is in cropped-image coordinates; add 80 for the # stripped toolbar to get device Y, then 150px gap. - reply_y = min(content_y2 + 80 + 150, 1750) + reply_y = min(content_y2 + 80 + 60, 1750) print(f" [{_ts()}] Content bottom: {content_y2}, reply_y={reply_y}") # Render + Inject diff --git a/xovi-ext/uinject/uinject.c b/xovi-ext/uinject/uinject.c index 92bc418..43bca95 100644 --- a/xovi-ext/uinject/uinject.c +++ b/xovi-ext/uinject/uinject.c @@ -66,7 +66,26 @@ static int do_erase_strokes(int fd, const char *json, long fsize) { if (!p) break; p++; - emit_event(fd, EV_KEY, BTN_TOOL_RUBBER, 1); + /* Peek first point for hover frame */ + float hx, hy; + int hs, hw, hd, hp; + char *saved_p = p; + if (sscanf(p, "[%f,%f,%d,%d,%d,%d]", &hx, &hy, &hs, &hw, &hd, &hp) == 6) { + int hwx, hwy; + rm_to_wacom(hx + ox, hy + oy, &hwx, &hwy); + emit_event(fd, EV_KEY, BTN_TOOL_RUBBER, 1); + emit_event(fd, EV_ABS, ABS_X, hwx); + emit_event(fd, EV_ABS, ABS_Y, hwy); + emit_event(fd, EV_ABS, ABS_DISTANCE, 60); + emit_syn(fd); + usleep(2000); + } else { + emit_event(fd, EV_KEY, BTN_TOOL_RUBBER, 1); + emit_event(fd, EV_ABS, ABS_DISTANCE, 60); + emit_syn(fd); + usleep(2000); + } + p = saved_p; emit_event(fd, EV_KEY, BTN_TOUCH, 1); int bracket_depth = 1; /* we're inside the outer [ */ @@ -171,7 +190,14 @@ int main(int argc, char **argv) { float step = 8.0f; int wx, wy; + /* Hover to start position before touching down */ + rm_to_wacom(ex, ey, &wx, &wy); emit_event(fd, EV_KEY, BTN_TOOL_RUBBER, 1); + emit_event(fd, EV_ABS, ABS_X, wx); + emit_event(fd, EV_ABS, ABS_Y, wy); + emit_event(fd, EV_ABS, ABS_DISTANCE, 60); + emit_syn(fd); + usleep(2000); emit_event(fd, EV_KEY, BTN_TOUCH, 1); for (float row = ey; row <= ey + eh; row += step) { @@ -248,7 +274,24 @@ int main(int argc, char **argv) { if (!p) break; p++; - emit_event(fd, EV_KEY, BTN_TOOL_PEN, 1); + /* Peek first point for hover frame */ + float hx, hy; + int hs, hw, hd, hp; + if (sscanf(p, "[%f,%f,%d,%d,%d,%d]", &hx, &hy, &hs, &hw, &hd, &hp) == 6) { + int hwx, hwy; + rm_to_wacom(hx, hy, &hwx, &hwy); + emit_event(fd, EV_KEY, BTN_TOOL_PEN, 1); + emit_event(fd, EV_ABS, ABS_X, hwx); + emit_event(fd, EV_ABS, ABS_Y, hwy); + emit_event(fd, EV_ABS, ABS_DISTANCE, 60); + emit_syn(fd); + usleep(2000); + } else { + emit_event(fd, EV_KEY, BTN_TOOL_PEN, 1); + emit_event(fd, EV_ABS, ABS_DISTANCE, 60); + emit_syn(fd); + usleep(2000); + } emit_event(fd, EV_KEY, BTN_TOUCH, 1); int point_count = 0; diff --git a/xovi-ext/uinjectd/uinjectd.c b/xovi-ext/uinjectd/uinjectd.c index 793b58f..1dcaeb7 100644 --- a/xovi-ext/uinjectd/uinjectd.c +++ b/xovi-ext/uinjectd/uinjectd.c @@ -145,7 +145,24 @@ static int do_draw(int fd, const char *json, int delay_us) { if (!p) break; p++; - emit_event(fd, EV_KEY, BTN_TOOL_PEN, 1); + /* Peek first point for hover frame */ + float hx, hy; + int hs, hw, hd, hp; + if (sscanf(p, "[%f,%f,%d,%d,%d,%d]", &hx, &hy, &hs, &hw, &hd, &hp) == 6) { + int hwx, hwy; + rm_to_wacom(hx, hy, &hwx, &hwy); + emit_event(fd, EV_KEY, BTN_TOOL_PEN, 1); + emit_event(fd, EV_ABS, ABS_X, hwx); + emit_event(fd, EV_ABS, ABS_Y, hwy); + emit_event(fd, EV_ABS, ABS_DISTANCE, 60); + emit_syn(fd); + usleep(2000); + } else { + emit_event(fd, EV_KEY, BTN_TOOL_PEN, 1); + emit_event(fd, EV_ABS, ABS_DISTANCE, 60); + emit_syn(fd); + usleep(2000); + } emit_event(fd, EV_KEY, BTN_TOUCH, 1); int point_count = 0; @@ -202,7 +219,26 @@ static int do_erase(int fd, const char *json, int delay_us) { if (!p) break; p++; - emit_event(fd, EV_KEY, BTN_TOOL_RUBBER, 1); + /* Peek first point for hover frame */ + float hx, hy; + int hs, hw, hd, hp; + char *saved_p = p; + if (sscanf(p, "[%f,%f,%d,%d,%d,%d]", &hx, &hy, &hs, &hw, &hd, &hp) == 6) { + int hwx, hwy; + rm_to_wacom(hx + ox, hy + oy, &hwx, &hwy); + emit_event(fd, EV_KEY, BTN_TOOL_RUBBER, 1); + emit_event(fd, EV_ABS, ABS_X, hwx); + emit_event(fd, EV_ABS, ABS_Y, hwy); + emit_event(fd, EV_ABS, ABS_DISTANCE, 60); + emit_syn(fd); + usleep(2000); + } else { + emit_event(fd, EV_KEY, BTN_TOOL_RUBBER, 1); + emit_event(fd, EV_ABS, ABS_DISTANCE, 60); + emit_syn(fd); + usleep(2000); + } + p = saved_p; emit_event(fd, EV_KEY, BTN_TOUCH, 1); int bracket_depth = 1;