diff --git a/.mcp.json b/.mcp.json index e18ef039d8..37dcb642a8 100644 --- a/.mcp.json +++ b/.mcp.json @@ -76,6 +76,13 @@ "args": [ "toolchain/robloxplorer/testing-mcp.mjs" ] + }, + "aesel": { + "type": "stdio", + "command": "node", + "args": [ + "slab/bin/aesel-mcp.mjs" + ] } } } diff --git a/apple/aesel/Aesel.xcodeproj/project.pbxproj b/apple/aesel/Aesel.xcodeproj/project.pbxproj index f8420788b5..57923dd0a0 100644 --- a/apple/aesel/Aesel.xcodeproj/project.pbxproj +++ b/apple/aesel/Aesel.xcodeproj/project.pbxproj @@ -31,6 +31,7 @@ A4B8792BDBB5F95822088FB0 /* AeselNotebook.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB9C447ACCA3590492C05AC5 /* AeselNotebook.swift */; }; A766FD4831AD3177B8902DA5 /* ApplePlatform.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7F4BC5239A9B70F128C1DC84 /* ApplePlatform.swift */; }; A989A8D07DAA87AB483EC748 /* AeselButtonStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = A512F7F818F52E3E58D3B7CB /* AeselButtonStyle.swift */; }; + AC4159B335B9A16CFF670F02 /* AeselAutomation.swift in Sources */ = {isa = PBXBuildFile; fileRef = E270B97E651FC0013152574F /* AeselAutomation.swift */; }; AC7B6241A0E7F2791E06F1CA /* ComicRelief-Bold.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 99FE2FC14157DB3F88708A6B /* ComicRelief-Bold.ttf */; }; B7D0EF78D26BBA38AB801393 /* slab-theme.json in Resources */ = {isa = PBXBuildFile; fileRef = AACC6D25723505FB156B46F0 /* slab-theme.json */; }; B8236CE8844912F069905928 /* AeselScene.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CB1DC30F6BBB5B2167272D2 /* AeselScene.swift */; }; @@ -39,6 +40,7 @@ C48716DE150CDF3FE383298E /* Session in Resources */ = {isa = PBXBuildFile; fileRef = 47AE0766266534A8EE665542 /* Session */; }; C5323C1BB9D8D59F14A1FF74 /* AeselModelPicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = F26176753E43E445E2CF551D /* AeselModelPicker.swift */; }; C5E4BB3C192D944F7CB281FC /* Session in Resources */ = {isa = PBXBuildFile; fileRef = 47AE0766266534A8EE665542 /* Session */; }; + D1D4EE707AFBE4CB22C77682 /* AeselAutomation.swift in Sources */ = {isa = PBXBuildFile; fileRef = E270B97E651FC0013152574F /* AeselAutomation.swift */; }; E269C78DB87E568CC6F63CC8 /* AeselProviderPicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3405C669DD22957E6E9FC52B /* AeselProviderPicker.swift */; }; E9877D833B3CE6FE4FB14192 /* AeselButtonStyle.swift in Sources */ = {isa = PBXBuildFile; fileRef = A512F7F818F52E3E58D3B7CB /* AeselButtonStyle.swift */; }; EE331E96C91793CBF5E52157 /* COMIC-RELIEF-OFL-1.1.txt in Resources */ = {isa = PBXBuildFile; fileRef = 4A54F121629B5E78C6F46F6A /* COMIC-RELIEF-OFL-1.1.txt */; }; @@ -65,6 +67,7 @@ B7CE6904A010A2BEC894D13C /* AeselApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AeselApp.swift; sourceTree = ""; }; BB9C447ACCA3590492C05AC5 /* AeselNotebook.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AeselNotebook.swift; sourceTree = ""; }; CC2D2E4941E446673BAF4F97 /* AeselHomeView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AeselHomeView.swift; sourceTree = ""; }; + E270B97E651FC0013152574F /* AeselAutomation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AeselAutomation.swift; sourceTree = ""; }; F26176753E43E445E2CF551D /* AeselModelPicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AeselModelPicker.swift; sourceTree = ""; }; /* End PBXFileReference section */ @@ -73,6 +76,7 @@ isa = PBXGroup; children = ( B7CE6904A010A2BEC894D13C /* AeselApp.swift */, + E270B97E651FC0013152574F /* AeselAutomation.swift */, A512F7F818F52E3E58D3B7CB /* AeselButtonStyle.swift */, CC2D2E4941E446673BAF4F97 /* AeselHomeView.swift */, F26176753E43E445E2CF551D /* AeselModelPicker.swift */, @@ -230,6 +234,7 @@ buildActionMask = 2147483647; files = ( 2DB3100BC43BD21115A1BA37 /* AeselApp.swift in Sources */, + D1D4EE707AFBE4CB22C77682 /* AeselAutomation.swift in Sources */, A989A8D07DAA87AB483EC748 /* AeselButtonStyle.swift in Sources */, FBC8984584385405A83CB639 /* AeselHomeView.swift in Sources */, C5323C1BB9D8D59F14A1FF74 /* AeselModelPicker.swift in Sources */, @@ -250,6 +255,7 @@ buildActionMask = 2147483647; files = ( 4A5FFF82FBF2EF81CD8D158A /* AeselApp.swift in Sources */, + AC4159B335B9A16CFF670F02 /* AeselAutomation.swift in Sources */, E9877D833B3CE6FE4FB14192 /* AeselButtonStyle.swift in Sources */, 8A7F1BBCD12CDD5FE9707061 /* AeselHomeView.swift in Sources */, 2026E504A366EFEB32F2F822 /* AeselModelPicker.swift in Sources */, diff --git a/apple/aesel/README.md b/apple/aesel/README.md index e4d78f62f1..147efc37e5 100644 --- a/apple/aesel/README.md +++ b/apple/aesel/README.md @@ -113,3 +113,7 @@ models are automatic, including resumed threads with old manual choices. Dollar values come from the allowance endpoint and distinguish free from purchased credit. Account character colors come from AC's saved palette, including the `@` character, with the same fallback palette as desktop. + +## MCP and visual acceptance + +The native app exposes a private same-user automation mailbox. The monorepo adapter and Aesthetic Eye workflow live at `slab/bin/aesel-mcp.mjs`, `slab/bin/aesel-eye.mjs` and `slab/AESEL-EYE.md`. Tests inspect and act through stable UI control IDs without activating the window. Preview URLs use the same `nogap`, `nolabel` and `autoreload` contract as Electron; the footer shows the persisted piece revision starting at v0. diff --git a/apple/aesel/Sources/AeselAutomation.swift b/apple/aesel/Sources/AeselAutomation.swift new file mode 100644 index 0000000000..ec9dcf5bd2 --- /dev/null +++ b/apple/aesel/Sources/AeselAutomation.swift @@ -0,0 +1,183 @@ +import Foundation +import WebKit +import CryptoKit +#if os(macOS) +import AppKit +import ScreenCaptureKit +#endif + +/// Same-user RPC mailbox for the monorepo MCP adapter. No listening port, +/// arbitrary JavaScript, credentials, or automatic window activation. +@MainActor +final class AeselAutomation { + var inspect: (() -> [String: Any])? + var perform: ((String, [String: Any]) async throws -> Void)? + weak var preview: WKWebView? + weak var notebook: WKWebView? + var retryPreview: (() -> Void)? + var previewFailure: String? + private let directory: URL + private var timer: Timer? + private var servicing = false + private var sequence = 0 + private var events: [[String: Any]] = [] + private let instance = UUID().uuidString + private let buildSha256: String + + init() { + var hash = SHA256() + let bundle = Bundle.main.bundleURL + func visit(_ folder: URL) { + let files = (try? FileManager.default.contentsOfDirectory(at: folder, includingPropertiesForKeys: [.isDirectoryKey, .isRegularFileKey])) ?? [] + for file in files.sorted(by: { $0.lastPathComponent.utf8.lexicographicallyPrecedes($1.lastPathComponent.utf8) }) { + if file.lastPathComponent == "_CodeSignature" { continue } + let values = try? file.resourceValues(forKeys: [.isDirectoryKey, .isRegularFileKey]) + if values?.isDirectory == true { visit(file) } + else if values?.isRegularFile == true, let data = try? Data(contentsOf: file) { + hash.update(data: Data(file.path.dropFirst(bundle.path.count + 1).utf8)) + hash.update(data: Data([0])) + hash.update(data: data) + } + } + } + visit(bundle) + buildSha256 = hash.finalize().map { String(format: "%02x", $0) }.joined() + let requested = ProcessInfo.processInfo.environment["AESEL_AUTOMATION_NAMESPACE"] ?? "" + let namespace = requested.range(of: "^[a-z0-9-]{1,32}$", options: .regularExpression) != nil ? "-" + requested : "" + directory = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + .appendingPathComponent(Bundle.main.bundleIdentifier ?? "computer.aesthetic.aesel.native") + .appendingPathComponent("automation" + namespace) + } + + func start() { + guard timer == nil else { return } + do { + for path in [directory, directory.appendingPathComponent("requests"), directory.appendingPathComponent("responses")] { + try FileManager.default.createDirectory(at: path, withIntermediateDirectories: true, attributes: [.posixPermissions: 0o700]) + } + try write(["schema": 1, "pid": ProcessInfo.processInfo.processIdentifier, "instance": instance, + "bundle": Bundle.main.bundleIdentifier ?? "", "startedAt": Date().timeIntervalSince1970], to: directory.appendingPathComponent("instance.json")) + } catch { NSLog("Aesel automation startup failed: %@", error.localizedDescription); return } + timer = Timer.scheduledTimer(withTimeInterval: 0.15, repeats: true) { [weak self] _ in + Task { @MainActor in await self?.poll() } + } + record("automation.ready") + } + + func record(_ kind: String) { + sequence += 1 + events.append(["sequence": sequence, "at": Date().timeIntervalSince1970, "kind": String(kind.prefix(100))]) + if events.count > 256 { events.removeFirst(events.count - 256) } + } + + private func write(_ value: [String: Any], to file: URL) throws { + let data = try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]) + try data.write(to: file, options: .atomic) + try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: file.path) + } + + private func poll() async { + guard !servicing else { return } + servicing = true + defer { servicing = false } + let files = (try? FileManager.default.contentsOfDirectory(at: directory.appendingPathComponent("requests"), includingPropertiesForKeys: [.fileSizeKey, .isSymbolicLinkKey])) ?? [] + for file in files.sorted(by: { $0.lastPathComponent < $1.lastPathComponent }).prefix(8) { + guard file.pathExtension == "json", UUID(uuidString: file.deletingPathExtension().lastPathComponent) != nil, + let info = try? file.resourceValues(forKeys: [.fileSizeKey, .isSymbolicLinkKey]), info.isSymbolicLink != true, + (info.fileSize ?? 0) <= 65536, let data = try? Data(contentsOf: file), + let request = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any] else { continue } + try? FileManager.default.removeItem(at: file) + let id = file.deletingPathExtension().lastPathComponent + guard request["id"] as? String == id, request["instance"] as? String == instance, + let created = request["createdAt"] as? Double, abs(Date().timeIntervalSince1970 - created) < 30 else { continue } + let result: [String: Any] + do { + let method = request["method"] as? String ?? "" + let params = request["params"] as? [String: Any] ?? [:] + result = ["id": id, "result": try await handle(method, params)] + } catch { + result = ["id": id, "error": error.localizedDescription] + } + try? write(result, to: directory.appendingPathComponent("responses/\(id).json")) + } + } + + private func handle(_ method: String, _ params: [String: Any]) async throws -> [String: Any] { + switch method { + case "state", "map": + var state = inspect?() ?? [:] + state["instance"] = instance + state["buildSha256"] = buildSha256 + state["bundlePath"] = Bundle.main.bundleURL.path + state["pid"] = ProcessInfo.processInfo.processIdentifier + state["appVersion"] = Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "" + state["telemetrySequence"] = sequence + state["previewFailure"] = previewFailure ?? "" + #if os(macOS) + if let window = NSApp.windows.first(where: { !($0 is NSPanel) && $0.contentView != nil }) { + state["window"] = ["number": window.windowNumber, "width": window.contentView!.bounds.width, "height": window.contentView!.bounds.height, "visible": window.isVisible] + } + #endif + return state + case "action": + guard let action = params["id"] as? String, let perform else { throw failure("UI is not ready") } + try await perform(action, params) + record("action.\(action)") + return ["accepted": true, "action": action, "state": inspect?() ?? [:]] + case "events": + let after = params["after"] as? Int ?? 0 + return ["events": events.filter { ($0["sequence"] as? Int ?? 0) > after }, "latest": sequence, + "oldest": events.first?["sequence"] ?? sequence, "capacity": 256] + case "preview": + guard let preview else { throw failure("Preview is not mounted") } + let value = try await preview.evaluateJavaScript("JSON.stringify({url:location.href,ready:!!window.preloaded,flags:Object.fromEntries(new URLSearchParams(location.search)),canvases:[...document.querySelectorAll('canvas')].map(c=>({width:c.width,height:c.height}))})") + return ["inspection": value as? String ?? "{}"] + case "capture": + let targetName = params["target"] as? String ?? "app" + guard ["app", "preview", "notebook", "window"].contains(targetName) else { throw failure("Unknown capture target") } + if targetName == "preview" || targetName == "notebook" { + guard let webView = targetName == "preview" ? preview : notebook else { throw failure("Requested web view is not mounted") } + let shot = try await webView.takeSnapshot(configuration: nil) + #if os(macOS) + guard let tiff = shot.tiffRepresentation, let bitmap = NSBitmapImageRep(data: tiff), + let png = bitmap.representation(using: .png, properties: [:]) else { throw failure("Could not encode preview") } + #else + guard let png = shot.pngData() else { throw failure("Could not encode preview") } + #endif + return ["mimeType": "image/png", "data": png.base64EncodedString(), "target": targetName] + } + #if os(macOS) + guard let window = NSApp.windows.first(where: { $0.identifier?.rawValue == "workspace" }) ?? NSApp.windows.first(where: { $0.isVisible && !($0 is NSPanel) }), + window.contentView != nil else { throw failure("No app window available") } + if targetName == "app" { + guard let view = window.contentView, let bitmap = view.bitmapImageRepForCachingDisplay(in: view.bounds) else { throw failure("No app content available") } + view.cacheDisplay(in: view.bounds, to: bitmap) + guard let png = bitmap.representation(using: .png, properties: [:]) else { throw failure("Could not encode app chrome") } + return ["mimeType": "image/png", "data": png.base64EncodedString(), "target": "app", "scope": "native-chrome", "windowNumber": window.windowNumber] + } + // Never raise the macOS permission prompt from a test capture. + guard CGPreflightScreenCaptureAccess() else { throw failure("Whole-window capture needs existing Screen Recording permission. Use app, notebook and preview captures separately.") } + // Capture the actual window, including WebKit and native overlays. + // AppKit cacheDisplay and hand compositing lose pixels or occlusion. + let content = try await SCShareableContent.excludingDesktopWindows(true, onScreenWindowsOnly: false) + guard let target = content.windows.first(where: { $0.windowID == CGWindowID(window.windowNumber) }) else { throw failure("App window is unavailable to ScreenCaptureKit") } + let filter = SCContentFilter(desktopIndependentWindow: target) + let config = SCStreamConfiguration() + config.width = Int(target.frame.width * window.backingScaleFactor) + config.height = Int(target.frame.height * window.backingScaleFactor) + config.showsCursor = false + config.captureResolution = .best + let shot = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) + let output = NSBitmapImageRep(cgImage: shot) + guard let png = output.representation(using: .png, properties: [:]) else { throw failure("Could not encode app window") } + return ["mimeType": "image/png", "data": png.base64EncodedString(), "target": "window", "windowNumber": window.windowNumber] + #else + throw failure("Whole-window capture is currently macOS-only; preview capture is supported") + #endif + default: throw failure("Unknown automation method") + } + } + + static func error(_ message: String) -> NSError { NSError(domain: "AeselAutomation", code: 1, userInfo: [NSLocalizedDescriptionKey: message]) } + private func failure(_ message: String) -> NSError { Self.error(message) } +} diff --git a/apple/aesel/Sources/AeselNotebook.swift b/apple/aesel/Sources/AeselNotebook.swift index aff3101457..89ef536028 100644 --- a/apple/aesel/Sources/AeselNotebook.swift +++ b/apple/aesel/Sources/AeselNotebook.swift @@ -4,6 +4,7 @@ import WebKit /// Desktop rich replies, packaged locally; the agent host remains separate. struct AeselNotebook: AeselWebViewRepresentable { let session: Session + var automation: AeselAutomation? = nil var paint = Paint.base @Binding var height: CGFloat var openLink: (URL) -> Void @@ -51,6 +52,7 @@ struct AeselNotebook: AeselWebViewRepresentable { config.userContentController.add(context.coordinator, name: "notebook") let view = WKWebView(frame: .zero, configuration: config) view.navigationDelegate = context.coordinator + automation?.notebook = view ApplePlatform.configureEmbeddedView(view) view.load(URLRequest(url: URL(string: "aesel-bundle://app/easel/phone/notebook.html")!)) return view diff --git a/apple/aesel/Sources/AeselPieceView.swift b/apple/aesel/Sources/AeselPieceView.swift index 06f3fac5a2..a3d6212ce2 100644 --- a/apple/aesel/Sources/AeselPieceView.swift +++ b/apple/aesel/Sources/AeselPieceView.swift @@ -6,12 +6,13 @@ import WebKit struct PieceView: View { let url: URL let source: String + var automation: AeselAutomation? = nil @State private var failure: String? @State private var attempt = 0 @Environment(\.paint) private var paint var body: some View { ZStack { - PieceWebView(url: url, source: source, failure: $failure).id(attempt) + PieceWebView(url: url, source: source, automation: automation, failure: $failure).id(attempt) .onChange(of: url) { _, _ in failure = nil } .onChange(of: source) { _, _ in failure = nil } if let failure { @@ -24,12 +25,16 @@ struct PieceView: View { .background(paint.deep) } } + .onAppear { automation?.retryPreview = { failure = nil; attempt += 1 } } + .onChange(of: failure) { _, value in automation?.previewFailure = value } + .onDisappear { automation?.previewFailure = nil } } } private struct PieceWebView: AeselWebViewRepresentable { let url: URL let source: String + var automation: AeselAutomation? @Environment(\.colorScheme) private var colorScheme @Binding var failure: String? @@ -90,7 +95,11 @@ private struct PieceWebView: AeselWebViewRepresentable { const source = window.__aeselSource; if (!ready || !window.acSEND || !source || source === rendered) return; rendered = source; - window.acSEND({type: 'dropped:piece', content: {name: 'aesel-preview', source, isKidLisp: false}}); + const flags = new URLSearchParams(); + const query = new URLSearchParams(location.search); + for (const key of ['nogap', 'nolabel', 'autoreload']) flags.set(key, 'true'); + for (const key of ['preview', 'icon']) if (query.has(key)) flags.set(key, query.get(key)); + window.acSEND({type: 'dropped:piece', content: {name: 'aesel-preview', source, search: flags.toString(), isKidLisp: false}}); }; window.addEventListener('message', event => { if (!ready && event.data?.type === 'ready') { @@ -110,6 +119,7 @@ private struct PieceWebView: AeselWebViewRepresentable { """, injectionTime: .atDocumentStart, forMainFrameOnly: true)) let view = WKWebView(frame: .zero, configuration: configuration) view.navigationDelegate = context.coordinator + automation?.preview = view ApplePlatform.configureEmbeddedView(view) return view } diff --git a/apple/aesel/Sources/ContentView.swift b/apple/aesel/Sources/ContentView.swift index a92f88bf9e..cee734ca8f 100644 --- a/apple/aesel/Sources/ContentView.swift +++ b/apple/aesel/Sources/ContentView.swift @@ -65,6 +65,9 @@ struct ContentView: View { .onChange(of: session.signedIn) { if session.signedIn { Task { await braincells.load() } } } .task { braincells.start(token: { host.accessToken() }, credited: { host.refreshCredits() }) } .onAppear { + host.automation.inspect = { automationState } + host.automation.perform = { action, params in try await automationAction(action, params) } + host.automation.start() #if DEBUG if ProcessInfo.processInfo.environment["AESEL_NOTEBOOK_PREVIEW"] == "1" { showHome = false @@ -125,7 +128,7 @@ struct ContentView: View { .buttonStyle(AeselButtonStyle()).foregroundStyle(paint.ink) .frame(height: row).padding(.horizontal, 10) } - AeselNotebook(session: session, paint: paint, height: $notebookHeight) { openURL($0) } + AeselNotebook(session: session, automation: host.automation, paint: paint, height: $notebookHeight) { openURL($0) } .frame(height: max(row, notebookHeight)) if session.fatal != nil { Button("/retry") { host.restore() } @@ -170,7 +173,7 @@ struct ContentView: View { private var previewBox: some View { ZStack(alignment: .topTrailing) { if let url = session.previewURL { - PieceView(url: url, source: session.source) + PieceView(url: url, source: session.source, automation: host.automation) .frame(width: previewSize.width, height: previewSize.height) .clipped() } @@ -196,7 +199,7 @@ struct ContentView: View { private var expandedPiece: some View { ZStack(alignment: .topTrailing) { if let url = session.previewURL { - PieceView(url: url, source: session.source) + PieceView(url: url, source: session.source, automation: host.automation) .frame(maxWidth: .infinity, maxHeight: .infinity) } Button { expandedPreview = false } label: { @@ -216,7 +219,8 @@ struct ContentView: View { /// desktop's prose prompt; the trailing 100pt stays clear for the version. private var prompt: some View { HStack(alignment: .top, spacing: 8) { - TextField("make something…", text: $draft, axis: .vertical) + TextField("", text: $draft, axis: .vertical) + .accessibilityLabel("Message") .font(Paint.font(16)).foregroundStyle(paint.userInk).tint(paint.userInk) .lineLimit(1...6).textFieldStyle(.plain) .focused($writing).aeselSendLabel() @@ -242,11 +246,11 @@ struct ContentView: View { /// the right and 8pt up, and the way into settings. private var versionLabel: some View { Button { openSettings() } label: { - AeselTitle(text: "v\(appVersion)", size: 16).padding(9) + AeselTitle(text: "v\(session.pieceVersion)", size: 16).padding(9) } .buttonStyle(AeselButtonStyle()) .padding(.trailing, 1) - .accessibilityLabel("Version \(appVersion). Settings") + .accessibilityLabel("Piece version \(session.pieceVersion). Settings") } // MARK: - Settings @@ -455,7 +459,98 @@ struct ContentView: View { draft = "" } + private var automationState: [String: Any] { + #if os(macOS) + let canResize = true + #else + let canResize = false + #endif + let controls: [(String, String, Bool)] = [ + ("settings.open", "Open settings", !showSettings), ("settings.close", "Close settings", showSettings), + ("home.open", "Show saved threads", !showHome), ("home.close", "Return to notebook", showHome), + ("help.open", "Open help", !showHelp), ("help.close", "Close help", showHelp), + ("composer.set", "Set message draft", true), ("composer.clear", "Clear message draft", true), + ("composer.send", "Send message; may run inference and publish", session.signedIn && !session.busy && !draft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty), + ("turn.stop", "Stop current turn", session.busy), + ("session.retry", "Restore failed session", session.fatal != nil), + ("session.new", "Start a blank piece", !session.busy), ("session.resume", "Resume a saved thread", !session.busy), + ("preview.expand", "Expand preview", previewVisible && !expandedPreview), + ("preview.collapse", "Return to notebook", expandedPreview), + ("preview.hide", "Hide preview", previewVisible), ("preview.show", "Show preview", previewHidden), + ("preview.reload", "Reload embedded preview", previewVisible), + ("preview.retry", "Retry failed preview", host.automation.previewFailure != nil), + ("window.resize", "Resize app window without focusing it", canResize), + ("piece.open", "Open public piece in browser", session.shareURL != nil), + ("piece.publish", "Publish current source", session.signedIn && !session.busy), + ("account.signin", "Open AC sign-in", !session.signedIn), + ("account.signin.close", "Close sign-in", session.showSignIn), + ("account.signin.retry", "Retry sign-in", session.showSignIn && session.signInError != nil), + ("account.signout", "Sign out of AC", session.signedIn), + ("balance.refresh", "Refresh balance", session.signedIn), + ("credits.buy", "Open App Store purchase confirmation", session.signedIn && !braincells.busy && braincells.product != nil), + ("provider.select", "AC only; desktop CLI providers are not ported", false), + ("model.select", "Model managed automatically by AC", false) + ] + return ["schema": 1, "surface": expandedPreview ? "preview" : showSettings ? "settings" : showHome ? "home" : "notebook", + "overlays": ["settings": showSettings, "home": showHome, "help": showHelp, "signin": session.showSignIn], + "piece": ["route": session.route, "version": session.pieceVersion, "sourceBytes": session.source.utf8.count, + "previewURL": session.previewURL?.absoluteString ?? "", "shareURL": session.shareURL?.absoluteString ?? ""], + "composer": ["characters": draft.count, "placeholder": "", "focused": writing], + "title": ["opacity": 1, "linked": session.shareURL != nil], "footer": "v\(session.pieceVersion)", + "session": ["id": session.currentSessionID, "busy": session.busy, "status": session.status, "signedIn": session.signedIn, + "entryCount": session.entries.count, "history": session.history.map { ["id": $0.id, "title": $0.title, "route": $0.route] }], + "preview": ["visible": previewVisible, "expanded": expandedPreview], + "notebook": ["visible": !expandedPreview && !showHome && !showSettings && !showHelp && !session.showSignIn], + "controls": controls.map { ["id": $0.0, "label": $0.1, "enabled": $0.2] }] + } + private func automationAction(_ action: String, _ params: [String: Any]) async throws { + let controls = automationState["controls"] as? [[String: Any]] ?? [] + guard let control = controls.first(where: { $0["id"] as? String == action }) else { throw AeselAutomation.error("Unknown control: \(action)") } + guard control["enabled"] as? Bool == true else { throw AeselAutomation.error("Control is disabled: \(action)") } + switch action { + case "settings.open": openSettings() + case "settings.close": closeSettings() + case "home.open": showHome = true + case "home.close": showHome = false + case "help.open": showHelp = true + case "help.close": showHelp = false + case "composer.set": + guard let text = params["text"] as? String, text.utf8.count <= 32768 else { throw AeselAutomation.error("text is required, at most 32768 bytes") } + draft = text + case "composer.clear": draft = "" + case "composer.send": send() + case "turn.stop": host.stop() + case "session.retry": host.restore() + case "session.new": host.newSession(medium: "piece"); showHome = false + case "session.resume": + guard let id = params["sessionId"] as? String, session.history.contains(where: { $0.id == id }) else { throw AeselAutomation.error("Known sessionId is required") } + host.resumeSession(id: id); showHome = false + case "preview.expand": expandedPreview = true; writing = false + case "preview.collapse": expandedPreview = false + case "preview.hide": previewHidden = true; expandedPreview = false + case "preview.show": previewHidden = false + case "preview.reload": host.automation.preview?.reload() + case "preview.retry": host.automation.retryPreview?() + case "window.resize": + #if os(macOS) + guard let width = params["width"] as? Double, let height = params["height"] as? Double, + (360...2400).contains(width), (420...1800).contains(height), + let window = NSApp.windows.first(where: { !($0 is NSPanel) && $0.contentView != nil }) else { throw AeselAutomation.error("Window width 360–2400 and height 420–1800 are required") } + window.setContentSize(NSSize(width: width, height: height)) + #else + throw AeselAutomation.error("Window resizing is macOS-only") + #endif + case "piece.open": if let url = session.shareURL { openURL(url) } + case "piece.publish": host.publish() + case "account.signin", "account.signin.retry": host.signIn() + case "account.signin.close": session.showSignIn = false + case "account.signout": host.signOut() + case "balance.refresh": host.refreshCredits() + case "credits.buy": await braincells.buy() + default: throw AeselAutomation.error("Unsupported control: \(action)") + } + } } /// Puts the hidden session webview in the hierarchy without drawing it. diff --git a/apple/aesel/Sources/Session.swift b/apple/aesel/Sources/Session.swift index dea2ea50a0..ea92de88f4 100644 --- a/apple/aesel/Sources/Session.swift +++ b/apple/aesel/Sources/Session.swift @@ -46,7 +46,16 @@ struct SessionSummary: Identifiable { /// something a view can draw. @Observable final class Session { - static let draftPreviewURL = URL(string: "https://aesthetic.computer/wipe?nogap=true&nolabel=true&noauth=true")! + static let draftPreviewURL = embeddedPreviewURL(URL(string: "https://aesthetic.computer/wipe?noauth=true")!) + static func embeddedPreviewURL(_ url: URL) -> URL { + guard var target = URLComponents(url: url, resolvingAgainstBaseURL: false) else { return url } + if target.host == "prompt.ac" { target.host = "aesthetic.computer" } + let flags = ["nogap", "nolabel", "autoreload"] + var query = (target.queryItems ?? []).filter { !flags.contains($0.name) } + query += flags.map { URLQueryItem(name: $0, value: "true") } + target.queryItems = query + return target.url ?? url + } var entries: [Entry] = [] var history: [SessionSummary] = [] var medium = "piece" @@ -58,6 +67,7 @@ final class Session { var status: String = "starting" var health: Health = .idle var route: String = "" + var pieceVersion = 0 var previewURL: URL? = Session.draftPreviewURL var shareURL: URL? var source = "" @@ -168,10 +178,12 @@ final class Session { case "source": source = event["source"] as? String ?? source + pieceVersion = event["version"] as? Int ?? pieceVersion previewURL = Self.draftPreviewURL case "piece": source = event["source"] as? String ?? source + pieceVersion = event["version"] as? Int ?? 0 shareURL = nil previewURL = Self.draftPreviewURL route = event["route"] as? String ?? "" @@ -179,7 +191,7 @@ final class Session { case "preview": if let text = event["url"] as? String { shareURL = URL(string: text) - previewURL = shareURL + previewURL = shareURL.map(Self.embeddedPreviewURL) } case "status": diff --git a/apple/aesel/Sources/SessionHost.swift b/apple/aesel/Sources/SessionHost.swift index 44ade69a1d..98d9c8c717 100644 --- a/apple/aesel/Sources/SessionHost.swift +++ b/apple/aesel/Sources/SessionHost.swift @@ -15,6 +15,7 @@ import Security /// ES modules. JavaScriptCore has none of those. @MainActor final class SessionHost: NSObject { + let automation = AeselAutomation() private var webView: WKWebView! private var loginWebView: WKWebView? private var loginTimeout: Task? @@ -205,6 +206,7 @@ extension SessionHost: WKScriptMessageHandler { return } let type = body["type"] as? String + if let type, type != "persist" { automation.record("session.\(type)") } NSLog("[aesel] event \(type ?? "?")") // `didFinish` is not readiness. It fires when the document has diff --git a/apple/aesel/Tests/SessionPreviewChecks.swift b/apple/aesel/Tests/SessionPreviewChecks.swift index b6b6e30f34..6b9dcd3088 100644 --- a/apple/aesel/Tests/SessionPreviewChecks.swift +++ b/apple/aesel/Tests/SessionPreviewChecks.swift @@ -12,15 +12,25 @@ struct SessionPreviewChecks { let published = "https://aesthetic.computer/@test/painting" session.receive(["type": "preview", "url": published]) - precondition(session.previewURL?.absoluteString == published) + let query = URLComponents(url: session.previewURL!, resolvingAgainstBaseURL: false)!.queryItems! + for flag in ["nogap", "nolabel", "autoreload"] { + precondition(query.contains(URLQueryItem(name: flag, value: "true"))) + } precondition(session.shareURL?.absoluteString == published) - session.receive(["type": "source", "source": "edited draft"]) + session.receive(["type": "source", "source": "edited draft", "version": 1]) + precondition(session.pieceVersion == 1) precondition(session.previewURL == Session.draftPreviewURL) precondition(session.source == "edited draft") session.receive(["type": "piece", "source": "next piece", "route": "next-piece"]) precondition(session.previewURL == Session.draftPreviewURL) precondition(session.shareURL == nil) + precondition(session.pieceVersion == 0) + let embedded = Session.embeddedPreviewURL(URL(string: "https://prompt.ac/@test/name?zoom=2&nolabel=false#frame")!) + let parts = URLComponents(url: embedded, resolvingAgainstBaseURL: false)! + precondition(parts.host == "aesthetic.computer" && parts.fragment == "frame") + precondition(parts.queryItems!.filter { $0.name == "nolabel" } == [URLQueryItem(name: "nolabel", value: "true")]) + precondition(parts.queryItems!.contains(URLQueryItem(name: "zoom", value: "2"))) print("Preview remains visible for launch, drafts, published pieces and thread changes.") } } diff --git a/apple/aesel/Tests/preview-bridge.test.mjs b/apple/aesel/Tests/preview-bridge.test.mjs new file mode 100644 index 0000000000..107675d061 --- /dev/null +++ b/apple/aesel/Tests/preview-bridge.test.mjs @@ -0,0 +1,23 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import vm from 'node:vm'; + +// Execute the shipped WKUserScript with a ready runtime; URL-only tests miss +// the dropped-source handoff which previously restored the corner HUD. +const swift=readFileSync(new URL('../Sources/AeselPieceView.swift',import.meta.url),'utf8'); +const script=swift.match(/addUserScript\(WKUserScript\(source: """\n([\s\S]*?)\n """, injectionTime/)[1]; +test('native source injection carries view flags without forwarding auth parameters',()=>{ + const sent=[],ticks=[]; + const window={preloaded:true,__aeselSource:'export function paint({wipe}) { wipe("orange"); }',acSEND:message=>sent.push(message),addEventListener(){},webkit:{messageHandlers:{previewFailure:{postMessage(){}}}}}; + vm.runInNewContext(script,{window,location:{search:'?noauth=true&nolabel=true&preview&code=private&state=private'},URLSearchParams,setInterval:fn=>(ticks.push(fn),1),clearInterval(){},setTimeout(){}}); + ticks[0]();ticks[0](); + assert.equal(sent.length,1); + assert.equal(sent[0].type,'dropped:piece'); + const flags=new URLSearchParams(sent[0].content.search); + for(const key of ['nogap','nolabel','autoreload'])assert.equal(flags.get(key),'true'); + assert(flags.has('preview')); + for(const key of ['code','state','noauth'])assert(!flags.has(key)); + window.__aeselSource+='\n// revised';window.__aeselRender(); + assert.equal(sent.length,2);assert.equal(sent[1].content.source,window.__aeselSource); +}); diff --git a/easel/phone/session.mjs b/easel/phone/session.mjs index d987af3175..ed518a6eca 100644 --- a/easel/phone/session.mjs +++ b/easel/phone/session.mjs @@ -96,6 +96,8 @@ export function createSession({ storage = memoryStore(), emit = () => {} } = {}) publishing: null, dirty: false, published: false, + version: 0, + revisionSource: "", id: "", medium: "piece", transcript: [], @@ -162,7 +164,7 @@ export function createSession({ storage = memoryStore(), emit = () => {} } = {}) const item = { id: state.id, title: state.title || state.slug, medium: state.medium, model: state.model, savedAt: new Date().toISOString(), handle: state.owner || state.handle, slug: state.slug, - source: vfs.readFileSync(state.file), published: state.published, + source: vfs.readFileSync(state.file), published: state.published, version: state.version, events: state.transcript, engine, }; const items = readThreads().filter(entry => entry.id !== state.id); @@ -196,7 +198,7 @@ export function createSession({ storage = memoryStore(), emit = () => {} } = {}) state.transcript = Array.isArray(item.events) ? item.events : []; state.engine = item.engine || null; state.model = DEFAULT_AC_MODEL; - mountPiece(item.slug, item.source || STARTER); + mountPiece(item.slug, item.source || STARTER, item.version); state.published = Boolean(item.published && (!item.handle || item.handle === state.handle)); write({ threadID: state.id, published: state.published }); emit({type: "thread", id: state.id, medium: state.medium, events: state.transcript}); @@ -231,25 +233,31 @@ export function createSession({ storage = memoryStore(), emit = () => {} } = {}) function pieceUrl() { if (!state.handle) return ""; // Cache-busted: the URL is stable and the bytes behind it are not. - return `${SITE}/@${state.handle}/${state.slug}?nolabel=true&nogap=true#${Date.now()}`; + return `${SITE}/@${state.handle}/${state.slug}?nolabel=true&nogap=true&autoreload=true#${Date.now()}`; } - function mountPiece(slug, source) { + function mountPiece(slug, source, version = 0) { state.slug = slug; state.file = `/piece/${slug}.mjs`; state.server = null; // a new piece is a new conversation vfs.mount(state.file, source); state.published = false; + state.version = Number.isInteger(version) && version >= 0 ? version : 0; + state.revisionSource = source; write({ slug, source, published: false }); - say("piece", { route: route(), slug, source }); + say("piece", { route: route(), slug, source, version: state.version }); } function onWritten(path, source) { if (path !== state.file) return; + if (source !== state.revisionSource) { + state.version += 1; + state.revisionSource = source; + } write({ slug: state.slug, source }); saveCurrent(); state.dirty = true; - say("source", { source }); + say("source", { source, version: state.version }); void publish(); } diff --git a/easel/test/phone-session.test.mjs b/easel/test/phone-session.test.mjs index 92b5bd3c3f..1aebbfec15 100644 --- a/easel/test/phone-session.test.mjs +++ b/easel/test/phone-session.test.mjs @@ -95,3 +95,25 @@ test('braincell model stays automatic across old threads and model commands', as await restored.open(); assert.equal(restored.state.model, DEFAULT_AC_MODEL); }); + +test('piece revision starts at zero, counts changed source, and survives thread reload', async () => { + const values=new Map(); + const events=[]; + const session=createSession({storage:{get:k=>values.get(k),set:(k,v)=>values.set(k,v)},emit:e=>events.push(e)}); + const originalFetch=globalThis.fetch; + globalThis.fetch=async()=>new Response('fixture guide'); + try { + await session.begin();await session.open(); + const id=session.state.id; + assert.equal(session.state.version,0); + const vfs=await import('../phone/shim/fs.mjs'); + vfs.writeFileSync(session.state.file,'export function paint() {}'); + assert.equal(session.state.version,1); + vfs.writeFileSync(session.state.file,'export function paint() {}'); + assert.equal(session.state.version,1); + assert.equal(events.filter(e=>e.type==='source').at(-1).version,1); + await session.newSession();assert.equal(session.state.version,0); + await session.resumeSession(id);assert.equal(session.state.version,1); + assert.equal(events.filter(e=>e.type==='piece').at(-1).version,1); + } finally {globalThis.fetch=originalFetch;} +}); diff --git a/slab/AESEL-EYE.md b/slab/AESEL-EYE.md new file mode 100644 index 0000000000..b948975464 --- /dev/null +++ b/slab/AESEL-EYE.md @@ -0,0 +1,116 @@ +# Aesthetic Eye for Aesel + +Aesel passes when its controls work and its rendered states make sense. A build, +an accessibility tree, or a successful RPC call is evidence, not visual acceptance. +This workflow extends [the papers gate](../papers/aesthetic-eye.mjs) to an interactive app. + +## Contract + +1. Build the app. Keep testing in the background; use Desktop 2 when WebKit needs + a visible window. Never activate a window merely to inspect it. +2. Read `aesel_map`. It reports stable control IDs, enabled states, the current + surface and overlays. Disabled or unported features remain explicit. +3. Exercise the affected controls with `aesel_act`, then read `aesel_state` and + `aesel_events`. An accepted action can still have asynchronous work pending. +4. Capture notebook, settings, expanded preview and narrow notebook. Add loading, + failure, account or other states whenever the change affects them. +5. Inspect each image at its intended size. Review type, contrast, spacing, + redundant chrome, preview content and the observed interaction. An unreadable + label, blank expected preview, misleading state or broken control fails. +6. Record the reviewer, timestamp, checks and concrete observations in + `aesthetic-eye.json`. Run the gate against the same built app and evidence. + +The capture command starts every scenario as **unreviewed**. It cannot certify +itself. The check fails missing required scenarios, missing review, failed checks, +modified evidence and a changed build. The running process reports its startup +bundle fingerprint, so pointing at a newer binary on disk cannot bless an old UI. + +```sh +node slab/bin/aesel-eye.mjs capture reports/aesel-eye-run notebook '/path/Aesel Native.app' +node slab/bin/aesel-eye.mjs check reports/aesel-eye-run +``` + +Use a new directory after rebuilding. Capture native chrome, notebook and preview as separate surfaces: AppKit's +view rasterization omits composited WebKit content. Notebook and preview images +are mandatory wherever those surfaces are visible. These captures verify the +components; they do not prove whole-window occlusion. Use the optional `window` +capture for changes involving overlays or occlusion; it requires existing macOS +Screen Recording permission and fails without raising a permission prompt. Hidden/off-space WebKit +may be throttled; inspect readiness and pixels rather than waiting blindly. + +## MCP + +`slab/bin/aesel-mcp.mjs` shares the stack's HTTP/stdio transport. The daemon +installer registers `aesel` at `http://127.0.0.1:7781/mcp`. Its tools are: + +| Tool | Purpose | +|---|---| +| `aesel_map` | Current surfaces, control IDs and availability | +| `aesel_state` | Revision, preview flags, session state and draft length | +| `aesel_act` | Named UI operation with validated arguments | +| `aesel_preview` | Actual WebKit URL, readiness and canvas sizes | +| `aesel_capture` | Native chrome, notebook, preview, or authorized whole-window PNG | +| `aesel_events` | Last 256 event kinds/timestamps, cursor by sequence | +| `aesel_eye_capture` | Save a build-bound, unreviewed scenario | +| `aesel_eye_check` | Enforce current evidence and explicit visual acceptance | + +The native bridge uses a private same-user mailbox inside its macOS sandbox. +Requests are correlated to the current process instance and expire after 30 +seconds. It opens no native listening port and provides no arbitrary evaluation. +The adapter does not launch or focus the app. Its HTTP endpoint binds loopback +and rejects browser-origin requests. For a remote Mac, run stdio through an +existing authenticated SSH connection; do not expose port 7781 publicly. + +```sh +ssh blueberry node /path/to/aesthetic-computer/slab/bin/aesel-mcp.mjs +``` + +`AESEL_AUTOMATION_DIR` selects a mailbox for the adapter. Launch a native test +instance with `AESEL_AUTOMATION_NAMESPACE=eye` to use `automation-eye` separately +from the everyday app. This is especially useful with the existing debug +notebook fixture; fixture tests do not demonstrate account, inference or purchase +success. Physical-device transport and whole-window iOS capture are not provided. + +Diagnostics record event names and timestamps, not prompts, tokens, source or +bridge payloads. State inspection includes visible piece/thread metadata. +Screenshots can contain whatever the user has displayed; capture only for an +authorized test. There is no PostHog or other analytics export. + +Sending, publishing, account changes and purchases retain their ordinary effects. +Test those only with explicit task authorization. The default acceptance loop +uses draft editing and local screen transitions, without inference or purchases. + +## Components + +`AeselAutomation.swift` owns RPC, captures and the diagnostic ring. `ContentView` +maps controls to the same handlers used by the UI. `SessionHost` records session +event kinds, while the shared JavaScript session owns the persisted piece +revision. `Session.embeddedPreviewURL` applies Electron's `nogap`, `nolabel` and +`autoreload` contract while preserving other query parameters and fragments. + +New UI controls must join the live map, define availability and effects, and be +covered by an appropriate state capture. Unsupported controls cannot silently +report success. Changes to the mailbox or gate also run: + +```sh +node --test slab/test/aesel-mcp.test.mjs +``` + +Precedent consulted: `papers/SCORE.md`, `papers/AESTHETIC-EYE.md` and +`papers/aesthetic-eye.mjs`. This is a development-workflow extension of the +existing gate; no scholarly paper or PDF is implied. + +## Initial validation — 21 September 2026 + +The macOS debug build and nine JavaScript checks passed, along with the Swift +preview/session checks. The live AC runtime rendered an injected orange frame +without its corner HUD in headless Chromium using the shipped native injection +script. Native MCP draft set/clear, settings open/close, preview expand/collapse, +and window resizing passed against the isolated debug fixture. + +Native visual acceptance remains **failed**: the locked remote Mac produced +blank WebKit previews and a runtime timeout. Notebook and native chrome were +inspected, including the 420-point layout; this does not substitute for a working +native preview. Whole-window capture also lacked Screen Recording permission. +No account, inference, publication, purchase or physical-iOS result is claimed. +Rerun the gate on an unlocked test seat before declaring native UI acceptance. diff --git a/slab/bin/aesel-eye.mjs b/slab/bin/aesel-eye.mjs new file mode 100644 index 0000000000..768dc78a2e --- /dev/null +++ b/slab/bin/aesel-eye.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +// Aesthetic Eye for Aesel: capture real states, then require explicit visual review. +import { createHash } from 'node:crypto'; +import { readFile, writeFile, mkdir, readdir } from 'node:fs/promises'; +import { resolve, join, relative } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { request } from './aesel-mcp.mjs'; + +const digest = value => createHash('sha256').update(value).digest('hex'); +export const CHECKS = ['type','contrast','spacing','chrome','preview','interaction']; +export async function buildFingerprint(bundle) { + const hash = createHash('sha256'); + async function visit(folder) { + for (const item of (await readdir(folder,{withFileTypes:true})).sort((a,b)=>Buffer.compare(Buffer.from(a.name),Buffer.from(b.name)))) { + if (item.name === '_CodeSignature') continue; + const path = join(folder,item.name); + if (item.isDirectory()) await visit(path); + else if (item.isFile()) { hash.update(relative(bundle,path)); hash.update('\0'); hash.update(await readFile(path)); } + } + } + await visit(bundle); + return hash.digest('hex'); +} + +export async function capture(directory, name, bundle, options = {}) { + if (!/^[a-z0-9-]{1,60}$/.test(name)) throw new Error('Use a short lowercase scenario name'); + directory = resolve(directory); bundle = resolve(bundle); + await mkdir(directory,{recursive:true,mode:0o700}); + const manifestPath = join(directory,'aesthetic-eye.json'); + let manifest; + try { manifest = JSON.parse(await readFile(manifestPath,'utf8')); } + catch (error) { if (error.code !== 'ENOENT') throw error; } + if(manifest && (manifest.schema!==1||manifest.kind!=='aesel-ui'))throw new Error('Existing manifest belongs to another workflow'); + const fingerprint = await buildFingerprint(bundle); + if (manifest && manifest.buildSha256 !== fingerprint) throw new Error('Build changed; use a new capture directory'); + manifest ||= {schema:1,kind:'aesel-ui',bundle,buildSha256:fingerprint,visualInference:false,reviewer:null,requiredScenarios:['notebook','settings','preview-expanded','notebook-narrow'],scenarios:[]}; + if (manifest.scenarios.some(item=>item.name===name)) throw new Error('Scenario already captured; preserve evidence and use a new name'); + const state = await request('state',{},options); + if(state.buildSha256!==fingerprint)throw new Error('Running app differs from the requested bundle; capture refused'); + const files=[]; + async function save(file, bytes) { + await writeFile(join(directory,file),bytes,{flag:'wx',mode:0o600}); + files.push({path:file,sha256:digest(bytes)}); + } + await save(`${name}-state.json`,Buffer.from(JSON.stringify(state,null,2)+'\n')); + for (const target of ['app',...(state.notebook?.visible ? ['notebook'] : []),...(state.preview?.visible ? ['preview'] : [])]) { + const result=await request('capture',{target},options); + await save(`${name}-${target}.png`,Buffer.from(result.data,'base64')); + } + if(state.preview?.visible){ + const preview=await request('preview',{},options); + await save(`${name}-preview.json`,Buffer.from(JSON.stringify(preview,null,2)+'\n')); + } + manifest.scenarios.push({name,capturedAt:new Date().toISOString(),instance:state.instance,files,design:'unreviewed',checks:Object.fromEntries(CHECKS.map(key=>[key,'unreviewed'])),notes:'',actions:[]}); + await writeFile(manifestPath,JSON.stringify(manifest,null,2)+'\n',{mode:0o600}); + return manifestPath; +} + +export async function check(directory) { + directory=resolve(directory); + const manifest=JSON.parse(await readFile(join(directory,'aesthetic-eye.json'),'utf8')); + const errors=[]; + if(manifest.schema!==1||manifest.kind!=='aesel-ui')errors.push('Invalid manifest schema/kind'); + if(manifest.buildSha256!==await buildFingerprint(manifest.bundle))errors.push('Stale build fingerprint'); + if(manifest.visualInference!==true||!manifest.reviewer?.name||manifest.reviewer.kind!=='visual-inference'||!manifest.reviewedAt)errors.push('Explicit visual review required'); + if(!manifest.scenarios?.length)errors.push('No captured scenarios'); + const names=new Set(); + for(const scenario of manifest.scenarios||[]){ + if(names.has(scenario.name))errors.push(`Duplicate scenario ${scenario.name}`);names.add(scenario.name); + if(scenario.design!=='pass'||CHECKS.some(key=>scenario.checks?.[key]!=='pass')||!scenario.notes?.trim())errors.push(`${scenario.name}: incomplete or failed review`); + if(!scenario.files?.some(file=>file.path.endsWith('-app.png'))||!scenario.files?.some(file=>file.path.endsWith('-state.json')))errors.push(`${scenario.name}: missing required evidence`); + const stateFile=scenario.files?.find(file=>file.path===`${scenario.name}-state.json`); + if(!stateFile)errors.push(`${scenario.name}: canonical state evidence missing`); + if(stateFile){ + try{ + const state=JSON.parse(await readFile(join(directory,stateFile.path),'utf8')); + if(state.buildSha256!==manifest.buildSha256)errors.push(`${scenario.name}: state build differs`); + if(state.notebook?.visible && !scenario.files.some(file=>file.path===`${scenario.name}-notebook.png`))errors.push(`${scenario.name}: visible notebook evidence missing`); + if(state.preview?.visible && ['preview.png','preview.json'].some(suffix=>!scenario.files.some(file=>file.path===`${scenario.name}-${suffix}`)))errors.push(`${scenario.name}: visible preview evidence missing`); + }catch{errors.push(`${scenario.name}: invalid state evidence`);} + } + for(const file of scenario.files||[]){ + const path=resolve(directory,file.path); + if(!path.startsWith(directory+'/')){errors.push('Evidence escapes capture directory');continue;} + try{if(digest(await readFile(path))!==file.sha256)errors.push(`${file.path}: evidence changed`);}catch{errors.push(`${file.path}: evidence missing`);} + } + } + for(const name of ['notebook','settings','preview-expanded','notebook-narrow',...(manifest.requiredScenarios||[])])if(!names.has(name))errors.push(`Required scenario missing: ${name}`); + return {pass:!errors.length,errors,scenarios:names.size}; +} + +if(process.argv[1]&&import.meta.url===pathToFileURL(resolve(process.argv[1])).href){ + const [command,directory,name,bundle]=process.argv.slice(2); + try{ + if(command==='capture'&&bundle)console.log(await capture(directory,name,bundle)); + else if(command==='check'&&directory){const result=await check(directory);console.log(JSON.stringify(result,null,2));process.exitCode=result.pass?0:1;} + else throw new Error('Usage: aesel-eye capture | check '); + }catch(error){console.error(error.message);process.exitCode=1;} +} diff --git a/slab/bin/aesel-mcp.mjs b/slab/bin/aesel-mcp.mjs new file mode 100644 index 0000000000..a20374f118 --- /dev/null +++ b/slab/bin/aesel-mcp.mjs @@ -0,0 +1,94 @@ +#!/usr/bin/env node +// Native Aesel automation through its same-user, sandbox-local RPC mailbox. +import { readFile, writeFile, rename, unlink, stat, lstat } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { homedir } from 'node:os'; +import { randomUUID } from 'node:crypto'; +import { pathToFileURL } from 'node:url'; +import { httpPort, serveHttp, serveStdio } from '../../toolchain/mcp/http-front.mjs'; + +export const DEFAULT_DIRECTORY = join(homedir(), 'Library/Containers/computer.aesthetic.aesel.native/Data/Library/Application Support/computer.aesthetic.aesel.native/automation'); +const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); + +export async function request(method, params = {}, options = {}) { + const directory = options.directory || process.env.AESEL_AUTOMATION_DIR || DEFAULT_DIRECTORY; + const metadata = await lstat(directory).catch(() => { throw new Error('Aesel automation is unavailable. Start an MCP-enabled Aesel Native build; this tool never opens or focuses the app.'); }); + if (!metadata.isDirectory() || metadata.isSymbolicLink() || (metadata.mode & 0o077) || metadata.uid !== process.getuid()) throw new Error('Automation directory must be private and owned by the current user'); + const instance = JSON.parse(await readFile(join(directory, 'instance.json'), 'utf8')); + if (instance.schema !== 1 || !Number.isInteger(instance.pid) || typeof instance.instance !== 'string') throw new Error('Invalid Aesel instance record'); + try { process.kill(instance.pid, 0); } catch { throw new Error('Aesel is not running; instance record is stale'); } + const id = randomUUID(); + const input = join(directory, 'requests', `${id}.json`); + const output = join(directory, 'responses', `${id}.json`); + const temporary = `${input}.tmp`; + const message = JSON.stringify({ schema: 1, id, instance: instance.instance, method, params, createdAt: Date.now() / 1000 }); + if (Buffer.byteLength(message) > 65536) throw new Error('Request exceeds 64 KiB'); + await writeFile(temporary, message, { mode: 0o600, flag: 'wx' }); + await rename(temporary, input); + const deadline = Date.now() + (options.timeoutMs ?? 20000); + try { + while (Date.now() < deadline) { + const size = await stat(output).then(s => s.size).catch(() => null); + if (size !== null) { + if (size > 24 * 1024 * 1024) throw new Error('Response exceeds 24 MiB'); + const response = JSON.parse(await readFile(output, 'utf8')); + if (response.id !== id) throw new Error('Response ID mismatch'); + if (response.error) throw new Error(response.error); + return response.result; + } + await pause(80); + } + throw new Error('Aesel request timed out; no UI activation attempted'); + } finally { + await Promise.all([input, output, temporary].map(file => unlink(file).catch(() => {}))); + } +} + +const object = properties => ({ type: 'object', properties, additionalProperties: false }); +export const TOOLS = [ + { name: 'aesel_map', description: 'Map the running native app: stable control IDs, enabled states, visible screen and overlays. Unsupported features remain explicitly disabled. Does not activate the app.', inputSchema: object({}), annotations: {readOnlyHint: true} }, + { name: 'aesel_state', description: 'Read Aesel UI and session state, piece revision, preview URL/query flags, draft length and saved-thread metadata. Does not return tokens, transcript text or source.', inputSchema: object({}), annotations: {readOnlyHint: true} }, + { name: 'aesel_act', description: 'Operate a named enabled control from aesel_map. composer.set accepts text; session.resume accepts sessionId; window.resize accepts width/height. Send can consume inference credits and auto-publish; piece.publish writes publicly; credits.buy opens App Store confirmation; piece.open opens a browser. Use only when that specific action is authorized. Returns acceptance and observed UI state; async work may still be pending. Never focuses the main app.', inputSchema: { ...object({ id: {type:'string'}, text: {type:'string',maxLength:32768}, sessionId: {type:'string'}, width:{type:'number',minimum:360,maximum:2400},height:{type:'number',minimum:420,maximum:1800} }), required:['id'] }, annotations: {readOnlyHint:false, destructiveHint:true, openWorldHint:true} }, + { name: 'aesel_events', description: 'Read the bounded local diagnostic event ring after a sequence number. Event kinds and timestamps only; no prompt, source, token or raw bridge payload. No analytics export.', inputSchema: object({after:{type:'integer',minimum:0}}), annotations:{readOnlyHint:true} }, + { name: 'aesel_preview', description: 'Inspect the actual embedded WebKit URL, query flags, runtime readiness and canvas dimensions. No arbitrary JavaScript execution.', inputSchema: object({}), annotations:{readOnlyHint:true} }, + { name: 'aesel_capture', description: 'Capture native app chrome, notebook WebKit, or preview WebKit without focusing or opening a window. Capture each visible surface for acceptance. Optional window capture requires existing Screen Recording permission and never requests it. Captures can contain visible user content.', inputSchema: object({target:{type:'string',enum:['app','notebook','preview','window'],default:'app'}}), annotations:{readOnlyHint:true} }, + { name: 'aesel_eye_capture', description: 'Save one named Aesthetic Eye scenario: running-build fingerprint, UI state, app and preview PNGs. Creates an unreviewed manifest; never certifies visual quality. Use a new directory after rebuilding.', inputSchema:{...object({directory:{type:'string'},scenario:{type:'string'},bundle:{type:'string'}}),required:['directory','scenario','bundle']},annotations:{readOnlyHint:false,destructiveHint:false} }, + { name: 'aesel_eye_check', description: 'Check an Aesel Aesthetic Eye manifest. Fails stale build/evidence, missing required scenarios, absent visual review or failed checks. Does not create or infer a pass.', inputSchema:{...object({directory:{type:'string'}}),required:['directory']},annotations:{readOnlyHint:true} }, +]; +const METHODS = {aesel_map:'map',aesel_state:'state',aesel_act:'action',aesel_events:'events',aesel_preview:'preview',aesel_capture:'capture'}; + +export async function handleMessage(message, context = {}) { + const {id, method, params} = message; + if (id === undefined || id === null) return null; + if (context.headers?.origin) return {jsonrpc:'2.0',id,error:{code:-32000,message:'Browser-origin requests are not accepted'}}; + try { + let result; + if (method === 'initialize') result = {protocolVersion:'2024-11-05',capabilities:{tools:{}},serverInfo:{name:'aesel-mcp',version:'1.0.0'}}; + else if (method === 'ping') result = {}; + else if (method === 'tools/list') result = {tools:TOOLS}; + else if (method === 'tools/call') { + if (params?.name === 'aesel_eye_capture' || params?.name === 'aesel_eye_check') { + const eye=await import('./aesel-eye.mjs'); + const args=params.arguments||{}; + if(typeof args.directory!=='string')throw new Error('directory is required'); + const value=params.name==='aesel_eye_check' ? await eye.check(args.directory) : await eye.capture(args.directory,args.scenario,args.bundle); + return {jsonrpc:'2.0',id,result:{content:[{type:'text',text:JSON.stringify(value)}],...(value?.pass===false?{isError:true}:{})}}; + } + const operation = METHODS[params?.name]; + if (!operation) throw new Error('Unknown Aesel tool'); + const response = await request(operation, params.arguments || {}); + result = operation === 'capture' + ? {content:[{type:'image',mimeType:response.mimeType,data:response.data},{type:'text',text:JSON.stringify({...response,data:undefined})}]} + : {content:[{type:'text',text:JSON.stringify(response)}]}; + } else return {jsonrpc:'2.0',id,error:{code:-32601,message:'Unknown MCP method'}}; + return {jsonrpc:'2.0',id,result}; + } catch (error) { + return {jsonrpc:'2.0',id,result:{isError:true,content:[{type:'text',text:error.message}]}}; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const port = httpPort(process.argv, 7781); + if (port) serveHttp({handleMessage,port,banner:'aesel-mcp'}); + else serveStdio({handleMessage,banner:'aesel-mcp'}); +} diff --git a/slab/test/aesel-mcp.test.mjs b/slab/test/aesel-mcp.test.mjs new file mode 100644 index 0000000000..29487d0dcc --- /dev/null +++ b/slab/test/aesel-mcp.test.mjs @@ -0,0 +1,63 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, writeFile, readFile, readdir, rm, chmod } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createHash } from 'node:crypto'; +import { request, handleMessage } from '../bin/aesel-mcp.mjs'; +import { check, buildFingerprint, CHECKS } from '../bin/aesel-eye.mjs'; + +test('MCP discovery and browser-origin rejection do not require a running app', async () => { + const reply=await handleMessage({id:1,method:'tools/list'}); + assert(reply.result.tools.some(tool=>tool.name==='aesel_capture')); + assert.equal(await handleMessage({method:'notifications/initialized'}),null); + assert((await handleMessage({id:2,method:'tools/list'},{headers:{origin:'https://untrusted.example'}})).error); +}); + +test('mailbox correlates responses, expires unanswered requests, and requires a private directory', async () => { + const dir=await mkdtemp(join(tmpdir(),'aesel-rpc-')); + await chmod(dir,0o700);await mkdir(join(dir,'requests'));await mkdir(join(dir,'responses')); + await writeFile(join(dir,'instance.json'),JSON.stringify({schema:1,pid:process.pid,instance:'fixture'})); + try { + const pending=request('state',{}, {directory:dir,timeoutMs:1000}); + let file; + for(let i=0;i<20&&!file;i++){file=(await readdir(join(dir,'requests'))).find(name=>name.endsWith('.json'));if(!file)await new Promise(r=>setTimeout(r,10));} + const input=JSON.parse(await readFile(join(dir,'requests',file),'utf8')); + assert.equal(input.instance,'fixture');assert.equal(input.method,'state'); + await writeFile(join(dir,'responses',file),JSON.stringify({id:input.id,result:{footer:'v0'}})); + assert.deepEqual(await pending,{footer:'v0'}); + assert.deepEqual(await readdir(join(dir,'requests')),[]); + await assert.rejects(request('state',{}, {directory:dir,timeoutMs:10}),/timed out/); + assert.deepEqual(await readdir(join(dir,'requests')),[]); + await chmod(dir,0o755); + await assert.rejects(request('state',{}, {directory:dir}),/private/); + } finally {await rm(dir,{recursive:true,force:true});} +}); + +test('Aesthetic Eye fails absent scenarios, unreviewed images, changed evidence, and stale builds', async () => { + const dir=await mkdtemp(join(tmpdir(),'aesel-eye-')); + const bundle=join(dir,'Fixture.app');await mkdir(bundle);await writeFile(join(bundle,'code'),'build1'); + const bytes=Buffer.from('fixture'); + const fingerprint=await buildFingerprint(bundle); + const scenarios=[]; + for(const name of ['notebook','settings','preview-expanded','notebook-narrow']){ + const files=[]; + for(const suffix of ['app.png','state.json']){const path=`${name}-${suffix}`;const content=suffix==='state.json'?Buffer.from(JSON.stringify({buildSha256:fingerprint,preview:{visible:false}})):bytes;await writeFile(join(dir,path),content);files.push({path,sha256:createHash('sha256').update(content).digest('hex')});} + scenarios.push({name,files,design:'pass',checks:Object.fromEntries(CHECKS.map(key=>[key,'pass'])),notes:'Test fixture; never product acceptance'}); + } + const manifest={schema:1,kind:'aesel-ui',bundle,buildSha256:await buildFingerprint(bundle),visualInference:true,reviewer:{name:'test fixture',kind:'visual-inference'},reviewedAt:new Date().toISOString(),scenarios}; + const save=()=>writeFile(join(dir,'aesthetic-eye.json'),JSON.stringify(manifest)); + try{ + await save();assert.equal((await check(dir)).pass,true); + const stateFile=scenarios[0].files.find(f=>f.path.endsWith('-state.json')); + const original=await readFile(join(dir,stateFile.path)); + const changed=Buffer.from(JSON.stringify({buildSha256:fingerprint,preview:{visible:true}})); + await writeFile(join(dir,stateFile.path),changed);stateFile.sha256=createHash('sha256').update(changed).digest('hex');await save(); + assert((await check(dir)).errors.some(e=>e.includes('visible preview evidence missing'))); + await writeFile(join(dir,stateFile.path),original);stateFile.sha256=createHash('sha256').update(original).digest('hex');await save(); + manifest.visualInference=false;await save();assert.equal((await check(dir)).pass,false); + manifest.visualInference=true;const last=manifest.scenarios.pop();await save();assert((await check(dir)).errors.some(e=>e.includes('Required scenario'))); + manifest.scenarios.push(last);await save();await writeFile(join(dir,'notebook-app.png'),'changed');assert((await check(dir)).errors.some(e=>e.includes('evidence changed'))); + await writeFile(join(bundle,'code'),'build2');assert((await check(dir)).errors.some(e=>e.includes('Stale build'))); + }finally{await rm(dir,{recursive:true,force:true});} +}); diff --git a/toolchain/mcp/install-daemons.sh b/toolchain/mcp/install-daemons.sh index 30b3d6a1a3..2f1b730570 100755 --- a/toolchain/mcp/install-daemons.sh +++ b/toolchain/mcp/install-daemons.sh @@ -21,6 +21,7 @@ # grafana-mcp→ http://127.0.0.1:7778/mcp (when fuser credentials exist) # photo-mcp → http://127.0.0.1:7779/mcp # instagram-mcp → http://127.0.0.1:7780/mcp +# aesel-mcp → http://127.0.0.1:7781/mcp # # This script also POINTS Claude and Codex at the daemons. Claude gets same-name # local-scope entries that shadow the stdio ones in .mcp.json; Codex gets its @@ -93,6 +94,7 @@ fleet 7776 toolchain/fleet/fleet-mcp.mjs paper 7777 slab/bin/paper-mcp.mjs photo 7779 slab/bin/photo-mcp.mjs instagram 7780 slab/bin/instagram-mcp.mjs +aesel 7781 slab/bin/aesel-mcp.mjs ROWS }