From f99e291a95c7ee40fd6edc7c28d076fe8ced537f Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sat, 7 Mar 2026 00:32:43 -0500 Subject: [PATCH 01/12] upgrade pyodide to 0.29.3 --- internals/scripts/InstallMNE.mjs | 177 ++++++++++++++++++++++++ internals/scripts/InstallPyodide.js | 66 --------- internals/scripts/InstallPyodide.mjs | 108 +++++++++++++++ package.json | 2 +- src/renderer/utils/pyodide/webworker.js | 54 ++++++-- 5 files changed, 332 insertions(+), 75 deletions(-) create mode 100644 internals/scripts/InstallMNE.mjs delete mode 100644 internals/scripts/InstallPyodide.js create mode 100644 internals/scripts/InstallPyodide.mjs diff --git a/internals/scripts/InstallMNE.mjs b/internals/scripts/InstallMNE.mjs new file mode 100644 index 0000000..23f1466 --- /dev/null +++ b/internals/scripts/InstallMNE.mjs @@ -0,0 +1,177 @@ +#!/usr/bin/env node +/** + * Downloads MNE-Python and its pure-Python dependencies from PyPI as wheel + * files for offline use with Pyodide. Binary dependencies (numpy, scipy, + * matplotlib, pandas) are already included via the `pyodide` npm package and + * do NOT need to be downloaded here. + * + * Downloaded wheels are saved to: + * src/renderer/utils/pyodide/src/packages/ + * + * A manifest.json is written there so the web worker knows which filenames + * to pass to micropip.install() at startup. + * + * Usage: node internals/scripts/InstallMNE.mjs + */ + +import fs from 'fs'; +import https from 'https'; +import path from 'path'; +import chalk from 'chalk'; + +const PACKAGES_DIR = path.resolve( + 'src/renderer/utils/pyodide/src/packages' +); +const MANIFEST_FILE = path.join(PACKAGES_DIR, 'manifest.json'); + +/** + * Pure-Python packages required by MNE that are not bundled with Pyodide. + * Each entry is resolved against the PyPI JSON API to find the latest + * pure-Python wheel (py3-none-any or py2.py3-none-any). + */ +const PACKAGES_TO_DOWNLOAD = [ + 'mne', + 'pooch', + 'tqdm', + 'platformdirs', +]; + +// --------------------------------------------------------------------------- +// Network helpers +// --------------------------------------------------------------------------- + +function httpsGet(url) { + return new Promise((resolve, reject) => { + const req = https.get(url, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + resolve(httpsGet(res.headers.location)); + return; + } + if (res.statusCode !== 200) { + reject(new Error(`HTTP ${res.statusCode} for ${url}`)); + return; + } + let body = ''; + res.setEncoding('utf8'); + res.on('data', (chunk) => { body += chunk; }); + res.on('end', () => resolve(body)); + res.on('error', reject); + }); + req.on('error', reject); + }); +} + +function downloadBinary(url, dest) { + return new Promise((resolve, reject) => { + const doGet = (reqUrl) => { + https.get(reqUrl, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + doGet(res.headers.location); + return; + } + if (res.statusCode !== 200) { + reject(new Error(`HTTP ${res.statusCode} for ${reqUrl}`)); + return; + } + const file = fs.createWriteStream(dest); + res.pipe(file); + file.on('finish', () => file.close(resolve)); + file.on('error', (err) => { fs.unlink(dest, () => {}); reject(err); }); + }).on('error', (err) => { fs.unlink(dest, () => {}); reject(err); }); + }; + doGet(url); + }); +} + +// --------------------------------------------------------------------------- +// PyPI helpers +// --------------------------------------------------------------------------- + +/** + * Returns the best pure-Python wheel for the latest release of `packageName`. + * Preference: py3-none-any > py2.py3-none-any > *-none-any + */ +async function resolvePureWheel(packageName) { + const raw = await httpsGet(`https://pypi.org/pypi/${packageName}/json`); + const data = JSON.parse(raw); + const version = data.info.version; + const urls = data.urls; // files for the latest release + + const wheels = urls.filter((f) => f.filename.endsWith('.whl')); + + const ranked = [ + wheels.find((f) => f.filename.endsWith('-py3-none-any.whl')), + wheels.find((f) => f.filename.endsWith('-py2.py3-none-any.whl')), + wheels.find((f) => f.filename.includes('-none-any.whl')), + ].filter(Boolean); + + if (ranked.length === 0) { + throw new Error( + `No pure-Python wheel found for ${packageName} ${version}. ` + + `Binary packages must come from the Pyodide npm bundle.` + ); + } + + return { version, wheel: ranked[0] }; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function installPackage(packageName, manifest) { + process.stdout.write(chalk.blue(` ${packageName}: `)); + + let version, wheel; + try { + ({ version, wheel } = await resolvePureWheel(packageName)); + } catch (err) { + console.log(chalk.red(`FAILED — ${err.message}`)); + return; + } + + const dest = path.join(PACKAGES_DIR, wheel.filename); + + if (fs.existsSync(dest)) { + console.log(chalk.gray(`${version} already present, skipping`)); + manifest[packageName] = { version, filename: wheel.filename }; + return; + } + + try { + await downloadBinary(wheel.url, dest); + console.log(chalk.green(`${version} downloaded`)); + manifest[packageName] = { version, filename: wheel.filename }; + } catch (err) { + console.log(chalk.red(`FAILED — ${err.message}`)); + if (fs.existsSync(dest)) fs.unlinkSync(dest); + } +} + +async function main() { + fs.mkdirSync(PACKAGES_DIR, { recursive: true }); + + // Preserve any previously downloaded packages in the manifest. + let manifest = {}; + if (fs.existsSync(MANIFEST_FILE)) { + try { + manifest = JSON.parse(fs.readFileSync(MANIFEST_FILE, 'utf8')); + } catch { + manifest = {}; + } + } + + console.log(chalk.blue.bold('Downloading MNE-Python wheels from PyPI…')); + for (const pkg of PACKAGES_TO_DOWNLOAD) { + await installPackage(pkg, manifest); + } + + fs.writeFileSync(MANIFEST_FILE, JSON.stringify(manifest, null, 2)); + console.log(chalk.green.bold('\nAll MNE wheels ready.')); + console.log(chalk.gray(`Manifest → ${MANIFEST_FILE}`)); +} + +main().catch((err) => { + console.error(chalk.red('Fatal error:'), err); + process.exit(1); +}); diff --git a/internals/scripts/InstallPyodide.js b/internals/scripts/InstallPyodide.js deleted file mode 100644 index aea88f8..0000000 --- a/internals/scripts/InstallPyodide.js +++ /dev/null @@ -1,66 +0,0 @@ -import chalk from 'chalk'; -import fs from 'fs'; -import https from 'https'; -import mkdirp from 'mkdirp'; -import tar from 'tar-fs'; -import url from 'url'; -import bz2 from 'unbzip2-stream'; - -const PYODIDE_VERSION = '0.27.0'; -const TAR_NAME = `pyodide-${PYODIDE_VERSION}.tar.bz2`; -const TAR_URL = `https://github.com/pyodide/pyodide/releases/download/${PYODIDE_VERSION}/pyodide-${PYODIDE_VERSION}.tar.bz2`; -const PYODIDE_DIR = 'src/renderer/utils/pyodide/src/'; - -const writeAndUnzipFile = (response) => { - const filePath = `${PYODIDE_DIR}${TAR_NAME}`; - const writeStream = fs.createWriteStream(filePath); - response.pipe(writeStream); - - writeStream.on('finish', () => { - console.log(`${chalk.green.bold(`Unzipping pyodide`)}`); - - const readStream = fs.createReadStream(filePath); - try { - readStream.pipe(bz2()).pipe(tar.extract(PYODIDE_DIR)); - } catch (e) { - throw new Error('Error in unzip:', e); - } - - readStream.on('end', () => { - console.log(`${chalk.green.bold(`Unzip successful`)}`); - }); - }); -}; - -const downloadFile = (response) => { - if ( - response.statusCode > 300 && - response.statusCode < 400 && - response.headers.location - ) { - if (url.parse(response.headers.location).hostname) { - https.get(response.headers.location, writeAndUnzipFile); - } else { - https.get( - url.resolve(url.parse(TAR_URL).hostname, response.headers.location), - writeAndUnzipFile - ); - } - } else { - writeAndUnzipFile(response); - } -}; - -(() => { - if (fs.existsSync(`${PYODIDE_DIR}${TAR_NAME}`)) { - console.log( - `${chalk.green.bold(`Pyodide is already present: ${PYODIDE_VERSION}...`)}` - ); - return; - } - console.log( - `${chalk.green.bold(`Downloading pyodide ${PYODIDE_VERSION}...`)}` - ); - mkdirp.sync(`src/renderer/utils/pyodide/src`); - https.get(TAR_URL, downloadFile); -})(); diff --git a/internals/scripts/InstallPyodide.mjs b/internals/scripts/InstallPyodide.mjs new file mode 100644 index 0000000..4ce6125 --- /dev/null +++ b/internals/scripts/InstallPyodide.mjs @@ -0,0 +1,108 @@ +#!/usr/bin/env node +/** + * Downloads the Pyodide core tarball from GitHub releases and extracts it + * into the renderer's public directory so Vite serves the runtime as a static + * asset at /pyodide/… (Vite publicDir → src/renderer/utils/pyodide/src/). + * + * The "core" tarball is ~40 MB and includes the runtime plus the most + * commonly used scientific packages (numpy, scipy, matplotlib, pandas, …). + * It is much smaller than the full Pyodide build (~500 MB). + * + * Usage: node internals/scripts/InstallPyodide.mjs + * Runs automatically via the postinstall npm hook. + */ + +import fs from 'fs'; +import https from 'https'; +import path from 'path'; +import { pipeline } from 'stream/promises'; +import chalk from 'chalk'; +import bz2 from 'unbzip2-stream'; +import tar from 'tar-fs'; + +const PYODIDE_VERSION = '0.29.3'; +const TARBALL_NAME = `pyodide-core-${PYODIDE_VERSION}.tar.bz2`; +const TARBALL_URL = `https://github.com/pyodide/pyodide/releases/download/${PYODIDE_VERSION}/${TARBALL_NAME}`; + +// Vite publicDir root — everything here is served verbatim by the dev server. +const PUBLIC_ROOT = path.resolve('src/renderer/utils/pyodide/src'); +// The tarball extracts into a `pyodide/` subdirectory, which ends up at: +// src/renderer/utils/pyodide/src/pyodide/ → served at /pyodide/ +const DEST_DIR = path.join(PUBLIC_ROOT, 'pyodide'); +const VERSION_FILE = path.join(DEST_DIR, '.pyodide-version'); + +// --------------------------------------------------------------------------- +// Network helpers (follow redirects) +// --------------------------------------------------------------------------- + +function httpsGetResponse(url) { + return new Promise((resolve, reject) => { + https + .get(url, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + resolve(httpsGetResponse(res.headers.location)); + } else { + resolve(res); + } + }) + .on('error', reject); + }); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + // Skip if this exact version is already extracted. + if ( + fs.existsSync(VERSION_FILE) && + fs.readFileSync(VERSION_FILE, 'utf8').trim() === PYODIDE_VERSION + ) { + console.log( + chalk.green.bold(`Pyodide ${PYODIDE_VERSION} already installed, skipping.`) + ); + return; + } + + fs.mkdirSync(PUBLIC_ROOT, { recursive: true }); + + const tarballPath = path.join(PUBLIC_ROOT, TARBALL_NAME); + + // Download the tarball if not already cached. + if (!fs.existsSync(tarballPath)) { + console.log( + chalk.blue.bold(`Downloading Pyodide ${PYODIDE_VERSION} core tarball…`) + ); + const res = await httpsGetResponse(TARBALL_URL); + if (res.statusCode !== 200) { + throw new Error(`Failed to download tarball: HTTP ${res.statusCode}`); + } + await pipeline(res, fs.createWriteStream(tarballPath)); + console.log(chalk.gray(` Saved → ${tarballPath}`)); + } else { + console.log(chalk.gray(` Tarball already cached, skipping download.`)); + } + + // Extract the tarball. The archive contains a top-level `pyodide/` + // directory, so extracting into PUBLIC_ROOT gives us PUBLIC_ROOT/pyodide/. + console.log(chalk.blue.bold(`Extracting…`)); + await pipeline( + fs.createReadStream(tarballPath), + bz2(), + tar.extract(PUBLIC_ROOT) + ); + + // Stamp the installed version and clean up the cached tarball. + fs.writeFileSync(VERSION_FILE, PYODIDE_VERSION); + fs.unlinkSync(tarballPath); + + console.log( + chalk.green.bold(`Pyodide ${PYODIDE_VERSION} installed successfully.`) + ); +} + +main().catch((err) => { + console.error(chalk.red('Fatal error:'), err); + process.exit(1); +}); diff --git a/package.json b/package.json index 4188e67..304e0a8 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "package-mac": "npm run build && electron-builder build --mac", "package-linux": "npm run build && electron-builder build --linux", "package-win": "npm run build && electron-builder build --win --x64", - "postinstall": "electron-builder install-app-deps && node internals/scripts/InstallPyodide.js && node internals/scripts/patchDeps.mjs", + "postinstall": "electron-builder install-app-deps && node internals/scripts/InstallPyodide.mjs && node internals/scripts/InstallMNE.mjs && node internals/scripts/patchDeps.mjs", "lint": "cross-env NODE_ENV=development eslint . --cache", "lint-fix": "npm run lint -- --fix", "lint-styles": "stylelint '**/*.*(css|scss)'", diff --git a/src/renderer/utils/pyodide/webworker.js b/src/renderer/utils/pyodide/webworker.js index eaa2e1f..b50af8c 100644 --- a/src/renderer/utils/pyodide/webworker.js +++ b/src/renderer/utils/pyodide/webworker.js @@ -1,29 +1,67 @@ /** - * This file has been copied from pyodide source and modified to allow - * pyodide to be used in a web worker within this + * Pyodide Web Worker + * + * Load order: + * 1. Pyodide runtime (served as a static asset at /pyodide/). + * 2. Binary packages bundled with Pyodide (numpy, scipy, matplotlib, pandas). + * 3. MNE-Python and its pure-Python deps, installed offline from local + * wheel files that were pre-downloaded by `npm run install-mne-wheels`. + * The manifest at /packages/manifest.json maps package names → filenames. */ -// pyodide is served as a static asset at /pyodide/ (via Vite publicDir). +// Pyodide is served as a static asset at /pyodide/ (via Vite publicDir). // An absolute path is required so importScripts resolves correctly regardless // of where the worker script itself is served from. importScripts('/pyodide/pyodide.js'); async function loadPyodideAndPackages() { self.pyodide = await loadPyodide({ indexURL: '/pyodide/' }); - await self.pyodide.loadPackage(['matplotlib', 'mne', 'pandas']); + + // Load binary packages that are bundled with the Pyodide npm package and + // therefore available locally without any network request. + await self.pyodide.loadPackage(['numpy', 'scipy', 'matplotlib', 'pandas']); + + // Load MNE and its pure-Python dependencies from pre-downloaded wheel files. + // The manifest was written by `node internals/scripts/InstallMNE.mjs`. + let manifest = {}; + try { + const response = await fetch('/packages/manifest.json'); + if (response.ok) { + manifest = await response.json(); + } else { + console.warn('[pyodide worker] manifest.json not found — MNE will not be available'); + } + } catch (err) { + console.warn('[pyodide worker] Could not fetch manifest.json:', err); + } + + const wheelUrls = Object.values(manifest) + .map((entry) => `/packages/${entry.filename}`); + + if (wheelUrls.length > 0) { + await self.pyodide.loadPackage('micropip'); + const micropip = self.pyodide.pyimport('micropip'); + // micropip resolves relative URLs against the worker's base URL. + // Pass absolute URLs so it works regardless of worker location. + const absoluteUrls = wheelUrls.map( + (u) => new URL(u, self.location.origin).href + ); + await micropip.install(absoluteUrls); + } else { + console.warn('[pyodide worker] No MNE wheels found in manifest.'); + } } + let pyodideReadyPromise = loadPyodideAndPackages(); self.onmessage = async (event) => { - // make sure loading is done await pyodideReadyPromise; - // Don't bother yet with this line, suppose our API is built in such a way: + const { data, ...context } = event.data; - // The worker copies the context in its own "memory" (an object mapping name to values) for (const key of Object.keys(context)) { self[key] = context[key]; } - // Now is the easy part, the one that is similar to working in the main thread: + try { self.postMessage({ results: await self.pyodide.runPythonAsync(data), -- 2.51.2 From f6e707750e46ba48c908973d2e2827b131236f81 Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sat, 7 Mar 2026 02:28:44 -0500 Subject: [PATCH 02/12] wip --- internals/scripts/InstallMNE.mjs | 251 ++++++++++++++++-------- internals/scripts/InstallPyodide.mjs | 141 ++++++------- package.json | 2 + src/renderer/epics/pyodideEpics.ts | 4 +- src/renderer/utils/pyodide/index.ts | 9 +- src/renderer/utils/pyodide/webworker.js | 91 +++++---- vite.config.ts | 13 +- 7 files changed, 314 insertions(+), 197 deletions(-) diff --git a/internals/scripts/InstallMNE.mjs b/internals/scripts/InstallMNE.mjs index 23f1466..090807b 100644 --- a/internals/scripts/InstallMNE.mjs +++ b/internals/scripts/InstallMNE.mjs @@ -1,17 +1,23 @@ #!/usr/bin/env node /** - * Downloads MNE-Python and its pure-Python dependencies from PyPI as wheel - * files for offline use with Pyodide. Binary dependencies (numpy, scipy, - * matplotlib, pandas) are already included via the `pyodide` npm package and - * do NOT need to be downloaded here. + * Downloads everything MNE-Python needs to run offline inside Pyodide: * - * Downloaded wheels are saved to: - * src/renderer/utils/pyodide/src/packages/ + * Part 1 — Pyodide binary packages (from the Pyodide CDN) + * Reads pyodide-lock.json that was extracted by InstallPyodide.mjs, + * recursively resolves all dependencies of numpy / scipy / matplotlib / + * pandas, and downloads each .whl (or .zip) into the same /pyodide/ + * directory as the runtime. loadPackage() will find them locally and + * will not need to reach the CDN at runtime. * - * A manifest.json is written there so the web worker knows which filenames - * to pass to micropip.install() at startup. + * Part 2 — Pure-Python packages (from PyPI) + * MNE itself and its pure-Python dependencies (pooch, tqdm, platformdirs) + * are not bundled with Pyodide. These are downloaded as py3-none-any + * wheels into src/renderer/utils/pyodide/src/packages/ and installed via + * micropip at worker startup. A manifest.json is written there so the + * worker knows the exact filenames. * * Usage: node internals/scripts/InstallMNE.mjs + * Runs automatically via the postinstall npm hook. */ import fs from 'fs'; @@ -19,85 +25,173 @@ import https from 'https'; import path from 'path'; import chalk from 'chalk'; -const PACKAGES_DIR = path.resolve( - 'src/renderer/utils/pyodide/src/packages' -); +// --------------------------------------------------------------------------- +// Paths +// --------------------------------------------------------------------------- + +const PYODIDE_DIR = path.resolve('src/renderer/utils/pyodide/src/pyodide'); +const LOCK_FILE = path.join(PYODIDE_DIR, 'pyodide-lock.json'); + +const PACKAGES_DIR = path.resolve('src/renderer/utils/pyodide/src/packages'); const MANIFEST_FILE = path.join(PACKAGES_DIR, 'manifest.json'); -/** - * Pure-Python packages required by MNE that are not bundled with Pyodide. - * Each entry is resolved against the PyPI JSON API to find the latest - * pure-Python wheel (py3-none-any or py2.py3-none-any). - */ -const PACKAGES_TO_DOWNLOAD = [ - 'mne', - 'pooch', - 'tqdm', - 'platformdirs', -]; +// --------------------------------------------------------------------------- +// Root packages whose full transitive dependency tree we need from Pyodide CDN +// --------------------------------------------------------------------------- + +const PYODIDE_ROOT_PACKAGES = ['numpy', 'scipy', 'matplotlib', 'pandas']; + +// --------------------------------------------------------------------------- +// Pure-Python packages to download from PyPI (not bundled with Pyodide) +// --------------------------------------------------------------------------- + +const PYPI_PACKAGES = ['mne', 'pooch', 'tqdm', 'platformdirs']; // --------------------------------------------------------------------------- -// Network helpers +// Shared network helpers // --------------------------------------------------------------------------- -function httpsGet(url) { +function downloadBinary(url, dest) { return new Promise((resolve, reject) => { - const req = https.get(url, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { - if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { - resolve(httpsGet(res.headers.location)); - return; - } - if (res.statusCode !== 200) { - reject(new Error(`HTTP ${res.statusCode} for ${url}`)); - return; - } - let body = ''; - res.setEncoding('utf8'); - res.on('data', (chunk) => { body += chunk; }); - res.on('end', () => resolve(body)); - res.on('error', reject); - }); - req.on('error', reject); + const doGet = (reqUrl) => { + https + .get(reqUrl, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + doGet(res.headers.location); + return; + } + if (res.statusCode !== 200) { + reject(new Error(`HTTP ${res.statusCode} for ${reqUrl}`)); + return; + } + const file = fs.createWriteStream(dest); + res.pipe(file); + file.on('finish', () => file.close(resolve)); + file.on('error', (err) => { fs.unlink(dest, () => {}); reject(err); }); + }) + .on('error', (err) => { fs.unlink(dest, () => {}); reject(err); }); + }; + doGet(url); }); } -function downloadBinary(url, dest) { +function httpsGetText(url) { return new Promise((resolve, reject) => { - const doGet = (reqUrl) => { - https.get(reqUrl, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { + https + .get(url, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { - doGet(res.headers.location); + resolve(httpsGetText(res.headers.location)); return; } if (res.statusCode !== 200) { - reject(new Error(`HTTP ${res.statusCode} for ${reqUrl}`)); + reject(new Error(`HTTP ${res.statusCode} for ${url}`)); return; } - const file = fs.createWriteStream(dest); - res.pipe(file); - file.on('finish', () => file.close(resolve)); - file.on('error', (err) => { fs.unlink(dest, () => {}); reject(err); }); - }).on('error', (err) => { fs.unlink(dest, () => {}); reject(err); }); - }; - doGet(url); + let body = ''; + res.setEncoding('utf8'); + res.on('data', (c) => { body += c; }); + res.on('end', () => resolve(body)); + res.on('error', reject); + }) + .on('error', reject); }); } // --------------------------------------------------------------------------- -// PyPI helpers +// Part 1 — Pyodide binary packages +// --------------------------------------------------------------------------- + +/** + * Recursively walks the `depends` graph in the lock file and returns every + * package entry (including root packages) needed to satisfy the given roots. + * Package name matching is case-insensitive. + */ +function resolveAllDeps(lockPackages, rootNames) { + // Build a lowercase → original-key index for case-insensitive lookup. + const index = {}; + for (const key of Object.keys(lockPackages)) { + index[key.toLowerCase()] = key; + } + + const resolved = new Set(); + const queue = rootNames.map((n) => n.toLowerCase()); + + while (queue.length) { + const lower = queue.shift(); + const key = index[lower]; + if (!key || resolved.has(key)) continue; + resolved.add(key); + for (const dep of lockPackages[key].depends ?? []) { + queue.push(dep.toLowerCase()); + } + } + + return [...resolved].map((key) => lockPackages[key]); +} + +async function downloadPyodidePackages() { + if (!fs.existsSync(LOCK_FILE)) { + console.warn( + chalk.yellow( + ' ⚠ pyodide-lock.json not found — run `npm install` first to ' + + 'extract the Pyodide runtime, then re-run this script.' + ) + ); + return; + } + + const lockData = JSON.parse(fs.readFileSync(LOCK_FILE, 'utf8')); + + // The lock file's info.version may be an internal dev label (e.g. "0.28.0.dev0"). + // Always derive the CDN URL from the installed npm package version instead. + const npmPkgPath = path.resolve('node_modules/pyodide/package.json'); + const cdnVersion = fs.existsSync(npmPkgPath) + ? JSON.parse(fs.readFileSync(npmPkgPath, 'utf8')).version + : lockData.info.version; + const cdnBase = `https://cdn.jsdelivr.net/pyodide/v${cdnVersion}/full/`; + + const allPkgs = resolveAllDeps(lockData.packages, PYODIDE_ROOT_PACKAGES); + + console.log( + chalk.blue.bold( + `Downloading ${allPkgs.length} Pyodide packages from CDN (v${cdnVersion})…` + ) + ); + + for (const pkg of allPkgs) { + process.stdout.write(chalk.blue(` ${pkg.name ?? pkg.file_name}: `)); + + const dest = path.join(PYODIDE_DIR, pkg.file_name); + if (fs.existsSync(dest)) { + console.log(chalk.gray('already present, skipping')); + continue; + } + + const url = cdnBase + pkg.file_name; + try { + await downloadBinary(url, dest); + console.log(chalk.green('downloaded')); + } catch (err) { + console.log(chalk.red(`FAILED — ${err.message}`)); + if (fs.existsSync(dest)) fs.unlinkSync(dest); + } + } +} + +// --------------------------------------------------------------------------- +// Part 2 — Pure-Python packages from PyPI // --------------------------------------------------------------------------- /** - * Returns the best pure-Python wheel for the latest release of `packageName`. + * Queries the PyPI JSON API for `packageName` and returns the best + * pure-Python wheel for the latest release. * Preference: py3-none-any > py2.py3-none-any > *-none-any */ async function resolvePureWheel(packageName) { - const raw = await httpsGet(`https://pypi.org/pypi/${packageName}/json`); + const raw = await httpsGetText(`https://pypi.org/pypi/${packageName}/json`); const data = JSON.parse(raw); const version = data.info.version; - const urls = data.urls; // files for the latest release - - const wheels = urls.filter((f) => f.filename.endsWith('.whl')); + const wheels = data.urls.filter((f) => f.filename.endsWith('.whl')); const ranked = [ wheels.find((f) => f.filename.endsWith('-py3-none-any.whl')), @@ -107,19 +201,15 @@ async function resolvePureWheel(packageName) { if (ranked.length === 0) { throw new Error( - `No pure-Python wheel found for ${packageName} ${version}. ` + - `Binary packages must come from the Pyodide npm bundle.` + `No pure-Python wheel found for ${packageName} ${version}. ` + + `Binary packages must come from the Pyodide CDN.` ); } return { version, wheel: ranked[0] }; } -// --------------------------------------------------------------------------- -// Main -// --------------------------------------------------------------------------- - -async function installPackage(packageName, manifest) { +async function installPyPIPackage(packageName, manifest) { process.stdout.write(chalk.blue(` ${packageName}: `)); let version, wheel; @@ -131,7 +221,6 @@ async function installPackage(packageName, manifest) { } const dest = path.join(PACKAGES_DIR, wheel.filename); - if (fs.existsSync(dest)) { console.log(chalk.gray(`${version} already present, skipping`)); manifest[packageName] = { version, filename: wheel.filename }; @@ -148,27 +237,33 @@ async function installPackage(packageName, manifest) { } } -async function main() { +async function downloadPyPIPackages() { fs.mkdirSync(PACKAGES_DIR, { recursive: true }); - // Preserve any previously downloaded packages in the manifest. + // Preserve previously downloaded packages already recorded in the manifest. let manifest = {}; if (fs.existsSync(MANIFEST_FILE)) { - try { - manifest = JSON.parse(fs.readFileSync(MANIFEST_FILE, 'utf8')); - } catch { - manifest = {}; - } + try { manifest = JSON.parse(fs.readFileSync(MANIFEST_FILE, 'utf8')); } + catch { manifest = {}; } } - console.log(chalk.blue.bold('Downloading MNE-Python wheels from PyPI…')); - for (const pkg of PACKAGES_TO_DOWNLOAD) { - await installPackage(pkg, manifest); + console.log(chalk.blue.bold('\nDownloading MNE-Python wheels from PyPI…')); + for (const pkg of PYPI_PACKAGES) { + await installPyPIPackage(pkg, manifest); } fs.writeFileSync(MANIFEST_FILE, JSON.stringify(manifest, null, 2)); - console.log(chalk.green.bold('\nAll MNE wheels ready.')); - console.log(chalk.gray(`Manifest → ${MANIFEST_FILE}`)); + console.log(chalk.gray(` Manifest → ${MANIFEST_FILE}`)); +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +async function main() { + await downloadPyodidePackages(); + await downloadPyPIPackages(); + console.log(chalk.green.bold('\nAll packages ready.')); } main().catch((err) => { diff --git a/internals/scripts/InstallPyodide.mjs b/internals/scripts/InstallPyodide.mjs index 4ce6125..16973e1 100644 --- a/internals/scripts/InstallPyodide.mjs +++ b/internals/scripts/InstallPyodide.mjs @@ -1,104 +1,105 @@ #!/usr/bin/env node /** - * Downloads the Pyodide core tarball from GitHub releases and extracts it - * into the renderer's public directory so Vite serves the runtime as a static - * asset at /pyodide/… (Vite publicDir → src/renderer/utils/pyodide/src/). + * Copies the Pyodide runtime from the installed npm package into the renderer's + * publicDir so Vite can serve it as static assets. * - * The "core" tarball is ~40 MB and includes the runtime plus the most - * commonly used scientific packages (numpy, scipy, matplotlib, pandas, …). - * It is much smaller than the full Pyodide build (~500 MB). + * Source: node_modules/pyodide/ + * Dest: src/renderer/utils/pyodide/src/pyodide/ + * + * Key files copied: + * pyodide.mjs – ESM entry point (imported by the web worker via npm) + * pyodide.js – UMD fallback + * pyodide.asm.js – compiled Python interpreter + * pyodide.asm.wasm – WebAssembly binary + * python_stdlib.zip – Python standard library + * pyodide-lock.json – package registry (read by InstallMNE.mjs) + * + * Intentionally skipped: + * package.json – would make Vite treat the dir as an npm package + * and attempt to transform pyodide.mjs as a module + * *.d.ts – TypeScript declaration files, not needed at runtime + * *.html – console demo pages + * README.md – documentation + * *.map – source maps (large, optional for debugging) + * + * A version stamp (.pyodide-version) is written so subsequent runs are skipped + * when the installed version has not changed. * * Usage: node internals/scripts/InstallPyodide.mjs * Runs automatically via the postinstall npm hook. */ import fs from 'fs'; -import https from 'https'; import path from 'path'; -import { pipeline } from 'stream/promises'; +import { createRequire } from 'module'; import chalk from 'chalk'; -import bz2 from 'unbzip2-stream'; -import tar from 'tar-fs'; -const PYODIDE_VERSION = '0.29.3'; -const TARBALL_NAME = `pyodide-core-${PYODIDE_VERSION}.tar.bz2`; -const TARBALL_URL = `https://github.com/pyodide/pyodide/releases/download/${PYODIDE_VERSION}/${TARBALL_NAME}`; +const _require = createRequire(import.meta.url); -// Vite publicDir root — everything here is served verbatim by the dev server. -const PUBLIC_ROOT = path.resolve('src/renderer/utils/pyodide/src'); -// The tarball extracts into a `pyodide/` subdirectory, which ends up at: -// src/renderer/utils/pyodide/src/pyodide/ → served at /pyodide/ -const DEST_DIR = path.join(PUBLIC_ROOT, 'pyodide'); +const DEST_DIR = path.resolve('src/renderer/utils/pyodide/src/pyodide'); const VERSION_FILE = path.join(DEST_DIR, '.pyodide-version'); -// --------------------------------------------------------------------------- -// Network helpers (follow redirects) -// --------------------------------------------------------------------------- +// Files to exclude from the copy. +const SKIP_EXTENSIONS = new Set(['.d.ts', '.map', '.html', '.md']); +const SKIP_FILES = new Set(['package.json', 'README.md']); -function httpsGetResponse(url) { - return new Promise((resolve, reject) => { - https - .get(url, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { - if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { - resolve(httpsGetResponse(res.headers.location)); - } else { - resolve(res); - } - }) - .on('error', reject); - }); +function shouldSkip(filename) { + if (SKIP_FILES.has(filename)) return true; + for (const ext of SKIP_EXTENSIONS) { + if (filename.endsWith(ext)) return true; + } + return false; } -// --------------------------------------------------------------------------- -// Main -// --------------------------------------------------------------------------- - async function main() { - // Skip if this exact version is already extracted. + // Locate the pyodide package directory via Node's module resolution. + let pyodideDir; + try { + pyodideDir = path.dirname(_require.resolve('pyodide/package.json')); + } catch { + console.error( + chalk.red( + 'pyodide not found in node_modules. Run `npm install` first.' + ) + ); + process.exit(1); + } + + const version = JSON.parse( + fs.readFileSync(path.join(pyodideDir, 'package.json'), 'utf8') + ).version; + + // Skip if this version was already installed. if ( fs.existsSync(VERSION_FILE) && - fs.readFileSync(VERSION_FILE, 'utf8').trim() === PYODIDE_VERSION + fs.readFileSync(VERSION_FILE, 'utf8').trim() === version ) { - console.log( - chalk.green.bold(`Pyodide ${PYODIDE_VERSION} already installed, skipping.`) - ); + console.log(chalk.gray(`Pyodide ${version} already installed — skipping.`)); return; } - fs.mkdirSync(PUBLIC_ROOT, { recursive: true }); + console.log( + chalk.blue.bold(`Installing Pyodide ${version} from node_modules…`) + ); + fs.mkdirSync(DEST_DIR, { recursive: true }); - const tarballPath = path.join(PUBLIC_ROOT, TARBALL_NAME); + const files = fs.readdirSync(pyodideDir); + for (const file of files) { + if (shouldSkip(file)) continue; - // Download the tarball if not already cached. - if (!fs.existsSync(tarballPath)) { - console.log( - chalk.blue.bold(`Downloading Pyodide ${PYODIDE_VERSION} core tarball…`) - ); - const res = await httpsGetResponse(TARBALL_URL); - if (res.statusCode !== 200) { - throw new Error(`Failed to download tarball: HTTP ${res.statusCode}`); - } - await pipeline(res, fs.createWriteStream(tarballPath)); - console.log(chalk.gray(` Saved → ${tarballPath}`)); - } else { - console.log(chalk.gray(` Tarball already cached, skipping download.`)); - } + const src = path.join(pyodideDir, file); + const dest = path.join(DEST_DIR, file); - // Extract the tarball. The archive contains a top-level `pyodide/` - // directory, so extracting into PUBLIC_ROOT gives us PUBLIC_ROOT/pyodide/. - console.log(chalk.blue.bold(`Extracting…`)); - await pipeline( - fs.createReadStream(tarballPath), - bz2(), - tar.extract(PUBLIC_ROOT) - ); + if (fs.statSync(src).isDirectory()) continue; - // Stamp the installed version and clean up the cached tarball. - fs.writeFileSync(VERSION_FILE, PYODIDE_VERSION); - fs.unlinkSync(tarballPath); + process.stdout.write(chalk.blue(` ${file}: `)); + fs.copyFileSync(src, dest); + console.log(chalk.green('copied')); + } + fs.writeFileSync(VERSION_FILE, version); console.log( - chalk.green.bold(`Pyodide ${PYODIDE_VERSION} installed successfully.`) + chalk.green.bold(`\nPyodide ${version} ready at ${DEST_DIR}`) ); } diff --git a/package.json b/package.json index 304e0a8..8f4ae5a 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "package-linux": "npm run build && electron-builder build --linux", "package-win": "npm run build && electron-builder build --win --x64", "postinstall": "electron-builder install-app-deps && node internals/scripts/InstallPyodide.mjs && node internals/scripts/InstallMNE.mjs && node internals/scripts/patchDeps.mjs", + "install-pyodide": "node internals/scripts/InstallPyodide.mjs && node internals/scripts/InstallMNE.mjs", "lint": "cross-env NODE_ENV=development eslint . --cache", "lint-fix": "npm run lint -- --fix", "lint-styles": "stylelint '**/*.*(css|scss)'", @@ -208,6 +209,7 @@ "papaparse": "^5.5.3", "pathe": "^2.0.3", "plotly.js": "^3.4.0", + "pyodide": "^0.29.3", "rc-slider": "9.2.4", "react": "^18.x", "react-dom": "^18.x", diff --git a/src/renderer/epics/pyodideEpics.ts b/src/renderer/epics/pyodideEpics.ts index ce0e58d..6337db2 100644 --- a/src/renderer/epics/pyodideEpics.ts +++ b/src/renderer/epics/pyodideEpics.ts @@ -76,8 +76,8 @@ const pyodideErrorEpic: Epic< ); // Once pyodide webworker is created, -// Create an observable of events that corresond to what it retjurns -// and then emite those events as redux actions +// Create an observable of events that corresond to what it returns +// and then emits those events as redux actions const pyodideMessageEpic: Epic< PyodideActionType, PyodideActionType, diff --git a/src/renderer/utils/pyodide/index.ts b/src/renderer/utils/pyodide/index.ts index 9572f38..37dbcf1 100644 --- a/src/renderer/utils/pyodide/index.ts +++ b/src/renderer/utils/pyodide/index.ts @@ -12,8 +12,10 @@ import utilsPy from './utils.py?raw'; // Imports and Utility functions export const loadPyodide = async () => { - // Classic worker (importScripts used inside cannot run in module workers) - const freshWorker = new Worker(new URL('./webworker.js', import.meta.url)); + // Module worker — required for Pyodide 0.26+ which ships pyodide.mjs as ESM. + const freshWorker = new Worker(new URL('./webworker.js', import.meta.url), { + type: 'module', + }); return freshWorker; }; @@ -126,7 +128,8 @@ export const plotTestPlot = async (worker: Worker | null) => { return; } return worker.postMessage({ - data: `import matplotlib.pyplot as plt; fig= plt.plot([1,2,3,4])`, + // data: `import matplotlib.pyplot as plt; fig= plt.plot([1,2,3,4])`, + data: `[1,2,3,4]` }); }; diff --git a/src/renderer/utils/pyodide/webworker.js b/src/renderer/utils/pyodide/webworker.js index b50af8c..4e37315 100644 --- a/src/renderer/utils/pyodide/webworker.js +++ b/src/renderer/utils/pyodide/webworker.js @@ -1,71 +1,82 @@ /** - * Pyodide Web Worker + * Pyodide Web Worker — ES module worker following the pattern from + * https://gitlab.com/castedo/pyodide-worker-example * - * Load order: - * 1. Pyodide runtime (served as a static asset at /pyodide/). - * 2. Binary packages bundled with Pyodide (numpy, scipy, matplotlib, pandas). - * 3. MNE-Python and its pure-Python deps, installed offline from local - * wheel files that were pre-downloaded by `npm run install-mne-wheels`. - * The manifest at /packages/manifest.json maps package names → filenames. + * Loading strategy + * ---------------- + * 1. `import { loadPyodide } from "pyodide"` — Vite resolves this to + * node_modules/pyodide/pyodide.mjs and serves it from /@fs/… in dev mode, + * completely bypassing any publicDir transform issues. + * + * 2. `indexURL: '/pyodide/'` — tells pyodide where to find pyodide-lock.json + * and binary package wheels (.whl). These are served from publicDir: + * src/renderer/utils/pyodide/src/pyodide/ + * which is populated by: + * • InstallPyodide.mjs (copies pyodide-lock.json + runtime from npm) + * • InstallMNE.mjs (downloads binary wheels from Pyodide CDN) + * + * 3. Binary packages (numpy / scipy / matplotlib / pandas) — loaded via + * pyodide.loadPackage(), resolved from local /pyodide/ files. + * + * 4. MNE + pure-Python deps — installed via micropip from pre-downloaded + * wheels in /packages/, listed in /packages/manifest.json. + * Populated by InstallMNE.mjs Part 2 (PyPI). */ -// Pyodide is served as a static asset at /pyodide/ (via Vite publicDir). -// An absolute path is required so importScripts resolves correctly regardless -// of where the worker script itself is served from. -importScripts('/pyodide/pyodide.js'); +import { loadPyodide } from 'pyodide'; -async function loadPyodideAndPackages() { - self.pyodide = await loadPyodide({ indexURL: '/pyodide/' }); +async function initPyodide() { + // indexURL tells pyodide where to load pyodide-lock.json and binary wheels. + // The publicDir (src/renderer/utils/pyodide/src/) is served at the web root, + // so /pyodide/ maps to src/renderer/utils/pyodide/src/pyodide/. + const pyodide = await loadPyodide({ indexURL: '/pyodide/' }); - // Load binary packages that are bundled with the Pyodide npm package and - // therefore available locally without any network request. - await self.pyodide.loadPackage(['numpy', 'scipy', 'matplotlib', 'pandas']); + // Load binary packages from locally served .whl files. + await pyodide.loadPackage(['numpy', 'scipy', 'matplotlib', 'pandas']); - // Load MNE and its pure-Python dependencies from pre-downloaded wheel files. - // The manifest was written by `node internals/scripts/InstallMNE.mjs`. + // Install MNE and its pure-Python deps from pre-downloaded wheels. let manifest = {}; try { - const response = await fetch('/packages/manifest.json'); - if (response.ok) { - manifest = await response.json(); + const res = await fetch(new URL('/packages/manifest.json', self.location.href).href); + if (res.ok) { + manifest = await res.json(); } else { - console.warn('[pyodide worker] manifest.json not found — MNE will not be available'); + console.warn('[pyodide worker] manifest.json not found — MNE unavailable'); } } catch (err) { console.warn('[pyodide worker] Could not fetch manifest.json:', err); } - const wheelUrls = Object.values(manifest) - .map((entry) => `/packages/${entry.filename}`); + const wheelUrls = Object.values(manifest).map( + (entry) => new URL(`/packages/${entry.filename}`, self.location.href).href + ); if (wheelUrls.length > 0) { - await self.pyodide.loadPackage('micropip'); - const micropip = self.pyodide.pyimport('micropip'); - // micropip resolves relative URLs against the worker's base URL. - // Pass absolute URLs so it works regardless of worker location. - const absoluteUrls = wheelUrls.map( - (u) => new URL(u, self.location.origin).href - ); - await micropip.install(absoluteUrls); + await pyodide.loadPackage('micropip'); + const micropip = pyodide.pyimport('micropip'); + await micropip.install(wheelUrls); } else { - console.warn('[pyodide worker] No MNE wheels found in manifest.'); + console.warn('[pyodide worker] No MNE wheels in manifest — skipping micropip install'); } + + return pyodide; } -let pyodideReadyPromise = loadPyodideAndPackages(); +// Start loading immediately so it is ready when the first message arrives. +const pyodideReadyPromise = initPyodide(); self.onmessage = async (event) => { - await pyodideReadyPromise; + const pyodide = await pyodideReadyPromise; const { data, ...context } = event.data; - for (const key of Object.keys(context)) { - self[key] = context[key]; + + // Expose context values as globals so Python can access them via the js module. + for (const [key, value] of Object.entries(context)) { + self[key] = value; } try { - self.postMessage({ - results: await self.pyodide.runPythonAsync(data), - }); + self.postMessage({ results: await pyodide.runPythonAsync(data) }); } catch (error) { self.postMessage({ error: error.message }); } diff --git a/vite.config.ts b/vite.config.ts index 05871b9..7ecc428 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -44,10 +44,8 @@ export default defineConfig({ // ------------------------------------------------------------------ renderer: { // Serve the pyodide runtime files as static assets so Vite does NOT - // transform them. importScripts() in a classic worker cannot load - // ES modules; Vite's HMR injection turns .js files into ESM, breaking - // the worker. Files in publicDir are served verbatim at the root URL: - // /pyodide/pyodide.js, /pyodide/pyodide.asm.js, etc. + // transform them. Files in publicDir are served verbatim at the root URL: + // /pyodide/pyodide.mjs, /pyodide/pyodide.asm.js, /packages/*.whl, etc. publicDir: path.resolve(__dirname, 'src/renderer/utils/pyodide/src'), plugins: [ react({ @@ -77,6 +75,13 @@ export default defineConfig({ }, optimizeDeps: { include: ['@neurosity/pipes'], + // Prevent Vite from pre-bundling pyodide. In dev mode it will be served + // raw from node_modules via /@fs/, which is what pyodide.mjs expects. + exclude: ['pyodide'], + }, + worker: { + // ES module workers are required for `import { loadPyodide } from "pyodide"`. + format: 'es', }, build: { rollupOptions: { -- 2.51.2 From 5dfb75407aeba7a27353518a053db7f0ca912f1e Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 15 Mar 2026 12:09:16 -0400 Subject: [PATCH 03/12] Update package-lock.json --- package-lock.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/package-lock.json b/package-lock.json index 8fcfc16..e0a77cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,6 +37,7 @@ "papaparse": "^5.5.3", "pathe": "^2.0.3", "plotly.js": "^3.4.0", + "pyodide": "^0.29.3", "rc-slider": "9.2.4", "react": "^18.x", "react-dom": "^18.x", @@ -4538,6 +4539,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -15304,6 +15311,19 @@ "integrity": "sha1-H+Bk+wrIUfDeYTIKi/eWg2Qi8z4= sha512-QFADYnsVoBMw1srW7OVKEYjG+MbIa49s54w1MA1EDY6r2r/sTcKKYqRX1f4GYvnXP7eN/Pe9HFcX+hwzmrXRHA==", "license": "MIT" }, + "node_modules/pyodide": { + "version": "0.29.3", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-0.29.3.tgz", + "integrity": "sha512-22UBuhOJawj7vKUnS7/F3xK+515LJdjiMAHoCfuS6/PbHiOrSQVnYwDe+2sbVwiOZ3sMMexdXICew6NqOMQGgA==", + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/qified": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/qified/-/qified-0.6.0.tgz", -- 2.51.2 From f798b15d3cc393149373f20afdc602810c17920d Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 15 Mar 2026 12:11:49 -0400 Subject: [PATCH 04/12] update folder so there's no duplicate name --- .gitignore | 2 +- src/renderer/utils/{pyodide => webworker}/functions.ts | 0 src/renderer/utils/{pyodide => webworker}/index.ts | 0 src/renderer/utils/{pyodide => webworker}/patches.py | 0 src/renderer/utils/{pyodide => webworker}/utils.py | 0 src/renderer/utils/{pyodide => webworker}/webworker.js | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename src/renderer/utils/{pyodide => webworker}/functions.ts (100%) rename src/renderer/utils/{pyodide => webworker}/index.ts (100%) rename src/renderer/utils/{pyodide => webworker}/patches.py (100%) rename src/renderer/utils/{pyodide => webworker}/utils.py (100%) rename src/renderer/utils/{pyodide => webworker}/webworker.js (100%) diff --git a/.gitignore b/.gitignore index 4666eb4..b177d2b 100644 --- a/.gitignore +++ b/.gitignore @@ -54,4 +54,4 @@ npm-debug.log.* keys.js app/utils/pyodide/src -src/renderer/utils/pyodide/src +src/renderer/utils/webworker/src diff --git a/src/renderer/utils/pyodide/functions.ts b/src/renderer/utils/webworker/functions.ts similarity index 100% rename from src/renderer/utils/pyodide/functions.ts rename to src/renderer/utils/webworker/functions.ts diff --git a/src/renderer/utils/pyodide/index.ts b/src/renderer/utils/webworker/index.ts similarity index 100% rename from src/renderer/utils/pyodide/index.ts rename to src/renderer/utils/webworker/index.ts diff --git a/src/renderer/utils/pyodide/patches.py b/src/renderer/utils/webworker/patches.py similarity index 100% rename from src/renderer/utils/pyodide/patches.py rename to src/renderer/utils/webworker/patches.py diff --git a/src/renderer/utils/pyodide/utils.py b/src/renderer/utils/webworker/utils.py similarity index 100% rename from src/renderer/utils/pyodide/utils.py rename to src/renderer/utils/webworker/utils.py diff --git a/src/renderer/utils/pyodide/webworker.js b/src/renderer/utils/webworker/webworker.js similarity index 100% rename from src/renderer/utils/pyodide/webworker.js rename to src/renderer/utils/webworker/webworker.js -- 2.51.2 From 3252e70a06d37ccfdb81eedc7ce1fcd5892a353f Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 15 Mar 2026 12:18:44 -0400 Subject: [PATCH 05/12] cleanup reference; remove old ignored files --- .gitignore | 29 +---------------------- .llms/CLAUDE.md | 4 ++-- eslint.config.mjs | 2 +- internals/scripts/InstallMNE.mjs | 6 ++--- internals/scripts/InstallPyodide.mjs | 4 ++-- package.json | 2 +- src/renderer/epics/pyodideEpics.ts | 4 ++-- src/renderer/utils/webworker/webworker.js | 6 ++--- vite.config.ts | 2 +- 9 files changed, 16 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index b177d2b..eb02e03 100644 --- a/.gitignore +++ b/.gitignore @@ -7,20 +7,10 @@ pids *.pid *.seed -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - # Coverage directory used by tools like istanbul coverage -# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (http://nodejs.org/api/addons.html) -build/Release +# Compiled binary addons .eslintcache # Dependency directory @@ -35,23 +25,6 @@ release out dist -# Old webpack build artifacts -app/main.prod.js -app/main.prod.js.map -app/renderer.prod.js -app/renderer.prod.js.map -app/style.css -app/style.css.map -dll -main.js -main.js.map - .idea -npm-debug.log.* -*.css.d.ts -*.sass.d.ts -*.scss.d.ts keys.js - -app/utils/pyodide/src src/renderer/utils/webworker/src diff --git a/.llms/CLAUDE.md b/.llms/CLAUDE.md index a133315..853b9ac 100644 --- a/.llms/CLAUDE.md +++ b/.llms/CLAUDE.md @@ -24,7 +24,7 @@ A priority for this codebase is extensibility modularity and hackability. There - `src/renderer/` — React renderer process - `src/preload/` — Electron preload scripts - `src/renderer/experiments/` — Lab.js experiment files -- `src/renderer/utils/pyodide/` — Pyodide WASM Python runtime +- `src/renderer/utils/webworker/` — Pyodide WASM Python runtime ## Dev Workflow ```bash @@ -45,7 +45,7 @@ npm run package # Build + package for current platform - Keep Electron main/renderer separation strict — use preload IPC bridges ## Out of Scope -- Do not modify `src/renderer/utils/pyodide/src/` directly; it is managed by `InstallPyodide.js` +- Do not modify `src/renderer/utils/webworker/src/` directly; it is managed by `InstallPyodide.js` - Do not alter `electron-builder` publish config without confirming release intent ## LLM Context diff --git a/eslint.config.mjs b/eslint.config.mjs index 7e24f86..50539aa 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -23,7 +23,7 @@ export default [ 'dist/**', 'coverage/**', '.worktrees/**', - 'src/renderer/utils/pyodide/src/**', + 'src/renderer/utils/webworker/src/**', '**/*.css.d.ts', '**/*.scss.d.ts', ], diff --git a/internals/scripts/InstallMNE.mjs b/internals/scripts/InstallMNE.mjs index 090807b..37e9870 100644 --- a/internals/scripts/InstallMNE.mjs +++ b/internals/scripts/InstallMNE.mjs @@ -12,7 +12,7 @@ * Part 2 — Pure-Python packages (from PyPI) * MNE itself and its pure-Python dependencies (pooch, tqdm, platformdirs) * are not bundled with Pyodide. These are downloaded as py3-none-any - * wheels into src/renderer/utils/pyodide/src/packages/ and installed via + * wheels into src/renderer/utils/webworker/src/packages/ and installed via * micropip at worker startup. A manifest.json is written there so the * worker knows the exact filenames. * @@ -29,10 +29,10 @@ import chalk from 'chalk'; // Paths // --------------------------------------------------------------------------- -const PYODIDE_DIR = path.resolve('src/renderer/utils/pyodide/src/pyodide'); +const PYODIDE_DIR = path.resolve('src/renderer/utils/webworker/src/pyodide'); const LOCK_FILE = path.join(PYODIDE_DIR, 'pyodide-lock.json'); -const PACKAGES_DIR = path.resolve('src/renderer/utils/pyodide/src/packages'); +const PACKAGES_DIR = path.resolve('src/renderer/utils/webworker/src/packages'); const MANIFEST_FILE = path.join(PACKAGES_DIR, 'manifest.json'); // --------------------------------------------------------------------------- diff --git a/internals/scripts/InstallPyodide.mjs b/internals/scripts/InstallPyodide.mjs index 16973e1..665f1c2 100644 --- a/internals/scripts/InstallPyodide.mjs +++ b/internals/scripts/InstallPyodide.mjs @@ -4,7 +4,7 @@ * publicDir so Vite can serve it as static assets. * * Source: node_modules/pyodide/ - * Dest: src/renderer/utils/pyodide/src/pyodide/ + * Dest: src/renderer/utils/webworker/src/pyodide/ * * Key files copied: * pyodide.mjs – ESM entry point (imported by the web worker via npm) @@ -36,7 +36,7 @@ import chalk from 'chalk'; const _require = createRequire(import.meta.url); -const DEST_DIR = path.resolve('src/renderer/utils/pyodide/src/pyodide'); +const DEST_DIR = path.resolve('src/renderer/utils/webworker/src/pyodide'); const VERSION_FILE = path.join(DEST_DIR, '.pyodide-version'); // Files to exclude from the copy. diff --git a/package.json b/package.json index 8f4ae5a..766fc98 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,7 @@ "filter": "**/*" }, { - "from": "./src/renderer/utils/pyodide/src/", + "from": "./src/renderer/utils/webworker/src/", "to": "pyodide", "filter": "**/*" } diff --git a/src/renderer/epics/pyodideEpics.ts b/src/renderer/epics/pyodideEpics.ts index 6337db2..dfc40b0 100644 --- a/src/renderer/epics/pyodideEpics.ts +++ b/src/renderer/epics/pyodideEpics.ts @@ -23,14 +23,14 @@ import { loadPatches, applyPatches, loadUtils, -} from '../utils/pyodide'; +} from '../utils/webworker'; import { EMOTIV_CHANNELS, DEVICES, MUSE_CHANNELS, PYODIDE_VARIABLE_NAMES, } from '../constants/constants'; -import { parseSingleQuoteJSON } from '../utils/pyodide/functions'; +import { parseSingleQuoteJSON } from '../utils/webworker/functions'; import { readFiles } from '../utils/filesystem/read'; diff --git a/src/renderer/utils/webworker/webworker.js b/src/renderer/utils/webworker/webworker.js index 4e37315..a599f27 100644 --- a/src/renderer/utils/webworker/webworker.js +++ b/src/renderer/utils/webworker/webworker.js @@ -10,7 +10,7 @@ * * 2. `indexURL: '/pyodide/'` — tells pyodide where to find pyodide-lock.json * and binary package wheels (.whl). These are served from publicDir: - * src/renderer/utils/pyodide/src/pyodide/ + * src/renderer/utils/webworker/src/pyodide/ * which is populated by: * • InstallPyodide.mjs (copies pyodide-lock.json + runtime from npm) * • InstallMNE.mjs (downloads binary wheels from Pyodide CDN) @@ -27,8 +27,8 @@ import { loadPyodide } from 'pyodide'; async function initPyodide() { // indexURL tells pyodide where to load pyodide-lock.json and binary wheels. - // The publicDir (src/renderer/utils/pyodide/src/) is served at the web root, - // so /pyodide/ maps to src/renderer/utils/pyodide/src/pyodide/. + // The publicDir (src/renderer/utils/webworker/src/) is served at the web root, + // so /pyodide/ maps to src/renderer/utils/webworker/src/pyodide/. const pyodide = await loadPyodide({ indexURL: '/pyodide/' }); // Load binary packages from locally served .whl files. diff --git a/vite.config.ts b/vite.config.ts index 7ecc428..fc29693 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -46,7 +46,7 @@ export default defineConfig({ // Serve the pyodide runtime files as static assets so Vite does NOT // transform them. Files in publicDir are served verbatim at the root URL: // /pyodide/pyodide.mjs, /pyodide/pyodide.asm.js, /packages/*.whl, etc. - publicDir: path.resolve(__dirname, 'src/renderer/utils/pyodide/src'), + publicDir: path.resolve(__dirname, 'src/renderer/utils/webworker/src'), plugins: [ react({ jsxRuntime: 'classic', // React 16 does not ship react/jsx-runtime -- 2.51.2 From ae23b182c09297fab7b1f78a1233cdfcf3788c47 Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 15 Mar 2026 12:29:51 -0400 Subject: [PATCH 06/12] initial cleanup --- src/renderer/utils/webworker/index.ts | 4 +-- src/renderer/utils/webworker/webworker.js | 39 +++++++++++++++++++---- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/renderer/utils/webworker/index.ts b/src/renderer/utils/webworker/index.ts index 37dbcf1..c8e965b 100644 --- a/src/renderer/utils/webworker/index.ts +++ b/src/renderer/utils/webworker/index.ts @@ -36,9 +36,7 @@ export const loadUtils = async (worker: Worker) => export const loadCSV = async (worker: Worker, csvArray: Array) => { // TODO: Pass attached variable name as parameter to load_data - // @ts-expect-error - window.csvArray = csvArray; - await worker.postMessage({ data: `raw = load_data()` }); + await worker.postMessage({ data: `raw = load_data()`, csvArray }); }; // --------------------------- diff --git a/src/renderer/utils/webworker/webworker.js b/src/renderer/utils/webworker/webworker.js index a599f27..99767bd 100644 --- a/src/renderer/utils/webworker/webworker.js +++ b/src/renderer/utils/webworker/webworker.js @@ -25,11 +25,31 @@ import { loadPyodide } from 'pyodide'; +/** + * Derive the renderer root URL from the worker's own location. + * + * Dev (Vite HTTP server): self.location.href is an http:// URL — the root is + * just the origin, so root-relative paths work as normal. + * + * Production (Electron file://): Vite bundles workers into assets/, so the + * renderer root is one directory above the worker file. + */ +function getRendererBaseUrl() { + const loc = self.location.href; + if (loc.startsWith('http')) { + return new URL('/', loc).href; + } + // file:// — go up from assets/webworker-[hash].js to the renderer root + return new URL('../', loc).href; +} + async function initPyodide() { + const base = getRendererBaseUrl(); + // indexURL tells pyodide where to load pyodide-lock.json and binary wheels. - // The publicDir (src/renderer/utils/webworker/src/) is served at the web root, - // so /pyodide/ maps to src/renderer/utils/webworker/src/pyodide/. - const pyodide = await loadPyodide({ indexURL: '/pyodide/' }); + // Resolved from the renderer root so it works under both HTTP (dev) and + // file:// (Electron production). + const pyodide = await loadPyodide({ indexURL: new URL('pyodide/', base).href }); // Load binary packages from locally served .whl files. await pyodide.loadPackage(['numpy', 'scipy', 'matplotlib', 'pandas']); @@ -37,7 +57,7 @@ async function initPyodide() { // Install MNE and its pure-Python deps from pre-downloaded wheels. let manifest = {}; try { - const res = await fetch(new URL('/packages/manifest.json', self.location.href).href); + const res = await fetch(new URL('packages/manifest.json', base).href); if (res.ok) { manifest = await res.json(); } else { @@ -48,7 +68,7 @@ async function initPyodide() { } const wheelUrls = Object.values(manifest).map( - (entry) => new URL(`/packages/${entry.filename}`, self.location.href).href + (entry) => new URL(`packages/${entry.filename}`, base).href ); if (wheelUrls.length > 0) { @@ -66,7 +86,14 @@ async function initPyodide() { const pyodideReadyPromise = initPyodide(); self.onmessage = async (event) => { - const pyodide = await pyodideReadyPromise; + // Propagate init failures back to the main thread rather than hanging silently. + let pyodide; + try { + pyodide = await pyodideReadyPromise; + } catch (error) { + self.postMessage({ error: `Pyodide init failed: ${error.message}` }); + return; + } const { data, ...context } = event.data; -- 2.51.2 From 0072375c8d472b7e1f435029411e951beb7f50f6 Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 15 Mar 2026 13:39:03 -0400 Subject: [PATCH 07/12] maybe a solution --- src/renderer/utils/webworker/webworker.js | 94 ++++------------------- vite.config.ts | 2 +- 2 files changed, 17 insertions(+), 79 deletions(-) diff --git a/src/renderer/utils/webworker/webworker.js b/src/renderer/utils/webworker/webworker.js index 99767bd..6ca003c 100644 --- a/src/renderer/utils/webworker/webworker.js +++ b/src/renderer/utils/webworker/webworker.js @@ -1,89 +1,27 @@ /** - * Pyodide Web Worker — ES module worker following the pattern from - * https://gitlab.com/castedo/pyodide-worker-example + * Pyodide Web Worker — local node_modules implementation. * * Loading strategy * ---------------- - * 1. `import { loadPyodide } from "pyodide"` — Vite resolves this to - * node_modules/pyodide/pyodide.mjs and serves it from /@fs/… in dev mode, - * completely bypassing any publicDir transform issues. + * Use Vite's `?url` suffix on 'pyodide/pyodide.mjs' to get the resolved file URL + * at build/dev time (/@fs/... in dev, an asset URL in prod), then dynamically + * import from that URL. This bypasses Vite's SPA fallback and lets pyodide.mjs + * resolve all sibling assets (pyodide.asm.wasm, pyodide-lock.json, etc.) via + * import.meta.url — no CDN required. * - * 2. `indexURL: '/pyodide/'` — tells pyodide where to find pyodide-lock.json - * and binary package wheels (.whl). These are served from publicDir: - * src/renderer/utils/webworker/src/pyodide/ - * which is populated by: - * • InstallPyodide.mjs (copies pyodide-lock.json + runtime from npm) - * • InstallMNE.mjs (downloads binary wheels from Pyodide CDN) - * - * 3. Binary packages (numpy / scipy / matplotlib / pandas) — loaded via - * pyodide.loadPackage(), resolved from local /pyodide/ files. - * - * 4. MNE + pure-Python deps — installed via micropip from pre-downloaded - * wheels in /packages/, listed in /packages/manifest.json. - * Populated by InstallMNE.mjs Part 2 (PyPI). + * Production builds use the files copied to publicDir by InstallPyodide.mjs. */ -import { loadPyodide } from 'pyodide'; - -/** - * Derive the renderer root URL from the worker's own location. - * - * Dev (Vite HTTP server): self.location.href is an http:// URL — the root is - * just the origin, so root-relative paths work as normal. - * - * Production (Electron file://): Vite bundles workers into assets/, so the - * renderer root is one directory above the worker file. - */ -function getRendererBaseUrl() { - const loc = self.location.href; - if (loc.startsWith('http')) { - return new URL('/', loc).href; - } - // file:// — go up from assets/webworker-[hash].js to the renderer root - return new URL('../', loc).href; -} - -async function initPyodide() { - const base = getRendererBaseUrl(); - - // indexURL tells pyodide where to load pyodide-lock.json and binary wheels. - // Resolved from the renderer root so it works under both HTTP (dev) and - // file:// (Electron production). - const pyodide = await loadPyodide({ indexURL: new URL('pyodide/', base).href }); - - // Load binary packages from locally served .whl files. - await pyodide.loadPackage(['numpy', 'scipy', 'matplotlib', 'pandas']); - - // Install MNE and its pure-Python deps from pre-downloaded wheels. - let manifest = {}; - try { - const res = await fetch(new URL('packages/manifest.json', base).href); - if (res.ok) { - manifest = await res.json(); - } else { - console.warn('[pyodide worker] manifest.json not found — MNE unavailable'); - } - } catch (err) { - console.warn('[pyodide worker] Could not fetch manifest.json:', err); - } - - const wheelUrls = Object.values(manifest).map( - (entry) => new URL(`packages/${entry.filename}`, base).href - ); - - if (wheelUrls.length > 0) { - await pyodide.loadPackage('micropip'); - const micropip = pyodide.pyimport('micropip'); - await micropip.install(wheelUrls); - } else { - console.warn('[pyodide worker] No MNE wheels in manifest — skipping micropip install'); - } - - return pyodide; -} +// ?url tells Vite to resolve the path and return a URL string rather than bundling +// the module. In dev mode this is a /@fs/ URL (bypasses SPA fallback); in prod it +// is an asset URL. We then dynamically import from that URL so pyodide.mjs can +// resolve all its sibling assets (pyodide.asm.wasm, etc.) via import.meta.url. +import pyodideMjsUrl from 'pyodide/pyodide.mjs?url'; -// Start loading immediately so it is ready when the first message arrives. -const pyodideReadyPromise = initPyodide(); +const pyodideReadyPromise = (async () => { + const { loadPyodide } = await import(/* @vite-ignore */ pyodideMjsUrl); + return loadPyodide(); +})(); self.onmessage = async (event) => { // Propagate init failures back to the main thread rather than hanging silently. diff --git a/vite.config.ts b/vite.config.ts index fc29693..4303eff 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -80,7 +80,7 @@ export default defineConfig({ exclude: ['pyodide'], }, worker: { - // ES module workers are required for `import { loadPyodide } from "pyodide"`. + // ES module workers are required for the CDN import in webworker.js. format: 'es', }, build: { -- 2.51.2 From 03bc068f0681e3402c9cb23360b13f65c2c6d4bb Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 15 Mar 2026 15:00:16 -0400 Subject: [PATCH 08/12] nearly there --- .llms/learnings.md | 28 +++++++++ internals/scripts/InstallMNE.mjs | 2 +- src/main/index.ts | 59 +++++++++++++++++++ src/renderer/index.html | 2 +- src/renderer/utils/webworker/index.ts | 2 +- src/renderer/utils/webworker/webworker.js | 70 +++++++++++++++++++---- vite.config.ts | 40 +++++++++++++ 7 files changed, 189 insertions(+), 14 deletions(-) diff --git a/.llms/learnings.md b/.llms/learnings.md index 30deab0..821a23a 100644 --- a/.llms/learnings.md +++ b/.llms/learnings.md @@ -21,6 +21,34 @@ The app uses shadcn/ui + Tailwind CSS. CSS modules have been fully removed. Key - **Background gradient** used on all main screens: `bg-gradient-to-b from-[#f9f9f9] to-[#f0f0ff]` - **`@radix-ui/react-select`** is installed for the shadcn Select component +## Pyodide Asset Serving — Vite SPA Fallback Problem + +Vite's `historyApiFallback` returns `index.html` for **all** `fetch()` requests from web workers, including `/@fs/` and `publicDir` paths. This breaks Pyodide's package loading entirely. + +**Solution (two-part):** +1. A custom Vite middleware in `vite.config.ts` intercepts `/pyodide/` and `/packages/` requests before the SPA fallback and serves them directly from `src/renderer/utils/webworker/src/`. +2. An Electron `http` server on **port 17173** (started in `src/main/index.ts`) serves the same directory. Web workers use `http://127.0.0.1:17173` as `PYODIDE_ASSET_BASE`. This is the authoritative path — web worker `fetch()` calls bypass Vite entirely. + +Port 17173 is hardcoded in both `src/main/index.ts` and `src/renderer/utils/webworker/webworker.js` and in the CSP (`src/renderer/index.html`). + +**Other Pyodide loading gotchas:** +- `pyodide.mjs` must be loaded via dynamic `import()` (not `fetch()`), using a `?url` Vite import — `import()` bypasses the SPA fallback, `fetch()` does not +- The lock file is embedded via `?raw` and wrapped in a `Blob` + `createObjectURL` to avoid an HTTP fetch +- Use `packageBaseUrl` (not `indexURL`) to tell Pyodide where to find `.whl` files; `indexURL` is for WASM/stdlib +- `checkIntegrity: false` is required — SHA256 hashes in the npm lock file don't match CDN-downloaded wheels +- Workers must be created with `type: 'module'` (Pyodide 0.26+ ships `pyodide.mjs` as ESM) +- `optimizeDeps.exclude: ['pyodide']` in `vite.config.ts` prevents Vite from pre-bundling it + +## Pyodide Offline Package Installation (InstallMNE.mjs) + +`internals/scripts/InstallMNE.mjs` runs on `postinstall` and downloads two sets of packages: +- **Pyodide binary packages** (numpy, scipy, matplotlib, pandas + transitive deps) from the Pyodide CDN → `src/renderer/utils/webworker/src/pyodide/` +- **Pure-Python packages** (mne, pooch, tqdm, platformdirs) from PyPI → `src/renderer/utils/webworker/src/packages/` + +A `manifest.json` is written to `packages/` so `webworker.js` knows the exact `.whl` filenames to pass to `micropip.install()`. + +The CDN version is derived from `node_modules/pyodide/package.json` — **not** from `pyodide-lock.json`'s `info.version`, which may be a dev label like `0.28.0.dev0`. + ## Pre-existing TypeScript errors (do not treat as regressions) - `src/renderer/epics/experimentEpics.ts` (lines 170, 205) — RxJS operator type mismatch diff --git a/internals/scripts/InstallMNE.mjs b/internals/scripts/InstallMNE.mjs index 37e9870..dd87fc7 100644 --- a/internals/scripts/InstallMNE.mjs +++ b/internals/scripts/InstallMNE.mjs @@ -39,7 +39,7 @@ const MANIFEST_FILE = path.join(PACKAGES_DIR, 'manifest.json'); // Root packages whose full transitive dependency tree we need from Pyodide CDN // --------------------------------------------------------------------------- -const PYODIDE_ROOT_PACKAGES = ['numpy', 'scipy', 'matplotlib', 'pandas']; +const PYODIDE_ROOT_PACKAGES = ['numpy', 'scipy', 'matplotlib', 'pandas', 'micropip']; // --------------------------------------------------------------------------- // Pure-Python packages to download from PyPI (not bundled with Pyodide) diff --git a/src/main/index.ts b/src/main/index.ts index 648f964..6320e05 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -8,6 +8,7 @@ import { app, BrowserWindow, ipcMain, dialog, shell, session } from 'electron'; import path from 'path'; import fs from 'fs'; +import http from 'http'; import os from 'os'; import Papa from 'papaparse'; import mkdirp from 'mkdirp'; @@ -23,6 +24,52 @@ app.commandLine.appendSwitch( 'true' ); +// Port for the local pyodide asset server (serves whl files to web workers, +// bypassing Vite's dev server which returns HTML for all fetch() requests). +const PYODIDE_ASSET_PORT = 17173; + +const PYODIDE_CONTENT_TYPES: Record = { + '.json': 'application/json', + '.whl': 'application/zip', + '.zip': 'application/zip', + '.wasm': 'application/wasm', + '.js': 'application/javascript', + '.mjs': 'application/javascript', +}; + +function startPyodideAssetServer(rootDir: string): void { + const server = http.createServer((req, res) => { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS'); + if (req.method === 'OPTIONS') { res.writeHead(204); res.end(); return; } + + const urlPath = (req.url || '').split('?')[0]; + const filePath = path.join(rootDir, urlPath); + const ext = path.extname(filePath).toLowerCase(); + + fs.stat(filePath, (statErr, stat) => { + if (statErr || !stat.isFile()) { + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end(`Not found: ${urlPath}`); + return; + } + res.setHeader('Content-Type', PYODIDE_CONTENT_TYPES[ext] || 'application/octet-stream'); + res.setHeader('Content-Length', stat.size); + res.setHeader('Cache-Control', 'no-cache'); + res.writeHead(200); + fs.createReadStream(filePath).pipe(res); + }); + }); + + server.listen(PYODIDE_ASSET_PORT, '127.0.0.1', () => { + console.log(`[main] Pyodide asset server: http://127.0.0.1:${PYODIDE_ASSET_PORT}`); + }); + + server.on('error', (err: NodeJS.ErrnoException) => { + console.error('[main] Pyodide asset server error:', err.message); + }); +} + export default class AppUpdater { constructor() { log.transports.file.level = 'info'; @@ -453,6 +500,18 @@ app.on('window-all-closed', () => { }); app.whenReady().then(async () => { + // Serve pyodide assets (whl files, runtime files) via a local HTTP server so + // web workers can fetch() them without hitting Vite's SPA fallback, which + // returns HTML for ALL fetch() requests regardless of path. + // Port 17173 is hardcoded and matched in webworker.js. + // In dev: files are in src/renderer/utils/webworker/src/ + // In prod: files are in resources/webworker/src/ (via extraResources) + const pyodideRoot = is.dev + ? path.join(app.getAppPath(), 'src/renderer/utils/webworker/src') + : path.join(process.resourcesPath, 'webworker/src'); + + startPyodideAssetServer(pyodideRoot); + // Enable F12 devtools shortcut and Ctrl+R reload in dev, disable in prod app.on('browser-window-created', (_, window) => { optimizer.watchWindowShortcuts(window); diff --git a/src/renderer/index.html b/src/renderer/index.html index ff70f86..3027434 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -4,7 +4,7 @@ BrainWaves diff --git a/src/renderer/utils/webworker/index.ts b/src/renderer/utils/webworker/index.ts index c8e965b..27f9825 100644 --- a/src/renderer/utils/webworker/index.ts +++ b/src/renderer/utils/webworker/index.ts @@ -127,7 +127,7 @@ export const plotTestPlot = async (worker: Worker | null) => { } return worker.postMessage({ // data: `import matplotlib.pyplot as plt; fig= plt.plot([1,2,3,4])`, - data: `[1,2,3,4]` + data: `sum([1,2,3,4])` }); }; diff --git a/src/renderer/utils/webworker/webworker.js b/src/renderer/utils/webworker/webworker.js index 6ca003c..f309d7e 100644 --- a/src/renderer/utils/webworker/webworker.js +++ b/src/renderer/utils/webworker/webworker.js @@ -3,24 +3,72 @@ * * Loading strategy * ---------------- - * Use Vite's `?url` suffix on 'pyodide/pyodide.mjs' to get the resolved file URL - * at build/dev time (/@fs/... in dev, an asset URL in prod), then dynamically - * import from that URL. This bypasses Vite's SPA fallback and lets pyodide.mjs - * resolve all sibling assets (pyodide.asm.wasm, pyodide-lock.json, etc.) via - * import.meta.url — no CDN required. + * pyodide.mjs is imported via Vite's `?url` suffix, which gives us an + * /@fs/... URL in dev. We use dynamic import() from that URL — this works + * because import() bypasses Vite's SPA fallback (only fetch() is affected). * - * Production builds use the files copied to publicDir by InstallPyodide.mjs. + * The lock file is embedded via `?raw` to avoid an HTTP fetch that Vite + * intercepts. A blob URL is created from the embedded JSON so loadPyodide + * can "fetch" it from memory. + * + * Package whl files (numpy, scipy, etc.) live in + * src/renderer/utils/webworker/src/pyodide/ and are served by a tiny Node.js + * HTTP server on port 17173 started in the Electron main process. This bypasses + * Vite's dev server, which returns HTML (SPA fallback) for ALL fetch() requests + * from web workers, including /@fs/ and publicDir paths. + * + * MNE and its pure-Python deps are installed via micropip from local .whl + * files served by the same pyodide-asset:// protocol under /packages/. */ -// ?url tells Vite to resolve the path and return a URL string rather than bundling -// the module. In dev mode this is a /@fs/ URL (bypasses SPA fallback); in prod it -// is an asset URL. We then dynamically import from that URL so pyodide.mjs can -// resolve all its sibling assets (pyodide.asm.wasm, etc.) via import.meta.url. +// ?url → Vite resolves to /@fs/... in dev; asset URL in prod. +// ?raw → Vite embeds file content as a string (no HTTP fetch at runtime). import pyodideMjsUrl from 'pyodide/pyodide.mjs?url'; +import lockFileRaw from 'pyodide/pyodide-lock.json?raw'; + +// A tiny Node.js HTTP server on port 17173 (started in the Electron main +// process) serves pyodide assets from src/renderer/utils/webworker/src/. +// This bypasses Vite's dev server, which returns index.html (SPA fallback) +// for ALL fetch() requests from web workers, including /@fs/ and publicDir paths. +const PYODIDE_ASSET_BASE = 'http://127.0.0.1:17173'; const pyodideReadyPromise = (async () => { const { loadPyodide } = await import(/* @vite-ignore */ pyodideMjsUrl); - return loadPyodide(); + + // Wrap the embedded lock file in a blob URL so loadPyodide can "fetch" it + // without making an HTTP request that Vite would intercept and transform. + const lockBlob = new Blob([lockFileRaw], { type: 'application/json' }); + const lockFileURL = URL.createObjectURL(lockBlob); + + // packageBaseUrl tells pyodide's PackageManager where to fetch .whl files. + // This is the correct option — NOT indexURL, which is for the runtime files + // (WASM, stdlib) that are already loaded via import.meta.url from node_modules. + const packageBaseUrl = `${PYODIDE_ASSET_BASE}/pyodide/`; + + const pyodide = await loadPyodide({ lockFileURL, packageBaseUrl }); + URL.revokeObjectURL(lockFileURL); + + // Load scientific packages from local whl files via the asset server. + // checkIntegrity: false skips SHA256 verification — hashes in the npm lock + // file may not match the CDN-downloaded whl files we're actually serving. + await pyodide.loadPackage( + ['numpy', 'scipy', 'matplotlib', 'pandas', 'pillow'], + { checkIntegrity: false } + ); + + // Load micropip so we can install MNE and its pure-Python deps. + await pyodide.loadPackage('micropip', { checkIntegrity: false }); + const micropip = pyodide.pyimport('micropip'); + + // MNE + pure-Python deps are served from /packages/ via pyodide-asset://. + const manifestUrl = `${PYODIDE_ASSET_BASE}/packages/manifest.json`; + const manifest = await fetch(manifestUrl).then((r) => r.json()); + const whlUrls = Object.values(manifest).map( + ({ filename }) => `${PYODIDE_ASSET_BASE}/packages/${filename}` + ); + await micropip.install(whlUrls); + + return pyodide; })(); self.onmessage = async (event) => { diff --git a/vite.config.ts b/vite.config.ts index 4303eff..24f01c7 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,6 +1,7 @@ import { defineConfig } from 'electron-vite'; import react from '@vitejs/plugin-react'; import path from 'path'; +import fs from 'node:fs'; import { createRequire } from 'module'; const _require = createRequire(import.meta.url); @@ -48,6 +49,45 @@ export default defineConfig({ // /pyodide/pyodide.mjs, /pyodide/pyodide.asm.js, /packages/*.whl, etc. publicDir: path.resolve(__dirname, 'src/renderer/utils/webworker/src'), plugins: [ + // Serve pyodide runtime and package .whl files directly from the filesystem + // before Vite's SPA fallback can intercept them. publicDir alone is not + // reliable — Vite's historyApiFallback returns index.html for fetch() + // requests to these paths in dev mode. + { + name: 'serve-pyodide-assets', + configureServer(server) { + const staticDir = path.resolve( + __dirname, + 'src/renderer/utils/webworker/src' + ); + const contentTypes: Record = { + '.json': 'application/json', + '.whl': 'application/zip', + '.zip': 'application/zip', + '.wasm': 'application/wasm', + '.js': 'application/javascript', + '.mjs': 'application/javascript', + }; + server.middlewares.use((req, res, next) => { + const url = req.url ?? ''; + if (url.startsWith('/pyodide/') || url.startsWith('/packages/')) { + console.log('[serve-pyodide-assets] intercepted:', url); + const filePath = path.join(staticDir, url.split('?')[0]); + if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) { + const ext = path.extname(filePath).toLowerCase(); + res.setHeader( + 'Content-Type', + contentTypes[ext] ?? 'application/octet-stream' + ); + res.setHeader('Cache-Control', 'no-cache'); + fs.createReadStream(filePath).pipe(res); + return; + } + } + next(); + }); + }, + }, react({ jsxRuntime: 'classic', // React 16 does not ship react/jsx-runtime babel: { -- 2.51.2 From b333f3a4b7e1c4f7e07a1cbf7b2a24969a5b4e2f Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 15 Mar 2026 15:24:53 -0400 Subject: [PATCH 09/12] packages are loading! --- .llms/learnings.md | 7 + docs/pyodide-in-electron-vite.md | 176 ++++++++++++++++++++++++ internals/scripts/InstallMNE.mjs | 4 +- src/renderer/utils/webworker/patches.py | 29 ---- 4 files changed, 185 insertions(+), 31 deletions(-) create mode 100644 docs/pyodide-in-electron-vite.md diff --git a/.llms/learnings.md b/.llms/learnings.md index 821a23a..bcf14e3 100644 --- a/.llms/learnings.md +++ b/.llms/learnings.md @@ -49,6 +49,13 @@ A `manifest.json` is written to `packages/` so `webworker.js` knows the exact `. The CDN version is derived from `node_modules/pyodide/package.json` — **not** from `pyodide-lock.json`'s `info.version`, which may be a dev label like `0.28.0.dev0`. +**Packages that must be listed explicitly** (not reachable from matplotlib/scipy/pandas deps in the lock file, but required at runtime): +- `jinja2` + `markupsafe` — used by matplotlib templates and MNE HTML reports +- `decorator` — MNE core dep +- `requests` (+ `certifi`, `charset-normalizer`, `idna`, `urllib3`) — pulled in by `pooch` at MNE import time + +**`micropip.install()` from JS accepts a JS array directly** — as of Pyodide 0.29.x, micropip handles the `JsProxy` conversion internally. `pyodide.toPy()` is not needed. + ## Pre-existing TypeScript errors (do not treat as regressions) - `src/renderer/epics/experimentEpics.ts` (lines 170, 205) — RxJS operator type mismatch diff --git a/docs/pyodide-in-electron-vite.md b/docs/pyodide-in-electron-vite.md new file mode 100644 index 0000000..34c5934 --- /dev/null +++ b/docs/pyodide-in-electron-vite.md @@ -0,0 +1,176 @@ +# Pyodide in Electron + Vite: What We Learned + +This document captures everything we learned getting Pyodide (Python-in-WASM) running reliably inside an Electron + electron-vite app. It is intended as a reference for anyone maintaining or upgrading the Pyodide integration. + +--- + +## The Core Problem: Vite's SPA Fallback + +Vite's dev server runs a `historyApiFallback` middleware that returns `index.html` for **every** `fetch()` request it doesn't recognise — including `/@fs/` paths, `publicDir` paths, and anything from a web worker. This completely breaks Pyodide's package loader, which `fetch()`es `.whl` files at runtime. + +This is not an obvious failure — Pyodide may partially initialise and then hang or throw cryptic errors when it tries to load packages. + +### Solution: Serve Pyodide Assets Out-of-Band + +We use two complementary mechanisms: + +**1. Custom Vite middleware (dev only)** + +In `vite.config.ts`, a plugin intercepts requests to `/pyodide/` and `/packages/` before the SPA fallback runs and streams the files directly from `src/renderer/utils/webworker/src/`: + +```ts +server.middlewares.use((req, res, next) => { + const url = req.url ?? ''; + if (url.startsWith('/pyodide/') || url.startsWith('/packages/')) { + const filePath = path.join(staticDir, url.split('?')[0]); + if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) { + res.setHeader('Content-Type', contentTypes[ext] ?? 'application/octet-stream'); + fs.createReadStream(filePath).pipe(res); + return; + } + } + next(); +}); +``` + +**2. Electron local HTTP server on port 17173 (dev + prod)** + +Web workers cannot use Vite's dev server at all — `fetch()` from a worker always hits the SPA fallback. The main process (`src/main/index.ts`) starts a plain Node.js `http` server at `http://127.0.0.1:17173` that serves `src/renderer/utils/webworker/src/` (dev) or `resources/webworker/src/` (prod). + +The web worker (`webworker.js`) uses this as its `PYODIDE_ASSET_BASE`: + +```js +const PYODIDE_ASSET_BASE = 'http://127.0.0.1:17173'; +``` + +Port 17173 is hardcoded in three places that must stay in sync: +- `src/main/index.ts` — server listen port +- `src/renderer/utils/webworker/webworker.js` — `PYODIDE_ASSET_BASE` +- `src/renderer/index.html` — CSP `connect-src` directive + +--- + +## Loading pyodide.mjs + +Pyodide 0.26+ ships as an ES module (`pyodide.mjs`). You cannot `fetch()` it — Vite would intercept it. Instead, import it with Vite's `?url` suffix and use dynamic `import()`: + +```js +import pyodideMjsUrl from 'pyodide/pyodide.mjs?url'; +// ... +const { loadPyodide } = await import(/* @vite-ignore */ pyodideMjsUrl); +``` + +`import()` bypasses Vite's SPA fallback; `fetch()` does not. + +Also required in `vite.config.ts`: + +```ts +optimizeDeps: { + exclude: ['pyodide'], // prevent Vite from pre-bundling it +}, +worker: { + format: 'es', // ES module workers required for pyodide.mjs +}, +``` + +And workers must be created with `type: 'module'`: + +```ts +new Worker(new URL('./webworker.js', import.meta.url), { type: 'module' }); +``` + +--- + +## Loading the Lock File Without a Fetch + +`loadPyodide` needs the lock file to resolve package names to filenames. Fetching it would hit Vite's SPA fallback. Instead, embed it at build time with Vite's `?raw` suffix and wrap it in a blob URL: + +```js +import lockFileRaw from 'pyodide/pyodide-lock.json?raw'; + +const lockBlob = new Blob([lockFileRaw], { type: 'application/json' }); +const lockFileURL = URL.createObjectURL(lockBlob); +const pyodide = await loadPyodide({ lockFileURL, packageBaseUrl }); +URL.revokeObjectURL(lockFileURL); +``` + +--- + +## packageBaseUrl vs indexURL + +These are easy to confuse: + +| Option | Purpose | +|--------|---------| +| `indexURL` | Where Pyodide looks for its **runtime** files (WASM, stdlib). Already resolved from `node_modules` via `import.meta.url`. Do not override. | +| `packageBaseUrl` | Where `loadPackage()` fetches **package `.whl` files**. Set this to `http://127.0.0.1:17173/pyodide/`. | + +--- + +## Package Integrity Checks + +```js +await pyodide.loadPackage(['numpy', 'scipy', ...], { checkIntegrity: false }); +``` + +`checkIntegrity: false` is required. The SHA-256 hashes in the npm package's `pyodide-lock.json` are computed against the CDN files, but we serve locally-downloaded copies that may differ (e.g. re-compressed). Integrity checks will fail without this flag. + +--- + +## micropip.install() from JavaScript + +`micropip` is a Python object loaded via `pyodide.pyimport()`. Passing a JavaScript array directly works fine in Pyodide 0.29.x — micropip handles the `JsProxy` conversion internally: + +```js +const micropip = pyodide.pyimport('micropip'); +await micropip.install(whlUrls); // JS array works directly +``` + +> Note: older guidance suggested wrapping with `pyodide.toPy(whlUrls)` — this is not necessary as of 0.29.x. + +--- + +## Offline Package Installation (InstallMNE.mjs) + +`internals/scripts/InstallMNE.mjs` runs on `postinstall` and pre-downloads all packages so the app works offline. + +### Part 1 — Pyodide Binary Packages (from Pyodide CDN) + +These are compiled packages bundled with Pyodide. The script reads `pyodide-lock.json`, recursively resolves transitive dependencies of the root packages, and downloads each `.whl` into `src/renderer/utils/webworker/src/pyodide/`. + +**Derive the CDN version from `node_modules/pyodide/package.json`**, not from `pyodide-lock.json`'s `info.version` field — that field may be a dev label like `0.28.0.dev0` and will produce a broken CDN URL. + +Current root packages and why each is listed explicitly: + +| Package | Reason | +|---------|--------| +| `numpy`, `scipy`, `matplotlib`, `pandas` | Core scientific stack | +| `micropip` | Needed to install pure-Python packages at worker startup | +| `pillow` | Used by matplotlib and MNE; loaded at runtime by `loadPackage()` | +| `jinja2` | MNE dep; **not** listed in matplotlib's lock-file `depends` array despite being a runtime requirement | +| `decorator` | MNE core dep; not reachable from the scientific stack in the lock file | +| `requests` | Pulled in by `pooch` at MNE import time; brings in `certifi`, `charset-normalizer`, `idna`, `urllib3` transitively | + +> **Gotcha:** The `depends` arrays in `pyodide-lock.json` are incomplete. Several packages that matplotlib, scipy, or MNE require at runtime are not listed as dependencies and will not be downloaded unless added explicitly as roots. + +### Part 2 — Pure-Python Packages (from PyPI) + +Packages not bundled with Pyodide must be downloaded as `py3-none-any` wheels from PyPI. They are stored in `src/renderer/utils/webworker/src/packages/` and a `manifest.json` is written so the worker knows the exact filenames. + +Current PyPI packages: `mne`, `pooch`, `tqdm`, `platformdirs`, `lazy-loader` + +`lazy-loader` is a core MNE dependency that does not appear in the Pyodide lock at all. + +--- + +## Summary of File Locations + +| What | Where | +|------|-------| +| Pyodide runtime + binary wheels | `src/renderer/utils/webworker/src/pyodide/` | +| Pure-Python wheels + manifest | `src/renderer/utils/webworker/src/packages/` | +| Web worker entry point | `src/renderer/utils/webworker/webworker.js` | +| JS wrappers for Python calls | `src/renderer/utils/webworker/index.ts` | +| Install script | `internals/scripts/InstallMNE.mjs` | +| Electron asset server | `src/main/index.ts` → `startPyodideAssetServer()` | +| Vite middleware | `vite.config.ts` → `serve-pyodide-assets` plugin | diff --git a/internals/scripts/InstallMNE.mjs b/internals/scripts/InstallMNE.mjs index dd87fc7..abb9eb8 100644 --- a/internals/scripts/InstallMNE.mjs +++ b/internals/scripts/InstallMNE.mjs @@ -39,13 +39,13 @@ const MANIFEST_FILE = path.join(PACKAGES_DIR, 'manifest.json'); // Root packages whose full transitive dependency tree we need from Pyodide CDN // --------------------------------------------------------------------------- -const PYODIDE_ROOT_PACKAGES = ['numpy', 'scipy', 'matplotlib', 'pandas', 'micropip']; +const PYODIDE_ROOT_PACKAGES = ['numpy', 'scipy', 'matplotlib', 'pandas', 'micropip', 'pillow', 'jinja2', 'decorator', 'requests']; // --------------------------------------------------------------------------- // Pure-Python packages to download from PyPI (not bundled with Pyodide) // --------------------------------------------------------------------------- -const PYPI_PACKAGES = ['mne', 'pooch', 'tqdm', 'platformdirs']; +const PYPI_PACKAGES = ['mne', 'pooch', 'tqdm', 'platformdirs', 'lazy-loader']; // --------------------------------------------------------------------------- // Shared network helpers diff --git a/src/renderer/utils/webworker/patches.py b/src/renderer/utils/webworker/patches.py index 43b5781..58041fb 100644 --- a/src/renderer/utils/webworker/patches.py +++ b/src/renderer/utils/webworker/patches.py @@ -1,31 +1,3 @@ -# patch implemented in Pyolite -# https://github.com/jupyterlite/jupyterlite/blob/0d563b9a4cca4b54411229128cb51ac4ba333c8f/packages/pyolite-kernel/py/pyolite/pyolite/patches.py -def patch_matplotlib(): - import os - from io import BytesIO - - # before importing matplotlib - # to avoid the wasm backend (which needs `js.document`, not available in worker) - os.environ["MPLBACKEND"] = "AGG" - - import matplotlib.pyplot - from IPython.display import display - - from .display import Image - - _old_show = matplotlib.pyplot.show - assert _old_show, "matplotlib.pyplot.show" - - def show(): - buf = BytesIO() - matplotlib.pyplot.savefig(buf, format="png") - buf.seek(0) - display(Image(buf.read())) - matplotlib.pyplot.clf() - - matplotlib.pyplot.show = show - - def patch_pillow(): import base64 @@ -43,7 +15,6 @@ def patch_pillow(): ALL_PATCHES = [ patch_pillow, - patch_matplotlib, ] -- 2.51.2 From 5306283f5f98f3688c350d29e8f3f2e6e97c821d Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 15 Mar 2026 15:41:05 -0400 Subject: [PATCH 10/12] update the test plot functionality --- .llms/CLAUDE.md | 8 ++-- src/renderer/epics/pyodideEpics.ts | 54 +++++++++------------ src/renderer/utils/webworker/index.ts | 57 ++++++++++++++++++----- src/renderer/utils/webworker/patches.py | 2 +- src/renderer/utils/webworker/webworker.js | 13 ++++-- 5 files changed, 83 insertions(+), 51 deletions(-) diff --git a/.llms/CLAUDE.md b/.llms/CLAUDE.md index 853b9ac..edd0bce 100644 --- a/.llms/CLAUDE.md +++ b/.llms/CLAUDE.md @@ -15,8 +15,8 @@ A priority for this codebase is extensibility modularity and hackability. There - **Bundler**: electron-vite / Vite - **State**: Redux Toolkit + redux-observable (RxJS epics) - **Language**: TypeScript (strict) -- **Styling**: Semantic UI React + SCSS -- **Testing**: Jest +- **Styling**: Tailwind CSS + shadcn/ui (components in `src/renderer/components/ui/`) +- **Testing**: Vitest - **Linting**: ESLint + Prettier (single quotes, ES5 trailing commas) ## Key Directories @@ -30,7 +30,7 @@ A priority for this codebase is extensibility modularity and hackability. There ```bash npm run dev # Start dev server (patches deps first) npm run build # Build all processes -npm test # Run Jest tests +npm test # Run Vitest tests npm run typecheck # TypeScript check (no emit) npm run lint # ESLint npm run lint-fix # ESLint + Prettier auto-fix @@ -45,7 +45,7 @@ npm run package # Build + package for current platform - Keep Electron main/renderer separation strict — use preload IPC bridges ## Out of Scope -- Do not modify `src/renderer/utils/webworker/src/` directly; it is managed by `InstallPyodide.js` +- Do not modify `src/renderer/utils/webworker/src/` directly; it is managed by `internals/scripts/InstallPyodide.mjs` (Pyodide runtime) and `internals/scripts/InstallMNE.mjs` (scientific packages) - Do not alter `electron-builder` publish config without confirming release intent ## LLM Context diff --git a/src/renderer/epics/pyodideEpics.ts b/src/renderer/epics/pyodideEpics.ts index dfc40b0..874bcb5 100644 --- a/src/renderer/epics/pyodideEpics.ts +++ b/src/renderer/epics/pyodideEpics.ts @@ -1,5 +1,5 @@ import { combineEpics, Epic } from 'redux-observable'; -import { fromEvent, Observable, ObservableInput, of } from 'rxjs'; +import { EMPTY, fromEvent, Observable, ObservableInput, of } from 'rxjs'; import { map, mergeMap, tap, pluck, filter } from 'rxjs/operators'; import { toast } from 'react-toastify'; import { isActionOf } from '../utils/redux'; @@ -87,21 +87,24 @@ const pyodideMessageEpic: Epic< filter(isActionOf(PyodideActions.SetPyodideWorker)), pluck('payload'), // eslint-disable-next-line @typescript-eslint/no-explicit-any - mergeMap>((worker) => { - // Worker message event — MessageEvent data shape is dynamic - return fromEvent(worker, 'message'); - }), - tap((e) => { - console.log(e); - const { results, error } = e.data; - - if (results && !error) { - toast.error(`Pyodide: ${results}`); - } else if (error) { + mergeMap>((worker) => fromEvent(worker, 'message')), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + mergeMap>((e) => { + const { results, error, plotKey } = e.data; + if (error) { toast.error(`Pyodide: ${error}`); + return of(PyodideActions.ReceiveError(error)); } - }), - map(PyodideActions.ReceiveMessage) + // Route plot results to the appropriate Redux state slot. + // results is a base64-encoded PNG string returned from Python. + const mimeBundle = results ? { 'image/png': results } : null; + switch (plotKey) { + case 'topo': return of(PyodideActions.SetTopoPlot(mimeBundle)); + case 'psd': return of(PyodideActions.SetPSDPlot(mimeBundle)); + case 'erp': return of(PyodideActions.SetERPPlot(mimeBundle)); + default: return of(PyodideActions.ReceiveMessage(e.data)); + } + }) ); const loadEpochsEpic: Epic = ( @@ -225,8 +228,8 @@ const loadPSDEpic: Epic = ( ) => action$.pipe( filter(isActionOf(PyodideActions.LoadPSD)), - mergeMap(() => plotPSD(state$.value.pyodide.worker!)), - map(PyodideActions.SetPSDPlot) + tap(() => plotPSD(state$.value.pyodide.worker!)), + mergeMap(() => EMPTY) ); const loadTopoEpic: Epic = ( @@ -235,19 +238,8 @@ const loadTopoEpic: Epic = ( ) => action$.pipe( filter(isActionOf(PyodideActions.LoadTopo)), - // mergeMap(plotTopoMap), - mergeMap(() => plotTestPlot(state$.value.pyodide.worker!)), - tap((e) => console.log('received topo map: ', e)), - mergeMap((topoPlot) => - of( - PyodideActions.SetTopoPlot(topoPlot) - // PyodideActions.LoadERP( - // state$.value.device.deviceType === DEVICES.EMOTIV - // ? EMOTIV_CHANNELS[0] - // : MUSE_CHANNELS[0] - // ) - ) - ) + tap(() => plotTestPlot(state$.value.pyodide.worker!)), + mergeMap(() => EMPTY) ); const loadERPEpic: Epic = ( @@ -273,8 +265,8 @@ const loadERPEpic: Epic = ( ); return parseInt(EMOTIV_CHANNELS[0], 10); }), - mergeMap((chanIndex) => plotERP(state$.value.pyodide.worker!, chanIndex)), - map(PyodideActions.SetERPPlot) + tap((chanIndex) => plotERP(state$.value.pyodide.worker!, chanIndex)), + mergeMap(() => EMPTY) ); export default combineEpics( diff --git a/src/renderer/utils/webworker/index.ts b/src/renderer/utils/webworker/index.ts index 27f9825..e97a44c 100644 --- a/src/renderer/utils/webworker/index.ts +++ b/src/renderer/utils/webworker/index.ts @@ -112,29 +112,62 @@ export const cleanEpochsPlot = async (worker: Worker) => { }; export const plotPSD = async (worker: Worker) => { - return worker.postMessage({ data: `raw.plot_psd(fmin=1, fmax=30)` }); + worker.postMessage({ + plotKey: 'psd', + data: [ + 'import io, base64', + '_fig = raw.plot_psd(fmin=1, fmax=30, show=False)', + '_buf = io.BytesIO()', + '_fig.savefig(_buf, format="png", bbox_inches="tight")', + 'plt.close(_fig)', + 'base64.b64encode(_buf.getvalue()).decode()', + ].join('\n'), + }); }; export const plotTopoMap = async (worker: Worker) => { - return worker.postMessage({ - data: `plot_topo(clean_epochs, conditions)`, + worker.postMessage({ + plotKey: 'topo', + data: [ + 'import io, base64', + '_fig = plot_topo(clean_epochs, conditions)', + '_buf = io.BytesIO()', + '_fig.savefig(_buf, format="png", bbox_inches="tight")', + 'plt.close(_fig)', + 'base64.b64encode(_buf.getvalue()).decode()', + ].join('\n'), }); }; export const plotTestPlot = async (worker: Worker | null) => { - if (!worker) { - return; - } - return worker.postMessage({ - // data: `import matplotlib.pyplot as plt; fig= plt.plot([1,2,3,4])`, - data: `sum([1,2,3,4])` + if (!worker) return; + worker.postMessage({ + plotKey: 'topo', + data: [ + 'import io, base64', + 'import matplotlib.pyplot as plt', + '_fig, _ax = plt.subplots()', + '_ax.plot([1, 2, 3, 4], [1, 4, 2, 3])', + '_ax.set_title("Test Plot")', + '_buf = io.BytesIO()', + '_fig.savefig(_buf, format="png", bbox_inches="tight")', + 'plt.close(_fig)', + 'base64.b64encode(_buf.getvalue()).decode()', + ].join('\n'), }); }; export const plotERP = async (worker: Worker, channelIndex: number) => { - return worker.postMessage({ - data: `X, y = plot_conditions(clean_epochs, ch_ind=${channelIndex}, conditions=conditions, - ci=97.5, n_boot=1000, title='', diff_waveform=None)`, + worker.postMessage({ + plotKey: 'erp', + data: [ + 'import io, base64', + `_fig, _ = plot_conditions(clean_epochs, ch_ind=${channelIndex}, conditions=conditions, ci=97.5, n_boot=1000, title='', diff_waveform=None)`, + '_buf = io.BytesIO()', + '_fig.savefig(_buf, format="png", bbox_inches="tight")', + 'plt.close(_fig)', + 'base64.b64encode(_buf.getvalue()).decode()', + ].join('\n'), }); }; diff --git a/src/renderer/utils/webworker/patches.py b/src/renderer/utils/webworker/patches.py index 58041fb..ca55ab1 100644 --- a/src/renderer/utils/webworker/patches.py +++ b/src/renderer/utils/webworker/patches.py @@ -25,4 +25,4 @@ def apply_patches(): try: patch() except Exception as err: - warnings.warn("faield to apply patch", patch, err) + warnings.warn("failed to apply patch", patch, err) diff --git a/src/renderer/utils/webworker/webworker.js b/src/renderer/utils/webworker/webworker.js index f309d7e..9cf665f 100644 --- a/src/renderer/utils/webworker/webworker.js +++ b/src/renderer/utils/webworker/webworker.js @@ -56,6 +56,13 @@ const pyodideReadyPromise = (async () => { { checkIntegrity: false } ); + // Set matplotlib backend before any imports so it takes effect on first import. + // Must be 'agg' (non-interactive, buffer-based) — web workers have no DOM, + // so WebAgg fails with "cannot import name 'document' from 'js'". + await pyodide.runPythonAsync( + 'import os; os.environ["MPLBACKEND"] = "agg"' + ); + // Load micropip so we can install MNE and its pure-Python deps. await pyodide.loadPackage('micropip', { checkIntegrity: false }); const micropip = pyodide.pyimport('micropip'); @@ -81,7 +88,7 @@ self.onmessage = async (event) => { return; } - const { data, ...context } = event.data; + const { data, plotKey, ...context } = event.data; // Expose context values as globals so Python can access them via the js module. for (const [key, value] of Object.entries(context)) { @@ -89,8 +96,8 @@ self.onmessage = async (event) => { } try { - self.postMessage({ results: await pyodide.runPythonAsync(data) }); + self.postMessage({ results: await pyodide.runPythonAsync(data), plotKey }); } catch (error) { - self.postMessage({ error: error.message }); + self.postMessage({ error: error.message, plotKey }); } }; -- 2.51.2 From 80d31d16acb37c58d0e3d5a1ec43ddbdbb0a07f3 Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 15 Mar 2026 15:53:14 -0400 Subject: [PATCH 11/12] add plot as an svg --- .llms/learnings.md | 4 + package-lock.json | 205 ------------------ package.json | 1 - src/main/index.ts | 2 +- src/renderer/components/PyodidePlotWidget.tsx | 81 ++----- src/renderer/epics/pyodideEpics.ts | 2 +- src/renderer/utils/webworker/index.ts | 24 +- 7 files changed, 34 insertions(+), 285 deletions(-) diff --git a/.llms/learnings.md b/.llms/learnings.md index bcf14e3..d93903e 100644 --- a/.llms/learnings.md +++ b/.llms/learnings.md @@ -56,6 +56,10 @@ The CDN version is derived from `node_modules/pyodide/package.json` — **not** **`micropip.install()` from JS accepts a JS array directly** — as of Pyodide 0.29.x, micropip handles the `JsProxy` conversion internally. `pyodide.toPy()` is not needed. +**WebAgg backend does not work in web workers** — WebAgg tries to access `js.document` to inject CSS/JS into the DOM on first import, which throws `ImportError: cannot import name 'document' from 'js'` in a worker context. Use `agg` instead. Set it via `os.environ["MPLBACKEND"] = "agg"` before any matplotlib import. `fig.savefig()` works with `agg` and is the correct way to get plot images back to the renderer. + +**Plot result routing pattern** — `worker.postMessage()` is fire-and-forget (returns `undefined`). Plot epics should use `tap()` to fire the worker message and `mergeMap(() => EMPTY)` to emit nothing. Results come back asynchronously on the worker `message` event. Add a `plotKey` field to each worker message; the worker echoes it back; `pyodideMessageEpic` switches on `plotKey` to dispatch `SetTopoPlot`/`SetPSDPlot`/`SetERPPlot` with a `{ 'image/png': base64string }` MIME bundle. `PyodidePlotWidget` renders this via `@nteract/transforms`. + ## Pre-existing TypeScript errors (do not treat as regressions) - `src/renderer/epics/experimentEpics.ts` (lines 170, 205) — RxJS operator type mismatch diff --git a/package-lock.json b/package-lock.json index e0a77cb..894cb7e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,6 @@ "@electron-toolkit/utils": "^4.0.0", "@fortawesome/fontawesome-free": "^5.13.0", "@neurosity/pipes": "^5.2.1", - "@nteract/transforms": "^3.2.0", "@radix-ui/react-dialog": "^1.1.0", "@radix-ui/react-dropdown-menu": "^2.1.0", "@radix-ui/react-select": "^2.2.6", @@ -597,18 +596,6 @@ "node": ">=6.9.0" } }, - "node_modules/@babel/runtime-corejs2": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.28.6.tgz", - "integrity": "sha512-pOHfxftxpetWUeBacCB3ZOPc/OO6hiT9MLv0qd9j474khiCcduwO8uuJI3N7vX3m8GJotTT6lxlA89TS/PylGg==", - "license": "MIT", - "dependencies": { - "core-js": "^2.6.12" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@babel/template": { "version": "7.28.6", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", @@ -2579,56 +2566,6 @@ "node": ">=10" } }, - "node_modules/@nteract/transform-vdom": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/@nteract/transform-vdom/-/transform-vdom-2.2.5.tgz", - "integrity": "sha512-q6FbWlrSEWUmQpDV1DBPcw5FZpUcQbKOQ2a59vY/qcQ/Qjh1KUCC+gortso+WIE4P36eHZRxKz5ptCu5i47OLg==", - "license": "BSD-3-Clause", - "dependencies": { - "@babel/runtime-corejs2": "^7.0.0", - "babel-runtime": "^6.26.0", - "lodash": "^4.17.4" - }, - "peerDependencies": { - "react": "^16.3.2" - } - }, - "node_modules/@nteract/transforms": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@nteract/transforms/-/transforms-3.2.0.tgz", - "integrity": "sha512-9P926e2tm0H1IHF2ER6f0+At5NPgrMgvNOPZTn+K6e9M9+EpNPbZq4q5YUX1xKG8YaK2fpiH+4XVkFBf06YOJg==", - "deprecated": "This package has been deprecated. Please access each transform through its own package.", - "license": "BSD-3-Clause", - "dependencies": { - "@nteract/transform-vdom": "^2.1.0", - "ansi-to-react": "^2.0.6", - "commonmark": "^0.28.0", - "commonmark-react-renderer": "^4.3.3", - "mathjax-electron": "^2.0.1", - "react-json-tree": "^0.11.0" - }, - "peerDependencies": { - "react": "^16.2.0" - } - }, - "node_modules/@nteract/transforms/node_modules/commonmark": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/commonmark/-/commonmark-0.28.1.tgz", - "integrity": "sha512-PklsZ9pgrfFQ5hQH9BRzoWnqI9db2LeR9MhvkNk8iz97kfaTNmhTU+IE8jKDHTEfivZZXoFqzGqzddXdk14EJw==", - "license": "BSD-2-Clause", - "dependencies": { - "entities": "~ 1.1.1", - "mdurl": "~ 1.0.1", - "minimist": "~ 1.2.0", - "string.prototype.repeat": "^0.2.0" - }, - "bin": { - "commonmark": "bin/commonmark" - }, - "engines": { - "node": "*" - } - }, "node_modules/@parcel/watcher": { "version": "2.5.6", "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.6.tgz", @@ -5281,12 +5218,6 @@ "integrity": "sha512-0V/PkoculFl5+0Lp47JoxUcO0xSxhIBvm+BxHdD/OgXNmdRpRHCFnKVuUoWyS9EzQP+otSGv0m9Lb4yVkQBn2A==", "license": "MIT" }, - "node_modules/anser": { - "version": "1.4.10", - "resolved": "https://registry.npmjs.org/anser/-/anser-1.4.10.tgz", - "integrity": "sha512-hCv9AqTQ8ycjpSd3upOJd7vFwW1JaoYQ7tpham03GJ1ca8/65rqn0RpaWpItOAd6ylW9wAw6luXYPJIyPFVOww==", - "license": "MIT" - }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -5314,20 +5245,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/ansi-to-react": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/ansi-to-react/-/ansi-to-react-2.0.6.tgz", - "integrity": "sha512-AnzmnQcMmCqbd72cRridR94RR0YQpv7Bvbm7YNSGnReTwFQmLkfaZzw4Ajg7HRfR6ZxCEa90sJWEZFhCOPcWhA==", - "license": "MPL-2.0", - "dependencies": { - "anser": "^1.4.1", - "escape-carriage": "^1.2.0" - }, - "peerDependencies": { - "react": "^16.2.0", - "react-dom": "^16.2.0" - } - }, "node_modules/any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", @@ -5996,12 +5913,6 @@ "dev": true, "license": "MIT" }, - "node_modules/base16": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/base16/-/base16-1.0.0.tgz", - "integrity": "sha1-4pf2DX7BAUp6lxo568ipjAtoHnA= sha512-pNdYkNPiJUnEhnfXV56+sQy8+AaPcG3POZAUnwr4EeqCUZFz4u2PePbo3e5Gj4ziYPCWGUZT9RHisvJKnwFuBQ==", - "license": "MIT" - }, "node_modules/base64-arraybuffer": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz", @@ -6865,22 +6776,6 @@ "node": ">=4.0.0" } }, - "node_modules/commonmark-react-renderer": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/commonmark-react-renderer/-/commonmark-react-renderer-4.3.5.tgz", - "integrity": "sha512-UwUgplz8kFSMCe9+Dg/BcV75lc7R/V6mvMYJq2p29i5aaIBd0252k9HeSGa2VtEPHfg2/trS9qC7iAxnO7r6ng==", - "license": "MIT", - "dependencies": { - "lodash.assign": "^4.2.0", - "lodash.isplainobject": "^4.0.6", - "pascalcase": "^0.1.1", - "xss-filters": "^1.2.6" - }, - "peerDependencies": { - "commonmark": "^0.27.0 || ^0.26.0 || ^0.24.0", - "react": ">=0.14.0" - } - }, "node_modules/compare-version": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/compare-version/-/compare-version-0.1.2.tgz", @@ -9186,12 +9081,6 @@ "once": "^1.4.0" } }, - "node_modules/entities": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", - "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", - "license": "BSD-2-Clause" - }, "node_modules/env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", @@ -9528,12 +9417,6 @@ "node": ">=6" } }, - "node_modules/escape-carriage": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/escape-carriage/-/escape-carriage-1.3.1.tgz", - "integrity": "sha512-GwBr6yViW3ttx1kb7/Oh+gKQ1/TrhYwxKqVmg5gS+BK+Qe2KrOa/Vh7w3HPBvgGf0LfcDGoY9I6NHKoA5Hozhw==", - "license": "MIT" - }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -12974,30 +12857,12 @@ "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", "license": "MIT" }, - "node_modules/lodash.assign": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz", - "integrity": "sha512-hFuH8TY+Yji7Eja3mGiuAxBqLagejScbG8GbG0j6o9vzn0YL14My+ktnqtZgFTosKymC9/44wP6s7xyuLfnClw==", - "license": "MIT" - }, - "node_modules/lodash.curry": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lodash.curry/-/lodash.curry-4.1.1.tgz", - "integrity": "sha1-JI42By7ekGUB11lmIAqG2riyMXA= sha512-/u14pXGviLaweY5JI0IUzgzF2J6Ne8INyzAZjImcryjgkZ+ebruBxy2/JaOOkTqScddcYtakjhSaeemV8lR0tA==", - "license": "MIT" - }, "node_modules/lodash.escaperegexp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", "license": "MIT" }, - "node_modules/lodash.flow": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/lodash.flow/-/lodash.flow-3.5.0.tgz", - "integrity": "sha1-h79AKSuM+D5OjOGjrkIJ4gBxZ1o= sha512-ff3BX/tSioo+XojX4MOsOMhJw0nZoUEF011LX8g8d3gvjVbxd89cCio4BCXronjxcTUIJUoqKEUA+n4CqvvRPw==", - "license": "MIT" - }, "node_modules/lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", @@ -13005,12 +12870,6 @@ "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", "license": "MIT" }, - "node_modules/lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", - "license": "MIT" - }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -13467,12 +13326,6 @@ "node": ">=0.10.0" } }, - "node_modules/mathjax-electron": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mathjax-electron/-/mathjax-electron-2.0.1.tgz", - "integrity": "sha512-bllJaZZUccbj1ReD9i0V6qwu27dZXbd7TG/Wy3M7F10NLEjl8yN0WgFFP9uYf35s0hkody6wSPO96txr68TOqg==", - "license": "MIT" - }, "node_modules/mathml-tag-names": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/mathml-tag-names/-/mathml-tag-names-4.0.0.tgz", @@ -13491,12 +13344,6 @@ "dev": true, "license": "CC0-1.0" }, - "node_modules/mdurl": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", - "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", - "license": "MIT" - }, "node_modules/meow": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/meow/-/meow-14.0.0.tgz", @@ -14621,15 +14468,6 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, - "node_modules/pascalcase": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", - "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -15305,12 +15143,6 @@ "node": ">=6" } }, - "node_modules/pure-color": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/pure-color/-/pure-color-1.3.0.tgz", - "integrity": "sha1-H+Bk+wrIUfDeYTIKi/eWg2Qi8z4= sha512-QFADYnsVoBMw1srW7OVKEYjG+MbIa49s54w1MA1EDY6r2r/sTcKKYqRX1f4GYvnXP7eN/Pe9HFcX+hwzmrXRHA==", - "license": "MIT" - }, "node_modules/pyodide": { "version": "0.29.3", "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-0.29.3.tgz", @@ -15545,18 +15377,6 @@ "node": ">=0.10.0" } }, - "node_modules/react-base16-styling": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/react-base16-styling/-/react-base16-styling-0.5.3.tgz", - "integrity": "sha1-OFjyTpxN2MvT9wLz901YHKKRcmk= sha512-EPuchwVvYPSFFIjGpH0k6wM0HQsmJ0vCk7BSl5ryxMVFIWW4hX4Kksu4PNtxfgOxDebTLkJQ8iC7zwAql0eusg==", - "license": "MIT", - "dependencies": { - "base16": "^1.0.0", - "lodash.curry": "^4.0.1", - "lodash.flow": "^3.3.0", - "pure-color": "^1.2.0" - } - }, "node_modules/react-dom": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", @@ -15576,21 +15396,6 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "license": "MIT" }, - "node_modules/react-json-tree": { - "version": "0.11.2", - "resolved": "https://registry.npmjs.org/react-json-tree/-/react-json-tree-0.11.2.tgz", - "integrity": "sha512-aYhUPj1y5jR3ZQ+G3N7aL8FbTyO03iLwnVvvEikLcNFqNTyabdljo9xDftZndUBFyyyL0aK3qGO9+8EilILHUw==", - "license": "MIT", - "dependencies": { - "babel-runtime": "^6.6.1", - "prop-types": "^15.5.8", - "react-base16-styling": "^0.5.1" - }, - "peerDependencies": { - "react": "^15.0.0 || ^16.0.0", - "react-dom": "^15.0.0 || ^16.0.0" - } - }, "node_modules/react-lifecycles-compat": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz", @@ -17133,11 +16938,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/string.prototype.repeat": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/string.prototype.repeat/-/string.prototype.repeat-0.2.0.tgz", - "integrity": "sha512-1BH+X+1hSthZFW+X+JaUkjkkUPwIlLEMJBLANN3hOob3RhEk5snLWNECDnYbgn/m5c5JV7Ersu1Yubaf+05cIA==" - }, "node_modules/string.prototype.trim": { "version": "1.2.10", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.10.tgz", @@ -19384,11 +19184,6 @@ "dev": true, "license": "MIT" }, - "node_modules/xss-filters": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/xss-filters/-/xss-filters-1.2.7.tgz", - "integrity": "sha512-KzcmYT/f+YzcYrYRqw6mXxd25BEZCxBQnf+uXTopQDIhrmiaLwO+f+yLsIvvNlPhYvgff8g3igqrBxYh9k8NbQ==" - }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 766fc98..089000c 100644 --- a/package.json +++ b/package.json @@ -186,7 +186,6 @@ "@electron-toolkit/utils": "^4.0.0", "@fortawesome/fontawesome-free": "^5.13.0", "@neurosity/pipes": "^5.2.1", - "@nteract/transforms": "^3.2.0", "@radix-ui/react-dialog": "^1.1.0", "@radix-ui/react-dropdown-menu": "^2.1.0", "@radix-ui/react-select": "^2.2.6", diff --git a/src/main/index.ts b/src/main/index.ts index 6320e05..652d458 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -265,7 +265,7 @@ ipcMain.handle( 'fs:storePyodideImage', (_event, title, imageTitle, rawData: ArrayBuffer) => { const dir = path.join(getWorkspaceDir(title), 'Results', 'Images'); - const filename = `${imageTitle}.png`; + const filename = `${imageTitle}.svg`; mkdirPathSync(dir); const buffer = Buffer.from(rawData); return new Promise((resolve, reject) => { diff --git a/src/renderer/components/PyodidePlotWidget.tsx b/src/renderer/components/PyodidePlotWidget.tsx index 4460563..325e8ab 100644 --- a/src/renderer/components/PyodidePlotWidget.tsx +++ b/src/renderer/components/PyodidePlotWidget.tsx @@ -1,88 +1,39 @@ import React, { Component } from 'react'; import { Button } from './ui/button'; -import { - richestMimetype, - standardDisplayOrder, - standardTransforms, -} from '@nteract/transforms'; -import { isNil } from 'lodash'; import { storePyodideImage } from '../utils/filesystem/storage'; interface Props { title: string; imageTitle: string; - plotMIMEBundle: - | { - [key: string]: string; - } - | null - | undefined; + plotMIMEBundle: { 'image/svg+xml': string } | null | undefined; } -interface State { - rawData: string; - mimeType: string; -} - -export default class PyodidePlotWidget extends Component { - // state: State; +export default class PyodidePlotWidget extends Component { constructor(props: Props) { super(props); - this.state = { - rawData: '', - mimeType: '', - }; this.handleSave = this.handleSave.bind(this); } - componentDidUpdate(prevProps: Props) { - if ( - this.props.plotMIMEBundle !== prevProps.plotMIMEBundle && - !isNil(this.props.plotMIMEBundle) - ) { - const bundle = this.props.plotMIMEBundle as { [key: string]: string }; - const mimeType = richestMimetype( - bundle, - standardDisplayOrder, - standardTransforms - ); - if (mimeType) { - this.setState({ rawData: bundle[mimeType], mimeType }); - } - } - } - handleSave() { - const buf = Buffer.from(this.state.rawData, 'base64'); - storePyodideImage( - this.props.title, - this.props.imageTitle, - buf.buffer as ArrayBuffer - ); - } - - renderResults() { - if (this.state.rawData) { - const Transform = standardTransforms[this.state.mimeType]; - return ; - } - } - - renderSaveButton() { - if (this.state.rawData) { - return ( - - ); - } + const svg = this.props.plotMIMEBundle?.['image/svg+xml']; + if (!svg) return; + const buf = Buffer.from(svg, 'utf8'); + storePyodideImage(this.props.title, this.props.imageTitle, buf.buffer as ArrayBuffer); } render() { + const svg = this.props.plotMIMEBundle?.['image/svg+xml']; + if (!svg) return
; return (
- {this.renderResults()} - {this.renderSaveButton()} + {this.props.imageTitle} +
); } diff --git a/src/renderer/epics/pyodideEpics.ts b/src/renderer/epics/pyodideEpics.ts index 874bcb5..8055015 100644 --- a/src/renderer/epics/pyodideEpics.ts +++ b/src/renderer/epics/pyodideEpics.ts @@ -97,7 +97,7 @@ const pyodideMessageEpic: Epic< } // Route plot results to the appropriate Redux state slot. // results is a base64-encoded PNG string returned from Python. - const mimeBundle = results ? { 'image/png': results } : null; + const mimeBundle = results ? { 'image/svg+xml': results } : null; switch (plotKey) { case 'topo': return of(PyodideActions.SetTopoPlot(mimeBundle)); case 'psd': return of(PyodideActions.SetPSDPlot(mimeBundle)); diff --git a/src/renderer/utils/webworker/index.ts b/src/renderer/utils/webworker/index.ts index e97a44c..1677605 100644 --- a/src/renderer/utils/webworker/index.ts +++ b/src/renderer/utils/webworker/index.ts @@ -115,12 +115,12 @@ export const plotPSD = async (worker: Worker) => { worker.postMessage({ plotKey: 'psd', data: [ - 'import io, base64', + 'import io', '_fig = raw.plot_psd(fmin=1, fmax=30, show=False)', '_buf = io.BytesIO()', - '_fig.savefig(_buf, format="png", bbox_inches="tight")', + '_fig.savefig(_buf, format="svg", bbox_inches="tight")', 'plt.close(_fig)', - 'base64.b64encode(_buf.getvalue()).decode()', + '_buf.getvalue().decode()', ].join('\n'), }); }; @@ -129,12 +129,12 @@ export const plotTopoMap = async (worker: Worker) => { worker.postMessage({ plotKey: 'topo', data: [ - 'import io, base64', + 'import io', '_fig = plot_topo(clean_epochs, conditions)', '_buf = io.BytesIO()', - '_fig.savefig(_buf, format="png", bbox_inches="tight")', + '_fig.savefig(_buf, format="svg", bbox_inches="tight")', 'plt.close(_fig)', - 'base64.b64encode(_buf.getvalue()).decode()', + '_buf.getvalue().decode()', ].join('\n'), }); }; @@ -144,15 +144,15 @@ export const plotTestPlot = async (worker: Worker | null) => { worker.postMessage({ plotKey: 'topo', data: [ - 'import io, base64', + 'import io', 'import matplotlib.pyplot as plt', '_fig, _ax = plt.subplots()', '_ax.plot([1, 2, 3, 4], [1, 4, 2, 3])', '_ax.set_title("Test Plot")', '_buf = io.BytesIO()', - '_fig.savefig(_buf, format="png", bbox_inches="tight")', + '_fig.savefig(_buf, format="svg", bbox_inches="tight")', 'plt.close(_fig)', - 'base64.b64encode(_buf.getvalue()).decode()', + '_buf.getvalue().decode()', ].join('\n'), }); }; @@ -161,12 +161,12 @@ export const plotERP = async (worker: Worker, channelIndex: number) => { worker.postMessage({ plotKey: 'erp', data: [ - 'import io, base64', + 'import io', `_fig, _ = plot_conditions(clean_epochs, ch_ind=${channelIndex}, conditions=conditions, ci=97.5, n_boot=1000, title='', diff_waveform=None)`, '_buf = io.BytesIO()', - '_fig.savefig(_buf, format="png", bbox_inches="tight")', + '_fig.savefig(_buf, format="svg", bbox_inches="tight")', 'plt.close(_fig)', - 'base64.b64encode(_buf.getvalue()).decode()', + '_buf.getvalue().decode()', ].join('\n'), }); }; -- 2.51.2 From 608229c8e99e6c9f6355bab829cec19dccf9c4f0 Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sun, 15 Mar 2026 16:18:44 -0400 Subject: [PATCH 12/12] Add SVG and PNG buttons --- docs/pyodide-in-electron-vite.md | 135 ++++++++++++++++++ src/main/index.ts | 21 ++- src/preload/index.ts | 11 +- src/renderer/actions/pyodideActions.ts | 1 + .../components/HomeComponent/index.tsx | 6 +- src/renderer/components/PyodidePlotWidget.tsx | 63 ++++++-- src/renderer/epics/pyodideEpics.ts | 1 + src/renderer/reducers/pyodideReducer.ts | 5 + src/renderer/utils/filesystem/storage.ts | 10 +- src/renderer/utils/webworker/index.ts | 1 + 10 files changed, 236 insertions(+), 18 deletions(-) diff --git a/docs/pyodide-in-electron-vite.md b/docs/pyodide-in-electron-vite.md index 34c5934..dc8ef5f 100644 --- a/docs/pyodide-in-electron-vite.md +++ b/docs/pyodide-in-electron-vite.md @@ -163,6 +163,138 @@ Current PyPI packages: `mne`, `pooch`, `tqdm`, `platformdirs`, `lazy-loader` --- +## Plot Pipeline + +### matplotlib Backend in Web Workers + +Use `agg`, not `webagg`. Set it before any Python imports run: + +```js +await pyodide.runPythonAsync('import os; os.environ["MPLBACKEND"] = "agg"'); +``` + +WebAgg (`webagg`) fails in web workers because it tries to inject CSS via `js.document` during initialisation — and `js.document` does not exist in worker scope. The error looks like: + +``` +ImportError: cannot import name 'document' from 'js' +``` + +`agg` is a non-interactive raster backend that writes to a buffer, which is exactly what we need. + +--- + +### plotKey Correlation Pattern (Fire-and-Forget Messaging) + +`worker.postMessage()` returns `undefined` — there is no return channel. Redux-Observable plot load epics cannot receive the worker's result directly. + +**Solution:** attach a `plotKey` string to every outgoing message; the worker echoes it back in the response object. `pyodideMessageEpic` routes by `plotKey` to the correct Redux action. + +```js +// webworker.js — echo plotKey back in every response +const { data, plotKey, ...context } = event.data; +self.postMessage({ results: await pyodide.runPythonAsync(data), plotKey }); +``` + +```ts +// pyodideMessageEpic — route by plotKey +switch (plotKey) { + case 'ready': return of(PyodideActions.SetWorkerReady()); + case 'topo': return of(PyodideActions.SetTopoPlot(mimeBundle)); + case 'psd': return of(PyodideActions.SetPSDPlot(mimeBundle)); + case 'erp': return of(PyodideActions.SetERPPlot(mimeBundle)); + default: return of(PyodideActions.ReceiveMessage(e.data)); +} +``` + +Plot load epics become fire-and-forget — they call `worker.postMessage()` as a side effect and emit nothing: + +```ts +// loadTopoEpic +action$.pipe( + filter(isActionOf(PyodideActions.LoadTopo)), + tap(() => plotTestPlot(state$.value.pyodide.worker!)), + mergeMap(() => EMPTY) +); +``` + +--- + +### Worker Readiness Gating + +`loadUtils` posts `plotKey: 'ready'` when `utils.py` finishes loading. This drives an `isWorkerReady` flag in Redux state that gates any UI that depends on Python being initialised. + +```ts +export const loadUtils = async (worker: Worker) => + worker.postMessage({ data: utilsPy, plotKey: 'ready' }); +``` + +`pyodideMessageEpic` dispatches `PyodideActions.SetWorkerReady()` on receiving `plotKey === 'ready'`. + +--- + +### SVG Output from matplotlib + +Produce SVG in Python — no base64 encoding needed: + +```python +import io +import matplotlib.pyplot as plt + +_fig, _ax = plt.subplots() +_ax.plot([1, 2, 3, 4], [1, 4, 2, 3]) +_buf = io.BytesIO() +_fig.savefig(_buf, format="svg", bbox_inches="tight") +plt.close(_fig) +_buf.getvalue().decode() # SVG string is the Python return value +``` + +The SVG string flows through `pyodide.runPythonAsync()` → worker `postMessage` → Redux state as `{ 'image/svg+xml': string }`. + +--- + +### Rendering SVG Safely in the Renderer + +Use a data URI on an `` tag — sandboxed, no script execution: + +```tsx + +``` + +Prefer this over `dangerouslySetInnerHTML` — inline SVG executes `