diff --git a/internals/scripts/InstallMNE.mjs b/internals/scripts/InstallMNE.mjs new file mode 100644 index 0000000..23f1466 --- /dev/null +++ b/internals/scripts/InstallMNE.mjs @@ -0,0 +1,177 @@ +#!/usr/bin/env node +/** + * Downloads MNE-Python and its pure-Python dependencies from PyPI as wheel + * files for offline use with Pyodide. Binary dependencies (numpy, scipy, + * matplotlib, pandas) are already included via the `pyodide` npm package and + * do NOT need to be downloaded here. + * + * Downloaded wheels are saved to: + * src/renderer/utils/pyodide/src/packages/ + * + * A manifest.json is written there so the web worker knows which filenames + * to pass to micropip.install() at startup. + * + * Usage: node internals/scripts/InstallMNE.mjs + */ + +import fs from 'fs'; +import https from 'https'; +import path from 'path'; +import chalk from 'chalk'; + +const PACKAGES_DIR = path.resolve( + 'src/renderer/utils/pyodide/src/packages' +); +const MANIFEST_FILE = path.join(PACKAGES_DIR, 'manifest.json'); + +/** + * Pure-Python packages required by MNE that are not bundled with Pyodide. + * Each entry is resolved against the PyPI JSON API to find the latest + * pure-Python wheel (py3-none-any or py2.py3-none-any). + */ +const PACKAGES_TO_DOWNLOAD = [ + 'mne', + 'pooch', + 'tqdm', + 'platformdirs', +]; + +// --------------------------------------------------------------------------- +// Network helpers +// --------------------------------------------------------------------------- + +function httpsGet(url) { + return new Promise((resolve, reject) => { + const req = https.get(url, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + resolve(httpsGet(res.headers.location)); + return; + } + if (res.statusCode !== 200) { + reject(new Error(`HTTP ${res.statusCode} for ${url}`)); + return; + } + let body = ''; + res.setEncoding('utf8'); + res.on('data', (chunk) => { body += chunk; }); + res.on('end', () => resolve(body)); + res.on('error', reject); + }); + req.on('error', reject); + }); +} + +function downloadBinary(url, dest) { + return new Promise((resolve, reject) => { + const doGet = (reqUrl) => { + https.get(reqUrl, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + doGet(res.headers.location); + return; + } + if (res.statusCode !== 200) { + reject(new Error(`HTTP ${res.statusCode} for ${reqUrl}`)); + return; + } + const file = fs.createWriteStream(dest); + res.pipe(file); + file.on('finish', () => file.close(resolve)); + file.on('error', (err) => { fs.unlink(dest, () => {}); reject(err); }); + }).on('error', (err) => { fs.unlink(dest, () => {}); reject(err); }); + }; + doGet(url); + }); +} + +// --------------------------------------------------------------------------- +// PyPI helpers +// --------------------------------------------------------------------------- + +/** + * Returns the best pure-Python wheel for the latest release of `packageName`. + * Preference: py3-none-any > py2.py3-none-any > *-none-any + */ +async function resolvePureWheel(packageName) { + const raw = await httpsGet(`https://pypi.org/pypi/${packageName}/json`); + const data = JSON.parse(raw); + const version = data.info.version; + const urls = data.urls; // files for the latest release + + const wheels = urls.filter((f) => f.filename.endsWith('.whl')); + + const ranked = [ + wheels.find((f) => f.filename.endsWith('-py3-none-any.whl')), + wheels.find((f) => f.filename.endsWith('-py2.py3-none-any.whl')), + wheels.find((f) => f.filename.includes('-none-any.whl')), + ].filter(Boolean); + + if (ranked.length === 0) { + throw new Error( + `No pure-Python wheel found for ${packageName} ${version}. ` + + `Binary packages must come from the Pyodide npm bundle.` + ); + } + + return { version, wheel: ranked[0] }; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function installPackage(packageName, manifest) { + process.stdout.write(chalk.blue(` ${packageName}: `)); + + let version, wheel; + try { + ({ version, wheel } = await resolvePureWheel(packageName)); + } catch (err) { + console.log(chalk.red(`FAILED — ${err.message}`)); + return; + } + + const dest = path.join(PACKAGES_DIR, wheel.filename); + + if (fs.existsSync(dest)) { + console.log(chalk.gray(`${version} already present, skipping`)); + manifest[packageName] = { version, filename: wheel.filename }; + return; + } + + try { + await downloadBinary(wheel.url, dest); + console.log(chalk.green(`${version} downloaded`)); + manifest[packageName] = { version, filename: wheel.filename }; + } catch (err) { + console.log(chalk.red(`FAILED — ${err.message}`)); + if (fs.existsSync(dest)) fs.unlinkSync(dest); + } +} + +async function main() { + fs.mkdirSync(PACKAGES_DIR, { recursive: true }); + + // Preserve any previously downloaded packages in the manifest. + let manifest = {}; + if (fs.existsSync(MANIFEST_FILE)) { + try { + manifest = JSON.parse(fs.readFileSync(MANIFEST_FILE, 'utf8')); + } catch { + manifest = {}; + } + } + + console.log(chalk.blue.bold('Downloading MNE-Python wheels from PyPI…')); + for (const pkg of PACKAGES_TO_DOWNLOAD) { + await installPackage(pkg, manifest); + } + + fs.writeFileSync(MANIFEST_FILE, JSON.stringify(manifest, null, 2)); + console.log(chalk.green.bold('\nAll MNE wheels ready.')); + console.log(chalk.gray(`Manifest → ${MANIFEST_FILE}`)); +} + +main().catch((err) => { + console.error(chalk.red('Fatal error:'), err); + process.exit(1); +}); diff --git a/internals/scripts/InstallPyodide.js b/internals/scripts/InstallPyodide.js deleted file mode 100644 index aea88f8..0000000 --- a/internals/scripts/InstallPyodide.js +++ /dev/null @@ -1,66 +0,0 @@ -import chalk from 'chalk'; -import fs from 'fs'; -import https from 'https'; -import mkdirp from 'mkdirp'; -import tar from 'tar-fs'; -import url from 'url'; -import bz2 from 'unbzip2-stream'; - -const PYODIDE_VERSION = '0.27.0'; -const TAR_NAME = `pyodide-${PYODIDE_VERSION}.tar.bz2`; -const TAR_URL = `https://github.com/pyodide/pyodide/releases/download/${PYODIDE_VERSION}/pyodide-${PYODIDE_VERSION}.tar.bz2`; -const PYODIDE_DIR = 'src/renderer/utils/pyodide/src/'; - -const writeAndUnzipFile = (response) => { - const filePath = `${PYODIDE_DIR}${TAR_NAME}`; - const writeStream = fs.createWriteStream(filePath); - response.pipe(writeStream); - - writeStream.on('finish', () => { - console.log(`${chalk.green.bold(`Unzipping pyodide`)}`); - - const readStream = fs.createReadStream(filePath); - try { - readStream.pipe(bz2()).pipe(tar.extract(PYODIDE_DIR)); - } catch (e) { - throw new Error('Error in unzip:', e); - } - - readStream.on('end', () => { - console.log(`${chalk.green.bold(`Unzip successful`)}`); - }); - }); -}; - -const downloadFile = (response) => { - if ( - response.statusCode > 300 && - response.statusCode < 400 && - response.headers.location - ) { - if (url.parse(response.headers.location).hostname) { - https.get(response.headers.location, writeAndUnzipFile); - } else { - https.get( - url.resolve(url.parse(TAR_URL).hostname, response.headers.location), - writeAndUnzipFile - ); - } - } else { - writeAndUnzipFile(response); - } -}; - -(() => { - if (fs.existsSync(`${PYODIDE_DIR}${TAR_NAME}`)) { - console.log( - `${chalk.green.bold(`Pyodide is already present: ${PYODIDE_VERSION}...`)}` - ); - return; - } - console.log( - `${chalk.green.bold(`Downloading pyodide ${PYODIDE_VERSION}...`)}` - ); - mkdirp.sync(`src/renderer/utils/pyodide/src`); - https.get(TAR_URL, downloadFile); -})(); diff --git a/internals/scripts/InstallPyodide.mjs b/internals/scripts/InstallPyodide.mjs new file mode 100644 index 0000000..4ce6125 --- /dev/null +++ b/internals/scripts/InstallPyodide.mjs @@ -0,0 +1,108 @@ +#!/usr/bin/env node +/** + * Downloads the Pyodide core tarball from GitHub releases and extracts it + * into the renderer's public directory so Vite serves the runtime as a static + * asset at /pyodide/… (Vite publicDir → src/renderer/utils/pyodide/src/). + * + * The "core" tarball is ~40 MB and includes the runtime plus the most + * commonly used scientific packages (numpy, scipy, matplotlib, pandas, …). + * It is much smaller than the full Pyodide build (~500 MB). + * + * Usage: node internals/scripts/InstallPyodide.mjs + * Runs automatically via the postinstall npm hook. + */ + +import fs from 'fs'; +import https from 'https'; +import path from 'path'; +import { pipeline } from 'stream/promises'; +import chalk from 'chalk'; +import bz2 from 'unbzip2-stream'; +import tar from 'tar-fs'; + +const PYODIDE_VERSION = '0.29.3'; +const TARBALL_NAME = `pyodide-core-${PYODIDE_VERSION}.tar.bz2`; +const TARBALL_URL = `https://github.com/pyodide/pyodide/releases/download/${PYODIDE_VERSION}/${TARBALL_NAME}`; + +// Vite publicDir root — everything here is served verbatim by the dev server. +const PUBLIC_ROOT = path.resolve('src/renderer/utils/pyodide/src'); +// The tarball extracts into a `pyodide/` subdirectory, which ends up at: +// src/renderer/utils/pyodide/src/pyodide/ → served at /pyodide/ +const DEST_DIR = path.join(PUBLIC_ROOT, 'pyodide'); +const VERSION_FILE = path.join(DEST_DIR, '.pyodide-version'); + +// --------------------------------------------------------------------------- +// Network helpers (follow redirects) +// --------------------------------------------------------------------------- + +function httpsGetResponse(url) { + return new Promise((resolve, reject) => { + https + .get(url, { headers: { 'User-Agent': 'BrainWaves-installer/1.0' } }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + resolve(httpsGetResponse(res.headers.location)); + } else { + resolve(res); + } + }) + .on('error', reject); + }); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + // Skip if this exact version is already extracted. + if ( + fs.existsSync(VERSION_FILE) && + fs.readFileSync(VERSION_FILE, 'utf8').trim() === PYODIDE_VERSION + ) { + console.log( + chalk.green.bold(`Pyodide ${PYODIDE_VERSION} already installed, skipping.`) + ); + return; + } + + fs.mkdirSync(PUBLIC_ROOT, { recursive: true }); + + const tarballPath = path.join(PUBLIC_ROOT, TARBALL_NAME); + + // Download the tarball if not already cached. + if (!fs.existsSync(tarballPath)) { + console.log( + chalk.blue.bold(`Downloading Pyodide ${PYODIDE_VERSION} core tarball…`) + ); + const res = await httpsGetResponse(TARBALL_URL); + if (res.statusCode !== 200) { + throw new Error(`Failed to download tarball: HTTP ${res.statusCode}`); + } + await pipeline(res, fs.createWriteStream(tarballPath)); + console.log(chalk.gray(` Saved → ${tarballPath}`)); + } else { + console.log(chalk.gray(` Tarball already cached, skipping download.`)); + } + + // Extract the tarball. The archive contains a top-level `pyodide/` + // directory, so extracting into PUBLIC_ROOT gives us PUBLIC_ROOT/pyodide/. + console.log(chalk.blue.bold(`Extracting…`)); + await pipeline( + fs.createReadStream(tarballPath), + bz2(), + tar.extract(PUBLIC_ROOT) + ); + + // Stamp the installed version and clean up the cached tarball. + fs.writeFileSync(VERSION_FILE, PYODIDE_VERSION); + fs.unlinkSync(tarballPath); + + console.log( + chalk.green.bold(`Pyodide ${PYODIDE_VERSION} installed successfully.`) + ); +} + +main().catch((err) => { + console.error(chalk.red('Fatal error:'), err); + process.exit(1); +}); diff --git a/package.json b/package.json index 4188e67..304e0a8 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "package-mac": "npm run build && electron-builder build --mac", "package-linux": "npm run build && electron-builder build --linux", "package-win": "npm run build && electron-builder build --win --x64", - "postinstall": "electron-builder install-app-deps && node internals/scripts/InstallPyodide.js && node internals/scripts/patchDeps.mjs", + "postinstall": "electron-builder install-app-deps && node internals/scripts/InstallPyodide.mjs && node internals/scripts/InstallMNE.mjs && node internals/scripts/patchDeps.mjs", "lint": "cross-env NODE_ENV=development eslint . --cache", "lint-fix": "npm run lint -- --fix", "lint-styles": "stylelint '**/*.*(css|scss)'", diff --git a/src/renderer/utils/pyodide/webworker.js b/src/renderer/utils/pyodide/webworker.js index eaa2e1f..b50af8c 100644 --- a/src/renderer/utils/pyodide/webworker.js +++ b/src/renderer/utils/pyodide/webworker.js @@ -1,29 +1,67 @@ /** - * This file has been copied from pyodide source and modified to allow - * pyodide to be used in a web worker within this + * Pyodide Web Worker + * + * Load order: + * 1. Pyodide runtime (served as a static asset at /pyodide/). + * 2. Binary packages bundled with Pyodide (numpy, scipy, matplotlib, pandas). + * 3. MNE-Python and its pure-Python deps, installed offline from local + * wheel files that were pre-downloaded by `npm run install-mne-wheels`. + * The manifest at /packages/manifest.json maps package names → filenames. */ -// pyodide is served as a static asset at /pyodide/ (via Vite publicDir). +// Pyodide is served as a static asset at /pyodide/ (via Vite publicDir). // An absolute path is required so importScripts resolves correctly regardless // of where the worker script itself is served from. importScripts('/pyodide/pyodide.js'); async function loadPyodideAndPackages() { self.pyodide = await loadPyodide({ indexURL: '/pyodide/' }); - await self.pyodide.loadPackage(['matplotlib', 'mne', 'pandas']); + + // Load binary packages that are bundled with the Pyodide npm package and + // therefore available locally without any network request. + await self.pyodide.loadPackage(['numpy', 'scipy', 'matplotlib', 'pandas']); + + // Load MNE and its pure-Python dependencies from pre-downloaded wheel files. + // The manifest was written by `node internals/scripts/InstallMNE.mjs`. + let manifest = {}; + try { + const response = await fetch('/packages/manifest.json'); + if (response.ok) { + manifest = await response.json(); + } else { + console.warn('[pyodide worker] manifest.json not found — MNE will not be available'); + } + } catch (err) { + console.warn('[pyodide worker] Could not fetch manifest.json:', err); + } + + const wheelUrls = Object.values(manifest) + .map((entry) => `/packages/${entry.filename}`); + + if (wheelUrls.length > 0) { + await self.pyodide.loadPackage('micropip'); + const micropip = self.pyodide.pyimport('micropip'); + // micropip resolves relative URLs against the worker's base URL. + // Pass absolute URLs so it works regardless of worker location. + const absoluteUrls = wheelUrls.map( + (u) => new URL(u, self.location.origin).href + ); + await micropip.install(absoluteUrls); + } else { + console.warn('[pyodide worker] No MNE wheels found in manifest.'); + } } + let pyodideReadyPromise = loadPyodideAndPackages(); self.onmessage = async (event) => { - // make sure loading is done await pyodideReadyPromise; - // Don't bother yet with this line, suppose our API is built in such a way: + const { data, ...context } = event.data; - // The worker copies the context in its own "memory" (an object mapping name to values) for (const key of Object.keys(context)) { self[key] = context[key]; } - // Now is the easy part, the one that is similar to working in the main thread: + try { self.postMessage({ results: await self.pyodide.runPythonAsync(data),