diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,20 +7,10 @@ *.pid *.seed -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - # Coverage directory used by tools like istanbul coverage -# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (http://nodejs.org/api/addons.html) -build/Release +# Compiled binary addons .eslintcache # Dependency directory @@ -35,23 +25,6 @@ out dist -# Old webpack build artifacts -app/main.prod.js -app/main.prod.js.map -app/renderer.prod.js -app/renderer.prod.js.map -app/style.css -app/style.css.map -dll -main.js -main.js.map - .idea -npm-debug.log.* -*.css.d.ts -*.sass.d.ts -*.scss.d.ts keys.js - -app/utils/pyodide/src -src/renderer/utils/pyodide/src +src/renderer/utils/webworker/src diff --git a/eslint.config.mjs b/eslint.config.mjs --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -23,7 +23,7 @@ 'dist/**', 'coverage/**', '.worktrees/**', - 'src/renderer/utils/pyodide/src/**', + 'src/renderer/utils/webworker/src/**', '**/*.css.d.ts', '**/*.scss.d.ts', ], diff --git a/package-lock.json b/package-lock.json --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,6 @@ "@electron-toolkit/utils": "^4.0.0", "@fortawesome/fontawesome-free": "^5.13.0", "@neurosity/pipes": "^5.2.1", - "@nteract/transforms": "^3.2.0", "@radix-ui/react-dialog": "^1.1.0", "@radix-ui/react-dropdown-menu": "^2.1.0", "@radix-ui/react-select": "^2.2.6", @@ -37,6 +36,7 @@ "papaparse": "^5.5.3", "pathe": "^2.0.3", "plotly.js": "^3.4.0", + "pyodide": "^0.29.3", "rc-slider": "9.2.4", "react": "^18.x", "react-dom": "^18.x", @@ -592,18 +592,6 @@ "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz", "integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==", "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/runtime-corejs2": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.28.6.tgz", - "integrity": "sha512-pOHfxftxpetWUeBacCB3ZOPc/OO6hiT9MLv0qd9j474khiCcduwO8uuJI3N7vX3m8GJotTT6lxlA89TS/PylGg==", - "license": "MIT", - "dependencies": { - "core-js": "^2.6.12" - }, "engines": { "node": ">=6.9.0" } @@ -2578,56 +2566,6 @@ "node": ">=10" } }, - "node_modules/@nteract/transform-vdom": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/@nteract/transform-vdom/-/transform-vdom-2.2.5.tgz", - "integrity": "sha512-q6FbWlrSEWUmQpDV1DBPcw5FZpUcQbKOQ2a59vY/qcQ/Qjh1KUCC+gortso+WIE4P36eHZRxKz5ptCu5i47OLg==", - "license": "BSD-3-Clause", - "dependencies": { - "@babel/runtime-corejs2": "^7.0.0", - "babel-runtime": "^6.26.0", - "lodash": "^4.17.4" - }, - "peerDependencies": { - "react": "^16.3.2" - } - }, - "node_modules/@nteract/transforms": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@nteract/transforms/-/transforms-3.2.0.tgz", - "integrity": "sha512-9P926e2tm0H1IHF2ER6f0+At5NPgrMgvNOPZTn+K6e9M9+EpNPbZq4q5YUX1xKG8YaK2fpiH+4XVkFBf06YOJg==", - "deprecated": "This package has been deprecated. Please access each transform through its own package.", - "license": "BSD-3-Clause", - "dependencies": { - "@nteract/transform-vdom": "^2.1.0", - "ansi-to-react": "^2.0.6", - "commonmark": "^0.28.0", - "commonmark-react-renderer": "^4.3.3", - "mathjax-electron": "^2.0.1", - "react-json-tree": "^0.11.0" - }, - "peerDependencies": { - "react": "^16.2.0" - } - }, - "node_modules/@nteract/transforms/node_modules/commonmark": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/commonmark/-/commonmark-0.28.1.tgz", - "integrity": "sha512-PklsZ9pgrfFQ5hQH9BRzoWnqI9db2LeR9MhvkNk8iz97kfaTNmhTU+IE8jKDHTEfivZZXoFqzGqzddXdk14EJw==", - "license": "BSD-2-Clause", - "dependencies": { - "entities": "~ 1.1.1", - "mdurl": "~ 1.0.1", - "minimist": "~ 1.2.0", - "string.prototype.repeat": "^0.2.0" - }, - "bin": { - "commonmark": "bin/commonmark" - }, - "engines": { - "node": "*" - } - }, "node_modules/@parcel/watcher": { "version": "2.5.6", "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.6.tgz", @@ -4538,6 +4476,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -5274,12 +5218,6 @@ "integrity": "sha512-0V/PkoculFl5+0Lp47JoxUcO0xSxhIBvm+BxHdD/OgXNmdRpRHCFnKVuUoWyS9EzQP+otSGv0m9Lb4yVkQBn2A==", "license": "MIT" }, - "node_modules/anser": { - "version": "1.4.10", - "resolved": "https://registry.npmjs.org/anser/-/anser-1.4.10.tgz", - "integrity": "sha512-hCv9AqTQ8ycjpSd3upOJd7vFwW1JaoYQ7tpham03GJ1ca8/65rqn0RpaWpItOAd6ylW9wAw6luXYPJIyPFVOww==", - "license": "MIT" - }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -5305,20 +5243,6 @@ }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/ansi-to-react": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/ansi-to-react/-/ansi-to-react-2.0.6.tgz", - "integrity": "sha512-AnzmnQcMmCqbd72cRridR94RR0YQpv7Bvbm7YNSGnReTwFQmLkfaZzw4Ajg7HRfR6ZxCEa90sJWEZFhCOPcWhA==", - "license": "MPL-2.0", - "dependencies": { - "anser": "^1.4.1", - "escape-carriage": "^1.2.0" - }, - "peerDependencies": { - "react": "^16.2.0", - "react-dom": "^16.2.0" } }, "node_modules/any-promise": { @@ -5987,12 +5911,6 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true, - "license": "MIT" - }, - "node_modules/base16": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/base16/-/base16-1.0.0.tgz", - "integrity": "sha1-4pf2DX7BAUp6lxo568ipjAtoHnA= sha512-pNdYkNPiJUnEhnfXV56+sQy8+AaPcG3POZAUnwr4EeqCUZFz4u2PePbo3e5Gj4ziYPCWGUZT9RHisvJKnwFuBQ==", "license": "MIT" }, "node_modules/base64-arraybuffer": { @@ -6856,22 +6774,6 @@ "license": "MIT", "engines": { "node": ">=4.0.0" - } - }, - "node_modules/commonmark-react-renderer": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/commonmark-react-renderer/-/commonmark-react-renderer-4.3.5.tgz", - "integrity": "sha512-UwUgplz8kFSMCe9+Dg/BcV75lc7R/V6mvMYJq2p29i5aaIBd0252k9HeSGa2VtEPHfg2/trS9qC7iAxnO7r6ng==", - "license": "MIT", - "dependencies": { - "lodash.assign": "^4.2.0", - "lodash.isplainobject": "^4.0.6", - "pascalcase": "^0.1.1", - "xss-filters": "^1.2.6" - }, - "peerDependencies": { - "commonmark": "^0.27.0 || ^0.26.0 || ^0.24.0", - "react": ">=0.14.0" } }, "node_modules/compare-version": { @@ -9179,12 +9081,6 @@ "once": "^1.4.0" } }, - "node_modules/entities": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", - "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", - "license": "BSD-2-Clause" - }, "node_modules/env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", @@ -9520,12 +9416,6 @@ "engines": { "node": ">=6" } - }, - "node_modules/escape-carriage": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/escape-carriage/-/escape-carriage-1.3.1.tgz", - "integrity": "sha512-GwBr6yViW3ttx1kb7/Oh+gKQ1/TrhYwxKqVmg5gS+BK+Qe2KrOa/Vh7w3HPBvgGf0LfcDGoY9I6NHKoA5Hozhw==", - "license": "MIT" }, "node_modules/escape-string-regexp": { "version": "4.0.0", @@ -12967,28 +12857,10 @@ "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", "license": "MIT" }, - "node_modules/lodash.assign": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz", - "integrity": "sha512-hFuH8TY+Yji7Eja3mGiuAxBqLagejScbG8GbG0j6o9vzn0YL14My+ktnqtZgFTosKymC9/44wP6s7xyuLfnClw==", - "license": "MIT" - }, - "node_modules/lodash.curry": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lodash.curry/-/lodash.curry-4.1.1.tgz", - "integrity": "sha1-JI42By7ekGUB11lmIAqG2riyMXA= sha512-/u14pXGviLaweY5JI0IUzgzF2J6Ne8INyzAZjImcryjgkZ+ebruBxy2/JaOOkTqScddcYtakjhSaeemV8lR0tA==", - "license": "MIT" - }, "node_modules/lodash.escaperegexp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", - "license": "MIT" - }, - "node_modules/lodash.flow": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/lodash.flow/-/lodash.flow-3.5.0.tgz", - "integrity": "sha1-h79AKSuM+D5OjOGjrkIJ4gBxZ1o= sha512-ff3BX/tSioo+XojX4MOsOMhJw0nZoUEF011LX8g8d3gvjVbxd89cCio4BCXronjxcTUIJUoqKEUA+n4CqvvRPw==", "license": "MIT" }, "node_modules/lodash.isequal": { @@ -12996,12 +12868,6 @@ "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA= sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", - "license": "MIT" - }, - "node_modules/lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", "license": "MIT" }, "node_modules/lodash.merge": { @@ -13460,12 +13326,6 @@ "node": ">=0.10.0" } }, - "node_modules/mathjax-electron": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mathjax-electron/-/mathjax-electron-2.0.1.tgz", - "integrity": "sha512-bllJaZZUccbj1ReD9i0V6qwu27dZXbd7TG/Wy3M7F10NLEjl8yN0WgFFP9uYf35s0hkody6wSPO96txr68TOqg==", - "license": "MIT" - }, "node_modules/mathml-tag-names": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/mathml-tag-names/-/mathml-tag-names-4.0.0.tgz", @@ -13483,12 +13343,6 @@ "integrity": "sha512-IEn+pegP1aManZuckezWCO+XZQDplx1366JoVhTpMpBB1sPey/SbveZQUosKiKiGYjg1wH4pMlNgXbCiYgihQA==", "dev": true, "license": "CC0-1.0" - }, - "node_modules/mdurl": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", - "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", - "license": "MIT" }, "node_modules/meow": { "version": "14.0.0", @@ -14614,15 +14468,6 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, - "node_modules/pascalcase": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", - "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -15298,11 +15143,18 @@ "node": ">=6" } }, - "node_modules/pure-color": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/pure-color/-/pure-color-1.3.0.tgz", - "integrity": "sha1-H+Bk+wrIUfDeYTIKi/eWg2Qi8z4= sha512-QFADYnsVoBMw1srW7OVKEYjG+MbIa49s54w1MA1EDY6r2r/sTcKKYqRX1f4GYvnXP7eN/Pe9HFcX+hwzmrXRHA==", - "license": "MIT" + "node_modules/pyodide": { + "version": "0.29.3", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-0.29.3.tgz", + "integrity": "sha512-22UBuhOJawj7vKUnS7/F3xK+515LJdjiMAHoCfuS6/PbHiOrSQVnYwDe+2sbVwiOZ3sMMexdXICew6NqOMQGgA==", + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } }, "node_modules/qified": { "version": "0.6.0", @@ -15525,18 +15377,6 @@ "node": ">=0.10.0" } }, - "node_modules/react-base16-styling": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/react-base16-styling/-/react-base16-styling-0.5.3.tgz", - "integrity": "sha1-OFjyTpxN2MvT9wLz901YHKKRcmk= sha512-EPuchwVvYPSFFIjGpH0k6wM0HQsmJ0vCk7BSl5ryxMVFIWW4hX4Kksu4PNtxfgOxDebTLkJQ8iC7zwAql0eusg==", - "license": "MIT", - "dependencies": { - "base16": "^1.0.0", - "lodash.curry": "^4.0.1", - "lodash.flow": "^3.3.0", - "pure-color": "^1.2.0" - } - }, "node_modules/react-dom": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", @@ -15555,21 +15395,6 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "license": "MIT" - }, - "node_modules/react-json-tree": { - "version": "0.11.2", - "resolved": "https://registry.npmjs.org/react-json-tree/-/react-json-tree-0.11.2.tgz", - "integrity": "sha512-aYhUPj1y5jR3ZQ+G3N7aL8FbTyO03iLwnVvvEikLcNFqNTyabdljo9xDftZndUBFyyyL0aK3qGO9+8EilILHUw==", - "license": "MIT", - "dependencies": { - "babel-runtime": "^6.6.1", - "prop-types": "^15.5.8", - "react-base16-styling": "^0.5.1" - }, - "peerDependencies": { - "react": "^15.0.0 || ^16.0.0", - "react-dom": "^15.0.0 || ^16.0.0" - } }, "node_modules/react-lifecycles-compat": { "version": "3.0.4", @@ -17112,11 +16937,6 @@ "funding": { "url": "https://github.com/sponsors/ljharb" } - }, - "node_modules/string.prototype.repeat": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/string.prototype.repeat/-/string.prototype.repeat-0.2.0.tgz", - "integrity": "sha512-1BH+X+1hSthZFW+X+JaUkjkkUPwIlLEMJBLANN3hOob3RhEk5snLWNECDnYbgn/m5c5JV7Ersu1Yubaf+05cIA==" }, "node_modules/string.prototype.trim": { "version": "1.2.10", @@ -19363,11 +19183,6 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true, "license": "MIT" - }, - "node_modules/xss-filters": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/xss-filters/-/xss-filters-1.2.7.tgz", - "integrity": "sha512-KzcmYT/f+YzcYrYRqw6mXxd25BEZCxBQnf+uXTopQDIhrmiaLwO+f+yLsIvvNlPhYvgff8g3igqrBxYh9k8NbQ==" }, "node_modules/xtend": { "version": "4.0.2", diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "package-mac": "npm run build && electron-builder build --mac", "package-linux": "npm run build && electron-builder build --linux", "package-win": "npm run build && electron-builder build --win --x64", - "postinstall": "electron-builder install-app-deps && node internals/scripts/InstallPyodide.js && node internals/scripts/patchDeps.mjs", + "postinstall": "electron-builder install-app-deps && node internals/scripts/InstallPyodide.mjs && node internals/scripts/InstallMNE.mjs && node internals/scripts/patchDeps.mjs", + "install-pyodide": "node internals/scripts/InstallPyodide.mjs && node internals/scripts/InstallMNE.mjs", "lint": "cross-env NODE_ENV=development eslint . --cache", "lint-fix": "npm run lint -- --fix", "lint-styles": "stylelint '**/*.*(css|scss)'", @@ -97,7 +98,7 @@ "filter": "**/*" }, { - "from": "./src/renderer/utils/pyodide/src/", + "from": "./src/renderer/utils/webworker/src/", "to": "pyodide", "filter": "**/*" } @@ -185,7 +186,6 @@ "@electron-toolkit/utils": "^4.0.0", "@fortawesome/fontawesome-free": "^5.13.0", "@neurosity/pipes": "^5.2.1", - "@nteract/transforms": "^3.2.0", "@radix-ui/react-dialog": "^1.1.0", "@radix-ui/react-dropdown-menu": "^2.1.0", "@radix-ui/react-select": "^2.2.6", @@ -208,6 +208,7 @@ "papaparse": "^5.5.3", "pathe": "^2.0.3", "plotly.js": "^3.4.0", + "pyodide": "^0.29.3", "rc-slider": "9.2.4", "react": "^18.x", "react-dom": "^18.x", diff --git a/vite.config.ts b/vite.config.ts --- a/vite.config.ts +++ b/vite.config.ts @@ -1,6 +1,7 @@ import { defineConfig } from 'electron-vite'; import react from '@vitejs/plugin-react'; import path from 'path'; +import fs from 'node:fs'; import { createRequire } from 'module'; const _require = createRequire(import.meta.url); @@ -44,12 +45,49 @@ // ------------------------------------------------------------------ renderer: { // Serve the pyodide runtime files as static assets so Vite does NOT - // transform them. importScripts() in a classic worker cannot load - // ES modules; Vite's HMR injection turns .js files into ESM, breaking - // the worker. Files in publicDir are served verbatim at the root URL: - // /pyodide/pyodide.js, /pyodide/pyodide.asm.js, etc. - publicDir: path.resolve(__dirname, 'src/renderer/utils/pyodide/src'), + // transform them. Files in publicDir are served verbatim at the root URL: + // /pyodide/pyodide.mjs, /pyodide/pyodide.asm.js, /packages/*.whl, etc. + publicDir: path.resolve(__dirname, 'src/renderer/utils/webworker/src'), plugins: [ + // Serve pyodide runtime and package .whl files directly from the filesystem + // before Vite's SPA fallback can intercept them. publicDir alone is not + // reliable — Vite's historyApiFallback returns index.html for fetch() + // requests to these paths in dev mode. + { + name: 'serve-pyodide-assets', + configureServer(server) { + const staticDir = path.resolve( + __dirname, + 'src/renderer/utils/webworker/src' + ); + const contentTypes: Record = { + '.json': 'application/json', + '.whl': 'application/zip', + '.zip': 'application/zip', + '.wasm': 'application/wasm', + '.js': 'application/javascript', + '.mjs': 'application/javascript', + }; + server.middlewares.use((req, res, next) => { + const url = req.url ?? ''; + if (url.startsWith('/pyodide/') || url.startsWith('/packages/')) { + console.log('[serve-pyodide-assets] intercepted:', url); + const filePath = path.join(staticDir, url.split('?')[0]); + if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) { + const ext = path.extname(filePath).toLowerCase(); + res.setHeader( + 'Content-Type', + contentTypes[ext] ?? 'application/octet-stream' + ); + res.setHeader('Cache-Control', 'no-cache'); + fs.createReadStream(filePath).pipe(res); + return; + } + } + next(); + }); + }, + }, react({ jsxRuntime: 'classic', // React 16 does not ship react/jsx-runtime babel: { @@ -77,6 +115,13 @@ }, optimizeDeps: { include: ['@neurosity/pipes'], + // Prevent Vite from pre-bundling pyodide. In dev mode it will be served + // raw from node_modules via /@fs/, which is what pyodide.mjs expects. + exclude: ['pyodide'], + }, + worker: { + // ES module workers are required for the CDN import in webworker.js. + format: 'es', }, build: { rollupOptions: { diff --git a/.llms/CLAUDE.md b/.llms/CLAUDE.md --- a/.llms/CLAUDE.md +++ b/.llms/CLAUDE.md @@ -15,8 +15,8 @@ - **Bundler**: electron-vite / Vite - **State**: Redux Toolkit + redux-observable (RxJS epics) - **Language**: TypeScript (strict) -- **Styling**: Semantic UI React + SCSS -- **Testing**: Jest +- **Styling**: Tailwind CSS + shadcn/ui (components in `src/renderer/components/ui/`) +- **Testing**: Vitest - **Linting**: ESLint + Prettier (single quotes, ES5 trailing commas) ## Key Directories @@ -24,13 +24,13 @@ - `src/renderer/` — React renderer process - `src/preload/` — Electron preload scripts - `src/renderer/experiments/` — Lab.js experiment files -- `src/renderer/utils/pyodide/` — Pyodide WASM Python runtime +- `src/renderer/utils/webworker/` — Pyodide WASM Python runtime ## Dev Workflow ```bash npm run dev # Start dev server (patches deps first) npm run build # Build all processes -npm test # Run Jest tests +npm test # Run Vitest tests npm run typecheck # TypeScript check (no emit) npm run lint # ESLint npm run lint-fix # ESLint + Prettier auto-fix @@ -45,7 +45,7 @@ - Keep Electron main/renderer separation strict — use preload IPC bridges ## Out of Scope -- Do not modify `src/renderer/utils/pyodide/src/` directly; it is managed by `InstallPyodide.js` +- Do not modify `src/renderer/utils/webworker/src/` directly; it is managed by `internals/scripts/InstallPyodide.mjs` (Pyodide runtime) and `internals/scripts/InstallMNE.mjs` (scientific packages) - Do not alter `electron-builder` publish config without confirming release intent ## LLM Context diff --git a/.llms/learnings.md b/.llms/learnings.md --- a/.llms/learnings.md +++ b/.llms/learnings.md @@ -21,6 +21,45 @@ - **Background gradient** used on all main screens: `bg-gradient-to-b from-[#f9f9f9] to-[#f0f0ff]` - **`@radix-ui/react-select`** is installed for the shadcn Select component +## Pyodide Asset Serving — Vite SPA Fallback Problem + +Vite's `historyApiFallback` returns `index.html` for **all** `fetch()` requests from web workers, including `/@fs/` and `publicDir` paths. This breaks Pyodide's package loading entirely. + +**Solution (two-part):** +1. A custom Vite middleware in `vite.config.ts` intercepts `/pyodide/` and `/packages/` requests before the SPA fallback and serves them directly from `src/renderer/utils/webworker/src/`. +2. An Electron `http` server on **port 17173** (started in `src/main/index.ts`) serves the same directory. Web workers use `http://127.0.0.1:17173` as `PYODIDE_ASSET_BASE`. This is the authoritative path — web worker `fetch()` calls bypass Vite entirely. + +Port 17173 is hardcoded in both `src/main/index.ts` and `src/renderer/utils/webworker/webworker.js` and in the CSP (`src/renderer/index.html`). + +**Other Pyodide loading gotchas:** +- `pyodide.mjs` must be loaded via dynamic `import()` (not `fetch()`), using a `?url` Vite import — `import()` bypasses the SPA fallback, `fetch()` does not +- The lock file is embedded via `?raw` and wrapped in a `Blob` + `createObjectURL` to avoid an HTTP fetch +- Use `packageBaseUrl` (not `indexURL`) to tell Pyodide where to find `.whl` files; `indexURL` is for WASM/stdlib +- `checkIntegrity: false` is required — SHA256 hashes in the npm lock file don't match CDN-downloaded wheels +- Workers must be created with `type: 'module'` (Pyodide 0.26+ ships `pyodide.mjs` as ESM) +- `optimizeDeps.exclude: ['pyodide']` in `vite.config.ts` prevents Vite from pre-bundling it + +## Pyodide Offline Package Installation (InstallMNE.mjs) + +`internals/scripts/InstallMNE.mjs` runs on `postinstall` and downloads two sets of packages: +- **Pyodide binary packages** (numpy, scipy, matplotlib, pandas + transitive deps) from the Pyodide CDN → `src/renderer/utils/webworker/src/pyodide/` +- **Pure-Python packages** (mne, pooch, tqdm, platformdirs) from PyPI → `src/renderer/utils/webworker/src/packages/` + +A `manifest.json` is written to `packages/` so `webworker.js` knows the exact `.whl` filenames to pass to `micropip.install()`. + +The CDN version is derived from `node_modules/pyodide/package.json` — **not** from `pyodide-lock.json`'s `info.version`, which may be a dev label like `0.28.0.dev0`. + +**Packages that must be listed explicitly** (not reachable from matplotlib/scipy/pandas deps in the lock file, but required at runtime): +- `jinja2` + `markupsafe` — used by matplotlib templates and MNE HTML reports +- `decorator` — MNE core dep +- `requests` (+ `certifi`, `charset-normalizer`, `idna`, `urllib3`) — pulled in by `pooch` at MNE import time + +**`micropip.install()` from JS accepts a JS array directly** — as of Pyodide 0.29.x, micropip handles the `JsProxy` conversion internally. `pyodide.toPy()` is not needed. + +**WebAgg backend does not work in web workers** — WebAgg tries to access `js.document` to inject CSS/JS into the DOM on first import, which throws `ImportError: cannot import name 'document' from 'js'` in a worker context. Use `agg` instead. Set it via `os.environ["MPLBACKEND"] = "agg"` before any matplotlib import. `fig.savefig()` works with `agg` and is the correct way to get plot images back to the renderer. + +**Plot result routing pattern** — `worker.postMessage()` is fire-and-forget (returns `undefined`). Plot epics should use `tap()` to fire the worker message and `mergeMap(() => EMPTY)` to emit nothing. Results come back asynchronously on the worker `message` event. Add a `plotKey` field to each worker message; the worker echoes it back; `pyodideMessageEpic` switches on `plotKey` to dispatch `SetTopoPlot`/`SetPSDPlot`/`SetERPPlot` with a `{ 'image/png': base64string }` MIME bundle. `PyodidePlotWidget` renders this via `@nteract/transforms`. + ## Pre-existing TypeScript errors (do not treat as regressions) - `src/renderer/epics/experimentEpics.ts` (lines 170, 205) — RxJS operator type mismatch diff --git a/docs/pyodide-in-electron-vite.md b/docs/pyodide-in-electron-vite.md new file mode 100644 --- /dev/null +++ b/docs/pyodide-in-electron-vite.md @@ -0,0 +1,311 @@ +# Pyodide in Electron + Vite: What We Learned + +This document captures everything we learned getting Pyodide (Python-in-WASM) running reliably inside an Electron + electron-vite app. It is intended as a reference for anyone maintaining or upgrading the Pyodide integration. + +--- + +## The Core Problem: Vite's SPA Fallback + +Vite's dev server runs a `historyApiFallback` middleware that returns `index.html` for **every** `fetch()` request it doesn't recognise — including `/@fs/` paths, `publicDir` paths, and anything from a web worker. This completely breaks Pyodide's package loader, which `fetch()`es `.whl` files at runtime. + +This is not an obvious failure — Pyodide may partially initialise and then hang or throw cryptic errors when it tries to load packages. + +### Solution: Serve Pyodide Assets Out-of-Band + +We use two complementary mechanisms: + +**1. Custom Vite middleware (dev only)** + +In `vite.config.ts`, a plugin intercepts requests to `/pyodide/` and `/packages/` before the SPA fallback runs and streams the files directly from `src/renderer/utils/webworker/src/`: + +```ts +server.middlewares.use((req, res, next) => { + const url = req.url ?? ''; + if (url.startsWith('/pyodide/') || url.startsWith('/packages/')) { + const filePath = path.join(staticDir, url.split('?')[0]); + if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) { + res.setHeader('Content-Type', contentTypes[ext] ?? 'application/octet-stream'); + fs.createReadStream(filePath).pipe(res); + return; + } + } + next(); +}); +``` + +**2. Electron local HTTP server on port 17173 (dev + prod)** + +Web workers cannot use Vite's dev server at all — `fetch()` from a worker always hits the SPA fallback. The main process (`src/main/index.ts`) starts a plain Node.js `http` server at `http://127.0.0.1:17173` that serves `src/renderer/utils/webworker/src/` (dev) or `resources/webworker/src/` (prod). + +The web worker (`webworker.js`) uses this as its `PYODIDE_ASSET_BASE`: + +```js +const PYODIDE_ASSET_BASE = 'http://127.0.0.1:17173'; +``` + +Port 17173 is hardcoded in three places that must stay in sync: +- `src/main/index.ts` — server listen port +- `src/renderer/utils/webworker/webworker.js` — `PYODIDE_ASSET_BASE` +- `src/renderer/index.html` — CSP `connect-src` directive + +--- + +## Loading pyodide.mjs + +Pyodide 0.26+ ships as an ES module (`pyodide.mjs`). You cannot `fetch()` it — Vite would intercept it. Instead, import it with Vite's `?url` suffix and use dynamic `import()`: + +```js +import pyodideMjsUrl from 'pyodide/pyodide.mjs?url'; +// ... +const { loadPyodide } = await import(/* @vite-ignore */ pyodideMjsUrl); +``` + +`import()` bypasses Vite's SPA fallback; `fetch()` does not. + +Also required in `vite.config.ts`: + +```ts +optimizeDeps: { + exclude: ['pyodide'], // prevent Vite from pre-bundling it +}, +worker: { + format: 'es', // ES module workers required for pyodide.mjs +}, +``` + +And workers must be created with `type: 'module'`: + +```ts +new Worker(new URL('./webworker.js', import.meta.url), { type: 'module' }); +``` + +--- + +## Loading the Lock File Without a Fetch + +`loadPyodide` needs the lock file to resolve package names to filenames. Fetching it would hit Vite's SPA fallback. Instead, embed it at build time with Vite's `?raw` suffix and wrap it in a blob URL: + +```js +import lockFileRaw from 'pyodide/pyodide-lock.json?raw'; + +const lockBlob = new Blob([lockFileRaw], { type: 'application/json' }); +const lockFileURL = URL.createObjectURL(lockBlob); +const pyodide = await loadPyodide({ lockFileURL, packageBaseUrl }); +URL.revokeObjectURL(lockFileURL); +``` + +--- + +## packageBaseUrl vs indexURL + +These are easy to confuse: + +| Option | Purpose | +|--------|---------| +| `indexURL` | Where Pyodide looks for its **runtime** files (WASM, stdlib). Already resolved from `node_modules` via `import.meta.url`. Do not override. | +| `packageBaseUrl` | Where `loadPackage()` fetches **package `.whl` files**. Set this to `http://127.0.0.1:17173/pyodide/`. | + +--- + +## Package Integrity Checks + +```js +await pyodide.loadPackage(['numpy', 'scipy', ...], { checkIntegrity: false }); +``` + +`checkIntegrity: false` is required. The SHA-256 hashes in the npm package's `pyodide-lock.json` are computed against the CDN files, but we serve locally-downloaded copies that may differ (e.g. re-compressed). Integrity checks will fail without this flag. + +--- + +## micropip.install() from JavaScript + +`micropip` is a Python object loaded via `pyodide.pyimport()`. Passing a JavaScript array directly works fine in Pyodide 0.29.x — micropip handles the `JsProxy` conversion internally: + +```js +const micropip = pyodide.pyimport('micropip'); +await micropip.install(whlUrls); // JS array works directly +``` + +> Note: older guidance suggested wrapping with `pyodide.toPy(whlUrls)` — this is not necessary as of 0.29.x. + +--- + +## Offline Package Installation (InstallMNE.mjs) + +`internals/scripts/InstallMNE.mjs` runs on `postinstall` and pre-downloads all packages so the app works offline. + +### Part 1 — Pyodide Binary Packages (from Pyodide CDN) + +These are compiled packages bundled with Pyodide. The script reads `pyodide-lock.json`, recursively resolves transitive dependencies of the root packages, and downloads each `.whl` into `src/renderer/utils/webworker/src/pyodide/`. + +**Derive the CDN version from `node_modules/pyodide/package.json`**, not from `pyodide-lock.json`'s `info.version` field — that field may be a dev label like `0.28.0.dev0` and will produce a broken CDN URL. + +Current root packages and why each is listed explicitly: + +| Package | Reason | +|---------|--------| +| `numpy`, `scipy`, `matplotlib`, `pandas` | Core scientific stack | +| `micropip` | Needed to install pure-Python packages at worker startup | +| `pillow` | Used by matplotlib and MNE; loaded at runtime by `loadPackage()` | +| `jinja2` | MNE dep; **not** listed in matplotlib's lock-file `depends` array despite being a runtime requirement | +| `decorator` | MNE core dep; not reachable from the scientific stack in the lock file | +| `requests` | Pulled in by `pooch` at MNE import time; brings in `certifi`, `charset-normalizer`, `idna`, `urllib3` transitively | + +> **Gotcha:** The `depends` arrays in `pyodide-lock.json` are incomplete. Several packages that matplotlib, scipy, or MNE require at runtime are not listed as dependencies and will not be downloaded unless added explicitly as roots. + +### Part 2 — Pure-Python Packages (from PyPI) + +Packages not bundled with Pyodide must be downloaded as `py3-none-any` wheels from PyPI. They are stored in `src/renderer/utils/webworker/src/packages/` and a `manifest.json` is written so the worker knows the exact filenames. + +Current PyPI packages: `mne`, `pooch`, `tqdm`, `platformdirs`, `lazy-loader` + +`lazy-loader` is a core MNE dependency that does not appear in the Pyodide lock at all. + +--- + +## Plot Pipeline + +### matplotlib Backend in Web Workers + +Use `agg`, not `webagg`. Set it before any Python imports run: + +```js +await pyodide.runPythonAsync('import os; os.environ["MPLBACKEND"] = "agg"'); +``` + +WebAgg (`webagg`) fails in web workers because it tries to inject CSS via `js.document` during initialisation — and `js.document` does not exist in worker scope. The error looks like: + +``` +ImportError: cannot import name 'document' from 'js' +``` + +`agg` is a non-interactive raster backend that writes to a buffer, which is exactly what we need. + +--- + +### plotKey Correlation Pattern (Fire-and-Forget Messaging) + +`worker.postMessage()` returns `undefined` — there is no return channel. Redux-Observable plot load epics cannot receive the worker's result directly. + +**Solution:** attach a `plotKey` string to every outgoing message; the worker echoes it back in the response object. `pyodideMessageEpic` routes by `plotKey` to the correct Redux action. + +```js +// webworker.js — echo plotKey back in every response +const { data, plotKey, ...context } = event.data; +self.postMessage({ results: await pyodide.runPythonAsync(data), plotKey }); +``` + +```ts +// pyodideMessageEpic — route by plotKey +switch (plotKey) { + case 'ready': return of(PyodideActions.SetWorkerReady()); + case 'topo': return of(PyodideActions.SetTopoPlot(mimeBundle)); + case 'psd': return of(PyodideActions.SetPSDPlot(mimeBundle)); + case 'erp': return of(PyodideActions.SetERPPlot(mimeBundle)); + default: return of(PyodideActions.ReceiveMessage(e.data)); +} +``` + +Plot load epics become fire-and-forget — they call `worker.postMessage()` as a side effect and emit nothing: + +```ts +// loadTopoEpic +action$.pipe( + filter(isActionOf(PyodideActions.LoadTopo)), + tap(() => plotTestPlot(state$.value.pyodide.worker!)), + mergeMap(() => EMPTY) +); +``` + +--- + +### Worker Readiness Gating + +`loadUtils` posts `plotKey: 'ready'` when `utils.py` finishes loading. This drives an `isWorkerReady` flag in Redux state that gates any UI that depends on Python being initialised. + +```ts +export const loadUtils = async (worker: Worker) => + worker.postMessage({ data: utilsPy, plotKey: 'ready' }); +``` + +`pyodideMessageEpic` dispatches `PyodideActions.SetWorkerReady()` on receiving `plotKey === 'ready'`. + +--- + +### SVG Output from matplotlib + +Produce SVG in Python — no base64 encoding needed: + +```python +import io +import matplotlib.pyplot as plt + +_fig, _ax = plt.subplots() +_ax.plot([1, 2, 3, 4], [1, 4, 2, 3]) +_buf = io.BytesIO() +_fig.savefig(_buf, format="svg", bbox_inches="tight") +plt.close(_fig) +_buf.getvalue().decode() # SVG string is the Python return value +``` + +The SVG string flows through `pyodide.runPythonAsync()` → worker `postMessage` → Redux state as `{ 'image/svg+xml': string }`. + +--- + +### Rendering SVG Safely in the Renderer + +Use a data URI on an `` tag — sandboxed, no script execution: + +```tsx + +``` + +Prefer this over `dangerouslySetInnerHTML` — inline SVG executes `