diff --git a/.llms/learnings.md b/.llms/learnings.md index 30deab0..821a23a 100644 --- a/.llms/learnings.md +++ b/.llms/learnings.md @@ -21,6 +21,34 @@ The app uses shadcn/ui + Tailwind CSS. CSS modules have been fully removed. Key - **Background gradient** used on all main screens: `bg-gradient-to-b from-[#f9f9f9] to-[#f0f0ff]` - **`@radix-ui/react-select`** is installed for the shadcn Select component +## Pyodide Asset Serving — Vite SPA Fallback Problem + +Vite's `historyApiFallback` returns `index.html` for **all** `fetch()` requests from web workers, including `/@fs/` and `publicDir` paths. This breaks Pyodide's package loading entirely. + +**Solution (two-part):** +1. A custom Vite middleware in `vite.config.ts` intercepts `/pyodide/` and `/packages/` requests before the SPA fallback and serves them directly from `src/renderer/utils/webworker/src/`. +2. An Electron `http` server on **port 17173** (started in `src/main/index.ts`) serves the same directory. Web workers use `http://127.0.0.1:17173` as `PYODIDE_ASSET_BASE`. This is the authoritative path — web worker `fetch()` calls bypass Vite entirely. + +Port 17173 is hardcoded in both `src/main/index.ts` and `src/renderer/utils/webworker/webworker.js` and in the CSP (`src/renderer/index.html`). + +**Other Pyodide loading gotchas:** +- `pyodide.mjs` must be loaded via dynamic `import()` (not `fetch()`), using a `?url` Vite import — `import()` bypasses the SPA fallback, `fetch()` does not +- The lock file is embedded via `?raw` and wrapped in a `Blob` + `createObjectURL` to avoid an HTTP fetch +- Use `packageBaseUrl` (not `indexURL`) to tell Pyodide where to find `.whl` files; `indexURL` is for WASM/stdlib +- `checkIntegrity: false` is required — SHA256 hashes in the npm lock file don't match CDN-downloaded wheels +- Workers must be created with `type: 'module'` (Pyodide 0.26+ ships `pyodide.mjs` as ESM) +- `optimizeDeps.exclude: ['pyodide']` in `vite.config.ts` prevents Vite from pre-bundling it + +## Pyodide Offline Package Installation (InstallMNE.mjs) + +`internals/scripts/InstallMNE.mjs` runs on `postinstall` and downloads two sets of packages: +- **Pyodide binary packages** (numpy, scipy, matplotlib, pandas + transitive deps) from the Pyodide CDN → `src/renderer/utils/webworker/src/pyodide/` +- **Pure-Python packages** (mne, pooch, tqdm, platformdirs) from PyPI → `src/renderer/utils/webworker/src/packages/` + +A `manifest.json` is written to `packages/` so `webworker.js` knows the exact `.whl` filenames to pass to `micropip.install()`. + +The CDN version is derived from `node_modules/pyodide/package.json` — **not** from `pyodide-lock.json`'s `info.version`, which may be a dev label like `0.28.0.dev0`. + ## Pre-existing TypeScript errors (do not treat as regressions) - `src/renderer/epics/experimentEpics.ts` (lines 170, 205) — RxJS operator type mismatch diff --git a/internals/scripts/InstallMNE.mjs b/internals/scripts/InstallMNE.mjs index 37e9870..dd87fc7 100644 --- a/internals/scripts/InstallMNE.mjs +++ b/internals/scripts/InstallMNE.mjs @@ -39,7 +39,7 @@ const MANIFEST_FILE = path.join(PACKAGES_DIR, 'manifest.json'); // Root packages whose full transitive dependency tree we need from Pyodide CDN // --------------------------------------------------------------------------- -const PYODIDE_ROOT_PACKAGES = ['numpy', 'scipy', 'matplotlib', 'pandas']; +const PYODIDE_ROOT_PACKAGES = ['numpy', 'scipy', 'matplotlib', 'pandas', 'micropip']; // --------------------------------------------------------------------------- // Pure-Python packages to download from PyPI (not bundled with Pyodide) diff --git a/src/main/index.ts b/src/main/index.ts index 648f964..6320e05 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -8,6 +8,7 @@ import { app, BrowserWindow, ipcMain, dialog, shell, session } from 'electron'; import path from 'path'; import fs from 'fs'; +import http from 'http'; import os from 'os'; import Papa from 'papaparse'; import mkdirp from 'mkdirp'; @@ -23,6 +24,52 @@ app.commandLine.appendSwitch( 'true' ); +// Port for the local pyodide asset server (serves whl files to web workers, +// bypassing Vite's dev server which returns HTML for all fetch() requests). +const PYODIDE_ASSET_PORT = 17173; + +const PYODIDE_CONTENT_TYPES: Record = { + '.json': 'application/json', + '.whl': 'application/zip', + '.zip': 'application/zip', + '.wasm': 'application/wasm', + '.js': 'application/javascript', + '.mjs': 'application/javascript', +}; + +function startPyodideAssetServer(rootDir: string): void { + const server = http.createServer((req, res) => { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS'); + if (req.method === 'OPTIONS') { res.writeHead(204); res.end(); return; } + + const urlPath = (req.url || '').split('?')[0]; + const filePath = path.join(rootDir, urlPath); + const ext = path.extname(filePath).toLowerCase(); + + fs.stat(filePath, (statErr, stat) => { + if (statErr || !stat.isFile()) { + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end(`Not found: ${urlPath}`); + return; + } + res.setHeader('Content-Type', PYODIDE_CONTENT_TYPES[ext] || 'application/octet-stream'); + res.setHeader('Content-Length', stat.size); + res.setHeader('Cache-Control', 'no-cache'); + res.writeHead(200); + fs.createReadStream(filePath).pipe(res); + }); + }); + + server.listen(PYODIDE_ASSET_PORT, '127.0.0.1', () => { + console.log(`[main] Pyodide asset server: http://127.0.0.1:${PYODIDE_ASSET_PORT}`); + }); + + server.on('error', (err: NodeJS.ErrnoException) => { + console.error('[main] Pyodide asset server error:', err.message); + }); +} + export default class AppUpdater { constructor() { log.transports.file.level = 'info'; @@ -453,6 +500,18 @@ app.on('window-all-closed', () => { }); app.whenReady().then(async () => { + // Serve pyodide assets (whl files, runtime files) via a local HTTP server so + // web workers can fetch() them without hitting Vite's SPA fallback, which + // returns HTML for ALL fetch() requests regardless of path. + // Port 17173 is hardcoded and matched in webworker.js. + // In dev: files are in src/renderer/utils/webworker/src/ + // In prod: files are in resources/webworker/src/ (via extraResources) + const pyodideRoot = is.dev + ? path.join(app.getAppPath(), 'src/renderer/utils/webworker/src') + : path.join(process.resourcesPath, 'webworker/src'); + + startPyodideAssetServer(pyodideRoot); + // Enable F12 devtools shortcut and Ctrl+R reload in dev, disable in prod app.on('browser-window-created', (_, window) => { optimizer.watchWindowShortcuts(window); diff --git a/src/renderer/index.html b/src/renderer/index.html index ff70f86..3027434 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -4,7 +4,7 @@ BrainWaves diff --git a/src/renderer/utils/webworker/index.ts b/src/renderer/utils/webworker/index.ts index c8e965b..27f9825 100644 --- a/src/renderer/utils/webworker/index.ts +++ b/src/renderer/utils/webworker/index.ts @@ -127,7 +127,7 @@ export const plotTestPlot = async (worker: Worker | null) => { } return worker.postMessage({ // data: `import matplotlib.pyplot as plt; fig= plt.plot([1,2,3,4])`, - data: `[1,2,3,4]` + data: `sum([1,2,3,4])` }); }; diff --git a/src/renderer/utils/webworker/webworker.js b/src/renderer/utils/webworker/webworker.js index 6ca003c..f309d7e 100644 --- a/src/renderer/utils/webworker/webworker.js +++ b/src/renderer/utils/webworker/webworker.js @@ -3,24 +3,72 @@ * * Loading strategy * ---------------- - * Use Vite's `?url` suffix on 'pyodide/pyodide.mjs' to get the resolved file URL - * at build/dev time (/@fs/... in dev, an asset URL in prod), then dynamically - * import from that URL. This bypasses Vite's SPA fallback and lets pyodide.mjs - * resolve all sibling assets (pyodide.asm.wasm, pyodide-lock.json, etc.) via - * import.meta.url — no CDN required. + * pyodide.mjs is imported via Vite's `?url` suffix, which gives us an + * /@fs/... URL in dev. We use dynamic import() from that URL — this works + * because import() bypasses Vite's SPA fallback (only fetch() is affected). * - * Production builds use the files copied to publicDir by InstallPyodide.mjs. + * The lock file is embedded via `?raw` to avoid an HTTP fetch that Vite + * intercepts. A blob URL is created from the embedded JSON so loadPyodide + * can "fetch" it from memory. + * + * Package whl files (numpy, scipy, etc.) live in + * src/renderer/utils/webworker/src/pyodide/ and are served by a tiny Node.js + * HTTP server on port 17173 started in the Electron main process. This bypasses + * Vite's dev server, which returns HTML (SPA fallback) for ALL fetch() requests + * from web workers, including /@fs/ and publicDir paths. + * + * MNE and its pure-Python deps are installed via micropip from local .whl + * files served by the same pyodide-asset:// protocol under /packages/. */ -// ?url tells Vite to resolve the path and return a URL string rather than bundling -// the module. In dev mode this is a /@fs/ URL (bypasses SPA fallback); in prod it -// is an asset URL. We then dynamically import from that URL so pyodide.mjs can -// resolve all its sibling assets (pyodide.asm.wasm, etc.) via import.meta.url. +// ?url → Vite resolves to /@fs/... in dev; asset URL in prod. +// ?raw → Vite embeds file content as a string (no HTTP fetch at runtime). import pyodideMjsUrl from 'pyodide/pyodide.mjs?url'; +import lockFileRaw from 'pyodide/pyodide-lock.json?raw'; + +// A tiny Node.js HTTP server on port 17173 (started in the Electron main +// process) serves pyodide assets from src/renderer/utils/webworker/src/. +// This bypasses Vite's dev server, which returns index.html (SPA fallback) +// for ALL fetch() requests from web workers, including /@fs/ and publicDir paths. +const PYODIDE_ASSET_BASE = 'http://127.0.0.1:17173'; const pyodideReadyPromise = (async () => { const { loadPyodide } = await import(/* @vite-ignore */ pyodideMjsUrl); - return loadPyodide(); + + // Wrap the embedded lock file in a blob URL so loadPyodide can "fetch" it + // without making an HTTP request that Vite would intercept and transform. + const lockBlob = new Blob([lockFileRaw], { type: 'application/json' }); + const lockFileURL = URL.createObjectURL(lockBlob); + + // packageBaseUrl tells pyodide's PackageManager where to fetch .whl files. + // This is the correct option — NOT indexURL, which is for the runtime files + // (WASM, stdlib) that are already loaded via import.meta.url from node_modules. + const packageBaseUrl = `${PYODIDE_ASSET_BASE}/pyodide/`; + + const pyodide = await loadPyodide({ lockFileURL, packageBaseUrl }); + URL.revokeObjectURL(lockFileURL); + + // Load scientific packages from local whl files via the asset server. + // checkIntegrity: false skips SHA256 verification — hashes in the npm lock + // file may not match the CDN-downloaded whl files we're actually serving. + await pyodide.loadPackage( + ['numpy', 'scipy', 'matplotlib', 'pandas', 'pillow'], + { checkIntegrity: false } + ); + + // Load micropip so we can install MNE and its pure-Python deps. + await pyodide.loadPackage('micropip', { checkIntegrity: false }); + const micropip = pyodide.pyimport('micropip'); + + // MNE + pure-Python deps are served from /packages/ via pyodide-asset://. + const manifestUrl = `${PYODIDE_ASSET_BASE}/packages/manifest.json`; + const manifest = await fetch(manifestUrl).then((r) => r.json()); + const whlUrls = Object.values(manifest).map( + ({ filename }) => `${PYODIDE_ASSET_BASE}/packages/${filename}` + ); + await micropip.install(whlUrls); + + return pyodide; })(); self.onmessage = async (event) => { diff --git a/vite.config.ts b/vite.config.ts index 4303eff..24f01c7 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,6 +1,7 @@ import { defineConfig } from 'electron-vite'; import react from '@vitejs/plugin-react'; import path from 'path'; +import fs from 'node:fs'; import { createRequire } from 'module'; const _require = createRequire(import.meta.url); @@ -48,6 +49,45 @@ export default defineConfig({ // /pyodide/pyodide.mjs, /pyodide/pyodide.asm.js, /packages/*.whl, etc. publicDir: path.resolve(__dirname, 'src/renderer/utils/webworker/src'), plugins: [ + // Serve pyodide runtime and package .whl files directly from the filesystem + // before Vite's SPA fallback can intercept them. publicDir alone is not + // reliable — Vite's historyApiFallback returns index.html for fetch() + // requests to these paths in dev mode. + { + name: 'serve-pyodide-assets', + configureServer(server) { + const staticDir = path.resolve( + __dirname, + 'src/renderer/utils/webworker/src' + ); + const contentTypes: Record = { + '.json': 'application/json', + '.whl': 'application/zip', + '.zip': 'application/zip', + '.wasm': 'application/wasm', + '.js': 'application/javascript', + '.mjs': 'application/javascript', + }; + server.middlewares.use((req, res, next) => { + const url = req.url ?? ''; + if (url.startsWith('/pyodide/') || url.startsWith('/packages/')) { + console.log('[serve-pyodide-assets] intercepted:', url); + const filePath = path.join(staticDir, url.split('?')[0]); + if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) { + const ext = path.extname(filePath).toLowerCase(); + res.setHeader( + 'Content-Type', + contentTypes[ext] ?? 'application/octet-stream' + ); + res.setHeader('Cache-Control', 'no-cache'); + fs.createReadStream(filePath).pipe(res); + return; + } + } + next(); + }); + }, + }, react({ jsxRuntime: 'classic', // React 16 does not ship react/jsx-runtime babel: {