diff --git a/packages/vitest/src/node/cache/fsModuleCache.ts b/packages/vitest/src/node/cache/fsModuleCache.ts index ac1e1dd4a..7bdb951ab 100644 --- a/packages/vitest/src/node/cache/fsModuleCache.ts +++ b/packages/vitest/src/node/cache/fsModuleCache.ts @@ -1,4 +1,5 @@ -import type { DevEnvironment, FetchResult } from 'vite' +import type { DevEnvironment } from 'vite' +import type { ModuleType, VitestFetchResult } from '../../types/general' import type { Vitest } from '../core' import type { ResolvedConfig } from '../types/config' import fs, { existsSync, mkdirSync, readFileSync } from 'node:fs' @@ -33,7 +34,7 @@ export class FileSystemModuleCache { private rootCache: string private metadataFilePath: string - private version = '1.0.0-beta.4' + private version = '1.0.0-beta.5' private fsCacheRoots = new WeakMap() private fsEnvironmentHashMap = new WeakMap() private fsCacheKeyGenerators = new Set() @@ -110,12 +111,13 @@ export class FileSystemModuleCache { code, importedUrls: meta.importedUrls, mappings: meta.mappings, + moduleType: meta.moduleType, } } - async saveCachedModule( + async saveCachedModule( cachedFilePath: string, - fetchResult: T, + fetchResult: VitestFetchResult, importedUrls: string[] = [], mappings: boolean = false, ): Promise { @@ -126,6 +128,7 @@ export class FileSystemModuleCache { url: fetchResult.url, importedUrls, mappings, + moduleType: fetchResult.moduleType, } satisfies Omit debugFs?.(`${c.yellow('[write]')} ${fetchResult.id} is cached in ${cachedFilePath}`) await atomicWriteFile(cachedFilePath, `${fetchResult.code}${cacheComment}${this.toBase64(result)}`) @@ -211,6 +214,7 @@ export class FileSystemModuleCache { mode: config.mode, consumer: config.consumer, resolve: config.resolve, + injectCjsGlobal: vitestConfig.injectCjsGlobals, // plugins can have different options, so this is not the best key, // but we cannot access the options because there is no standard API for it plugins: config.plugins @@ -366,6 +370,7 @@ export interface CachedInlineModuleMeta { code: string mappings: boolean importedUrls: string[] + moduleType?: ModuleType } /** diff --git a/packages/vitest/src/node/environments/fetchModule.ts b/packages/vitest/src/node/environments/fetchModule.ts index cd9d6818a..cb0975721 100644 --- a/packages/vitest/src/node/environments/fetchModule.ts +++ b/packages/vitest/src/node/environments/fetchModule.ts @@ -1,7 +1,7 @@ import type { Span } from '@opentelemetry/api' import type { DevEnvironment, EnvironmentModuleNode, Rollup, TransformResult } from 'vite' import type { FetchFunctionOptions, FetchResult } from 'vite/module-runner' -import type { FetchCachedFileSystemResult, VitestFetchResult } from '../../types/general' +import type { FetchCachedFileSystemResult, ModuleType, VitestFetchResult } from '../../types/general' import type { OTELCarrier, Traces } from '../../utils/traces' import type { FileSystemModuleCache } from '../cache/fsModuleCache' import type { VitestResolver } from '../resolver' @@ -142,7 +142,14 @@ class ModuleFetcher { const map = moduleGraphModule.transformResult?.map const mappings = map && !('version' in map) && map.mappings === '' - return this.cacheResult(result, cachePath, importedUrls, !!mappings) + const cachedResult = await this.cacheResult(result, cachePath, importedUrls, !!mappings) + // remember where the code is stored on disk so that repeat fetches and the + // `fetchWarmModules` snapshot can point at it in this session already, not + // only after the cache is read back in the next one + if ('code' in result && moduleGraphModule.transformResult) { + moduleGraphModule.transformResult.__vitestTmp = cachePath + } + return cachedResult } // we need this for UI to be able to show a module graph @@ -239,13 +246,11 @@ class ModuleFetcher { tmp: moduleGraphModule.transformResult.__vitestTmp, url: moduleGraphModule.url, invalidate: false, - moduleType: this.detectModuleType - ? await detectModuleType( - moduleGraphModule.file, - moduleGraphModule.transformResult.code, - this.sourceLoader(moduleGraphModule.file), - ) - : undefined, + moduleType: await this.cachedModuleType( + moduleGraphModule.file, + moduleGraphModule.transformResult.code, + moduleGraphModule.transformResult, + ), } } @@ -264,11 +269,13 @@ class ModuleFetcher { if (!map && cachedModule.mappings) { map = { mappings: '' } } + const moduleType = cachedModule.moduleType moduleGraphModule.transformResult = { code: cachedModule.code, map, ssr: true, __vitestTmp: cachePath, + __vitestModuleType: moduleType, } // we populate the module graph to make the watch mode work because it relies on importers @@ -294,9 +301,7 @@ class ModuleFetcher { tmp: cachePath, url: cachedModule.url, invalidate: false, - moduleType: this.detectModuleType - ? await detectModuleType(cachedModule.file, cachedModule.code, this.sourceLoader(cachedModule.file)) - : undefined, + moduleType, } } @@ -318,8 +323,8 @@ class ModuleFetcher { ).catch(handleRollupError) const result: VitestFetchResult = processResultSource(environment, moduleRunnerModule) - if (this.detectModuleType && 'code' in result) { - result.moduleType = await detectModuleType(result.file, result.code, this.sourceLoader(result.file)) + if ('code' in result) { + result.moduleType = await this.cachedModuleType(result.file, result.code, moduleGraphModule.transformResult) } return result } @@ -331,6 +336,28 @@ class ModuleFetcher { return () => this.readFileConcurrently(file) } + // the module type is a pure function of the module, so detect it at most once + // and memoize the verdict on the transform result. repeat fetches, the on-disk + // cache (`cached`), and the `fetchWarmModules` snapshot all reuse it instead of + // re-detecting. a no-op unless `injectCjsGlobals` is disabled — otherwise every + // module receives the CJS globals and the type is irrelevant. + private async cachedModuleType( + file: string | null, + code: string, + transformResult: TransformResult | null | undefined, + ): Promise { + if (!this.detectModuleType) { + return undefined + } + const moduleType + = transformResult?.__vitestModuleType + ?? await detectModuleType(file, code, this.sourceLoader(file)) + if (transformResult) { + transformResult.__vitestModuleType = moduleType + } + return moduleType + } + private async cacheResult( result: FetchResult, cachePath: string, @@ -543,5 +570,6 @@ export function handleRollupError(e: unknown): never { declare module 'vite' { export interface TransformResult { __vitestTmp?: string + __vitestModuleType?: ModuleType } } diff --git a/packages/vitest/src/node/pool.ts b/packages/vitest/src/node/pool.ts index 781125776..4fe4f2d04 100644 --- a/packages/vitest/src/node/pool.ts +++ b/packages/vitest/src/node/pool.ts @@ -127,6 +127,17 @@ export function createPool(ctx: Vitest): ProcessPool { ...project.config.env, } + // V8 serializes compile-cached scripts without the source positions + // that precise coverage relies on, so the compile cache must stay off + // for the v8 provider (and custom providers, whose mechanism we can't + // assume) in workers and any process they spawn. istanbul instruments + // the source at transform time, so the cache is harmless there and the + // boot speedup is kept. + if (ctx.config.coverage.enabled && ctx.config.coverage.provider !== 'istanbul') { + delete env.NODE_COMPILE_CACHE + env.NODE_DISABLE_COMPILE_CACHE = '1' + } + // env are case-insensitive on Windows, but spawned processes don't support it if (isWindows) { for (const name in env) { diff --git a/packages/vitest/src/node/pools/rpc.ts b/packages/vitest/src/node/pools/rpc.ts index bb5bd4150..8bf2d8697 100644 --- a/packages/vitest/src/node/pools/rpc.ts +++ b/packages/vitest/src/node/pools/rpc.ts @@ -1,3 +1,5 @@ +import type { DevEnvironment, EnvironmentModuleNode, FetchResult } from 'vite' +import type { FetchCachedFileSystemResult } from '../../types/general' import type { RuntimeRPC } from '../../types/rpc' import type { TestProject } from '../project' import type { ResolveSnapshotPathHandlerContext } from '../types/config' @@ -14,6 +16,19 @@ interface MethodsOptions { collect?: boolean } +// externalize verdicts served during this session, shared with fresh workers +// via `fetchWarmModules`. Only verdicts for already-resolved urls are stored: +// an unresolved specifier (a runtime-variable dynamic import of a bare name) +// resolves through the requesting environment's plugin container, so its +// verdict is importer-specific and cannot be shared. +// Keyed by the DevEnvironment, not the server: a leading-slash url still +// resolves to its id through that environment's plugin container, so a plugin +// that resolves conditionally (e.g. on `this.environment`) can externalize the +// same url in one environment and inline it in another — sharing the verdict +// across environments would serve the wrong one. Per-environment keying also +// drops the verdicts on a server restart, since environments are recreated. +const warmExternals = new WeakMap>() + export function createMethodsRPC(project: TestProject, methodsOptions: MethodsOptions = {}): RuntimeRPC { const vitest = project.vitest const cacheFs = methodsOptions.cacheFs ?? false @@ -47,6 +62,16 @@ export function createMethodsRPC(project: TestProject, methodsOptions: MethodsOp const metadata = project.vitest.state.metadata[project.name] if ('externalize' in result) { metadata.externalized[url] = result.externalize + // builtins and network urls are already resolved inside the worker + // without a round-trip, only module externalizations are worth sharing + if (result.type === 'module' && url[0] === '/') { + let externals = warmExternals.get(environment) + if (!externals) { + externals = Object.create(null) as Record + warmExternals.set(environment, externals) + } + externals[url] = result + } } if ('tmp' in result) { metadata.tmps[url] = result.tmp @@ -56,6 +81,75 @@ export function createMethodsRPC(project: TestProject, methodsOptions: MethodsOp return result }) }, + async fetchWarmModules(environmentName, files) { + const environment = project.vite.environments[environmentName] + if (!environment) { + throw new Error(`The environment ${environmentName} was not defined in the Vite config.`) + } + + const warm: Record = Object.create(null) + + // walk the import graphs of the requested files instead of dumping the + // whole module graph — in large (watch) sessions the graph accumulates + // modules this worker will never load + const moduleGraph = environment.moduleGraph + const queue: EnvironmentModuleNode[] = [] + for (const file of [...files, ...project.config.setupFiles]) { + const nodes = moduleGraph.getModulesByFile(file) + if (nodes) { + queue.push(...nodes) + } + } + + const seen = new Set() + while (queue.length) { + const node = queue.pop()! + if (seen.has(node)) { + continue + } + seen.add(node) + queue.push(...node.importedModules) + + const transformResult = node.transformResult + if (!transformResult || node.id == null) { + continue + } + // the transformed code is already stored on disk either by the forks + // pool (`cacheFs`) or by `experimental.fsModuleCache` — the worker can + // read the file itself instead of fetching each module separately. + // invalidated modules lose `transformResult` and drop out automatically + const tmp = transformResult.__vitestTmp ?? (transformResult as { _vitest_tmp?: string })._vitest_tmp + if (typeof tmp !== 'string') { + continue + } + const entry: FetchCachedFileSystemResult = { + cached: true, + file: node.file, + id: node.id, + tmp, + url: node.url, + invalidate: false, + // the fetch that stored this module on disk also memoized its module + // type on the transform result (only when `injectCjsGlobals` is + // disabled); reuse it so the evaluator injects the CJS globals for the + // same modules it would on the direct-fetch path, no re-detection here + moduleType: transformResult.__vitestModuleType, + } + warm[node.url] = entry + if (node.id !== node.url) { + warm[node.id] = entry + } + } + + const externals = warmExternals.get(environment) + if (externals) { + for (const url in externals) { + warm[url] ??= externals[url] + } + } + + return warm + }, async resolve(id, importer, environmentName) { const environment = project.vite.environments[environmentName] if (!environment) { diff --git a/packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts b/packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts index 9c50be2d0..f9ae4d98c 100644 --- a/packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts +++ b/packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts @@ -1,5 +1,6 @@ import type vm from 'node:vm' -import type { EvaluatedModules } from 'vite/module-runner' +import type { EvaluatedModules, FetchResult } from 'vite/module-runner' +import type { FetchCachedFileSystemResult } from '../../types/general' import type { WorkerGlobalState } from '../../types/worker' import type { Traces } from '../../utils/traces' import type { ExternalModulesExecutor } from '../external-executor' @@ -54,6 +55,29 @@ export function startVitestModuleRunner(options: ContextModuleRunnerOptions): Vi } : undefined + // A fresh worker pays one strictly sequential `fetch` round-trip per module + // in its test files' import graphs, even when the server processed all of + // them already. Ask the server ONCE per run request for everything it has on + // disk and answer those fetches locally. A file change invalidates the module + // server-side, dropping it from the snapshot of every subsequent run request, + // which keeps reused (isolate: false) workers in sync; an edit DURING a run + // was racy before this fast path existed and stays racy with it — the + // scheduled rerun always sees the fresh transform. + let warmModules: Promise | null> | undefined + let warmModulesContext: unknown + + function fetchWarmModules() { + const workerState = state() + if (warmModulesContext !== workerState.ctx) { + warmModulesContext = workerState.ctx + warmModules = rpc() + .fetchWarmModules(environment(), workerState.ctx.files.map(file => file.filepath)) + // if the snapshot cannot be fetched, fall back to per-module fetches + .catch(() => null) + } + return warmModules! + } + const evaluator = options.evaluator || new VitestModuleEvaluator( vm, { @@ -144,6 +168,37 @@ export function startVitestModuleRunner(options: ContextModuleRunnerOptions): Vi return { cache: true } } + // only dependency fetches consult the snapshot: by the time the + // first dependency is requested, the entry file is transformed and + // its import graph is connected on the server, so the snapshot + // actually covers the file's transitive dependencies + if (importer != null) { + const warm = await fetchWarmModules() + // the null prototype is not preserved by the IPC serialization, so + // ids like "constructor" must not fall through to Object.prototype + const warmResult = warm && ( + Object.hasOwn(warm, id) + ? warm[id] + : Object.hasOwn(warm, rawId) + ? warm[rawId] + : undefined + ) + if (warmResult) { + if ('tmp' in warmResult) { + try { + const code = readFileSync(warmResult.tmp, 'utf-8') + return { code, ...warmResult } + } + catch { + // the tmp file is gone — fall back to a live fetch + } + } + else { + return warmResult + } + } + } + const otelCarrier = traces?.getContextCarrier() const result = await rpc().fetch( id, diff --git a/packages/vitest/src/types/rpc.ts b/packages/vitest/src/types/rpc.ts index e6eb37ef0..907cdeab0 100644 --- a/packages/vitest/src/types/rpc.ts +++ b/packages/vitest/src/types/rpc.ts @@ -13,6 +13,12 @@ export interface RuntimeRPC { otelCarrier?: OTELCarrier, ) => Promise resolve: (id: string, importer: string | undefined, environment: string) => Promise + /** + * Returns the modules of the given test files' import graphs that the server + * has already processed, so a fresh worker can load them from disk without + * paying a `fetch` round-trip per module. + */ + fetchWarmModules: (environment: string, files: string[]) => Promise> transform: (id: string) => Promise<{ code?: string }> onUserConsoleLog: (log: UserConsoleLog) => void diff --git a/packages/vitest/vitest.mjs b/packages/vitest/vitest.mjs index 02dd4714b..57c2f2ce6 100755 --- a/packages/vitest/vitest.mjs +++ b/packages/vitest/vitest.mjs @@ -1,2 +1,19 @@ #!/usr/bin/env node -import './dist/cli.js' +import * as module from 'node:module' + +// Enable Node's on-disk compile cache before importing the CLI so both the CLI +// graph and (via the inherited env variable) every spawned worker skip V8 +// recompilation of unchanged modules. `enableCompileCache()` only affects the +// current process — child processes pick the cache up from NODE_COMPILE_CACHE. +// Respects an explicit NODE_COMPILE_CACHE and NODE_DISABLE_COMPILE_CACHE; the +// API is not available before Node 22.8. +try { + const result = module.enableCompileCache?.() + if (result?.directory && !process.env.NODE_COMPILE_CACHE) { + process.env.NODE_COMPILE_CACHE = result.directory + } +} +catch {} + +// eslint-disable-next-line antfu/no-top-level-await -- the import must not be hoisted above `enableCompileCache` +await import('./dist/cli.js') diff --git a/test/e2e/test/config/injectCjsGlobals.test.ts b/test/e2e/test/config/injectCjsGlobals.test.ts index eca265b6b..93f384417 100644 --- a/test/e2e/test/config/injectCjsGlobals.test.ts +++ b/test/e2e/test/config/injectCjsGlobals.test.ts @@ -63,6 +63,50 @@ test.for(pools)('inlined ".cjs" modules keep the module scope when injectCjsGlob expect(exitCode).toBe(0) }) +test('cjs dep served from the warm-modules snapshot keeps the module scope when injectCjsGlobals is disabled', async () => { + // the `fetchWarmModules` fast path hands a module the server already + // transformed to a fresh worker without the per-module `fetch` that tags its + // `moduleType`. that tag is what tells the evaluator to inject the CommonJS + // scope when `injectCjsGlobals` is disabled, so the snapshot has to carry it. + // isolate + sequential files make the ordering deterministic: the first file + // transforms and caches the dependency (direct fetch), every later file reads + // it back from the snapshot — without the tag those files throw + // "require is not defined" while the first one still passes. + const structure: Record = { + 'package.json': '{ "type": "module" }', + 'cjs-dep.cjs': ts` + const path = require('node:path') + module.exports = { + answer: 42, + base: path.basename(__filename), + dir: typeof __dirname, + } + `, + } + for (const name of ['a', 'b', 'c', 'd']) { + structure[`${name}.test.js`] = ts` + import { expect, test } from 'vitest' + import cjs from './cjs-dep.cjs' + + test('cjs module keeps its scope', () => { + expect(cjs.answer).toBe(42) + expect(cjs.base).toBe('cjs-dep.cjs') + expect(cjs.dir).toBe('string') + expect(typeof module).toBe('undefined') + }) + ` + } + const { stderr, exitCode } = await runInlineTests(structure, { + pool: 'forks', + isolate: true, + fileParallelism: false, + injectCjsGlobals: false, + experimental: { fsModuleCache: true }, + }) + expect(stderr).toBe('') + expect(exitCode).toBe(0) +}) + test('".js" modules without ESM syntax are detected as commonjs in a typeless package', async () => { const { stderr, exitCode } = await runInlineTests({ 'package.json': '{}',