From 3e3624c5e5ecf77b20065da1aeb60f1182db0ccd Mon Sep 17 00:00:00 2001 From: Vladimir Date: Thu, 8 Oct 2026 13:08:32 +0200 Subject: [PATCH] refactor: resolve maxWorkers once in config (#11524) --- docs/config/coverage.md | 2 +- packages/vitest/src/defaults.ts | 2 +- packages/vitest/src/node/affected-modules.ts | 2 +- packages/vitest/src/node/cli/doctor.ts | 8 +--- .../vitest/src/node/config/resolveConfig.ts | 18 +++------ packages/vitest/src/node/pool.ts | 25 +----------- packages/vitest/src/node/pools/browser.ts | 22 +++-------- .../src/node/projects/resolveProjects.ts | 11 +++++- packages/vitest/src/node/reporters/base.ts | 14 ++----- packages/vitest/src/node/types/config.ts | 1 + packages/vitest/src/node/types/coverage.ts | 2 +- packages/vitest/src/utils/memory-limit.ts | 16 +------- packages/vitest/src/utils/workers.ts | 17 ++++++--- test/e2e/test/config/browser-configs.test.ts | 38 +++++++++++-------- test/e2e/test/workers-option.test.ts | 33 ++++++++++------ test/unit/test/memory-limit.test.ts | 10 +---- 16 files changed, 93 insertions(+), 128 deletions(-) diff --git a/docs/config/coverage.md b/docs/config/coverage.md index ff1a22a1d..a4d01ad3e 100644 --- a/docs/config/coverage.md +++ b/docs/config/coverage.md @@ -439,7 +439,7 @@ Watermarks for statements, lines, branches and functions. See [istanbul document ## coverage.processingConcurrency - **Type:** `boolean` -- **Default:** `Math.min(20, os.availableParallelism?.() ?? os.cpus().length)` +- **Default:** `Math.min(20, os.availableParallelism())` - **Available for providers:** `'v8' | 'istanbul'` - **CLI:** `--coverage.processingConcurrency=` diff --git a/packages/vitest/src/defaults.ts b/packages/vitest/src/defaults.ts index b58ff6dac..58d808c13 100644 --- a/packages/vitest/src/defaults.ts +++ b/packages/vitest/src/defaults.ts @@ -31,7 +31,7 @@ export const coverageConfigDefaults: Required() - private transformConcurrency = os.availableParallelism?.() ?? os.cpus().length + private transformConcurrency = os.availableParallelism() private activeTransforms = 0 private transformQueue: Array<() => void> = [] diff --git a/packages/vitest/src/node/cli/doctor.ts b/packages/vitest/src/node/cli/doctor.ts index cfb9b5b81..13d925649 100644 --- a/packages/vitest/src/node/cli/doctor.ts +++ b/packages/vitest/src/node/cli/doctor.ts @@ -1,7 +1,7 @@ import type { CliOptions } from './cli-api' import { spawn } from 'node:child_process' import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { availableParallelism, tmpdir } from 'node:os' +import { tmpdir } from 'node:os' import { join } from 'node:path' import { performance } from 'node:perf_hooks' import { pathToFileURL } from 'node:url' @@ -199,11 +199,7 @@ export async function doctor(cliFilters: string[], options: CliOptions): Promise fsModuleCache: project.config.fsModuleCache === true, })) const fileCount = (await ctx.getRelevantTestSpecifications(cliFilters)).length - const configuredMaxWorkers = ctx.config.maxWorkers - const effectiveMaxWorkers = - typeof configuredMaxWorkers === 'number' && configuredMaxWorkers > 0 - ? configuredMaxWorkers - : Math.max(1, availableParallelism() - 1) + const effectiveMaxWorkers = ctx.config.maxWorkers await ctx.close() // the environments import 'happy-dom' relative to the vitest package (it is diff --git a/packages/vitest/src/node/config/resolveConfig.ts b/packages/vitest/src/node/config/resolveConfig.ts index 9b41728ed..97fe6d783 100644 --- a/packages/vitest/src/node/config/resolveConfig.ts +++ b/packages/vitest/src/node/config/resolveConfig.ts @@ -24,7 +24,7 @@ import { configFiles, defaultInspectPort } from '../../constants' import { benchmarkConfigDefaults, configDefaults } from '../../defaults' import { wildcardPatternToRegExp } from '../../utils/base' import { isAgent, isCI, stdProvider } from '../../utils/env' -import { getWorkersCountByPercentage } from '../../utils/workers' +import { getDefaultMaxWorkers, resolveMaxWorkers } from '../../utils/workers' import { BrowserLoaderPlugin } from '../plugins/browserLoader' import { ViteConfigPlugin } from '../plugins/config' import { VitestCorePlugin } from '../plugins/index' @@ -153,14 +153,6 @@ export function resolveApiServerConfig( return api } -function resolveInlineWorkerOption(value: string | number): number { - if (typeof value === 'string' && value.trim().endsWith('%')) { - return getWorkersCountByPercentage(value) - } else { - return Number(value) - } -} - /** * Records which options the user provided explicitly. Must be computed from * the raw user config sources BEFORE `configDefaults` is merged in - the @@ -179,6 +171,7 @@ function captureProvidedOptions( (source?.experimental as { fsModuleCache?: boolean } | undefined)?.fsModuleCache != null, ), silent: sources.some((source) => source?.silent != null), + maxWorkers: sources.some((source) => source?.maxWorkers != null), } } @@ -383,9 +376,9 @@ export function resolveTestConfig( throw new Error(`Cannot merge reports with --watch enabled`) } - if (resolved.maxWorkers) { - resolved.maxWorkers = resolveInlineWorkerOption(resolved.maxWorkers) - } + resolved.maxWorkers = resolved.maxWorkers + ? resolveMaxWorkers(resolved.maxWorkers) + : getDefaultMaxWorkers(resolved.watch) // `browser.fileParallelism` was replaced by the top-level `fileParallelism`. Map // it (only when browser is enabled, since it was a browser-only option) so @@ -716,6 +709,7 @@ export function resolveTestConfig( if (process.env.VITEST_MAX_WORKERS) { resolved.maxWorkers = Number.parseInt(process.env.VITEST_MAX_WORKERS) + resolved.providedOptions.maxWorkers = true } if (typeof resolved.diff === 'string') { diff --git a/packages/vitest/src/node/pool.ts b/packages/vitest/src/node/pool.ts index c92eeb72e..01d6dc4aa 100644 --- a/packages/vitest/src/node/pool.ts +++ b/packages/vitest/src/node/pool.ts @@ -299,27 +299,6 @@ function resolveConditions(project: TestProject) { .flatMap((c) => ['--conditions', c]) } -function resolveMaxWorkers(project: TestProject) { - if (project.config.maxWorkers) { - return project.config.maxWorkers - } - - if (project.vitest.config.maxWorkers) { - return project.vitest.config.maxWorkers - } - - const numCpus = - typeof nodeos.availableParallelism === 'function' - ? nodeos.availableParallelism() - : nodeos.cpus().length - - if (project.vitest.config.watch) { - return Math.max(Math.floor(numCpus / 2), 1) - } - - return Math.max(numCpus - 1, 1) -} - function getMemoryLimit(config: ResolvedConfig, pool: string) { if (pool !== 'vmForks' && pool !== 'vmThreads') { return null @@ -418,7 +397,7 @@ function groupSpecs( return sequential.specs.push([spec]) } - const maxWorkers = resolveMaxWorkers(spec.project) + const maxWorkers = spec.project.config.maxWorkers groups[order] ||= { specs: [], maxWorkers } // Multiple projects with different maxWorkers but same groupOrder @@ -458,7 +437,7 @@ function groupSpecs( let order = Math.max(0, ...groups.keys()) + 1 for (const projectName in typechecks) { - const maxWorkers = resolveMaxWorkers(typechecks[projectName][0].project) + const maxWorkers = typechecks[projectName][0].project.config.maxWorkers const previous = groups[order - 1] if (previous && previous.typecheck && maxWorkers !== previous.maxWorkers) { order += 1 diff --git a/packages/vitest/src/node/pools/browser.ts b/packages/vitest/src/node/pools/browser.ts index 84811b67a..5fcdabddf 100644 --- a/packages/vitest/src/node/pools/browser.ts +++ b/packages/vitest/src/node/pools/browser.ts @@ -10,7 +10,6 @@ import type { BrowserProvider, CDPSession } from '../types/browser' import crypto from 'node:crypto' import { statfsSync } from 'node:fs' import { readFile } from 'node:fs/promises' -import * as nodeos from 'node:os' import { createDefer } from '@vitest/utils/helpers' import { stringify } from 'flatted' import { createDebugger } from '../../utils/debugger' @@ -24,18 +23,6 @@ const PROVIDER_CLOSE_TIMEOUT = 10_000 export function createBrowserPool(vitest: Vitest): ProcessPool { const providers = new Set() - const numCpus = - typeof nodeos.availableParallelism === 'function' - ? nodeos.availableParallelism() - : nodeos.cpus().length - - // if there are more than ~12 threads (optimistically), the main thread chokes - // https://github.com/vitest-dev/vitest/issues/7871 - const maxThreadsCount = Math.min(12, numCpus - 1) - const threadsCount = vitest.config.watch - ? Math.max(Math.floor(maxThreadsCount / 2), 1) - : Math.max(maxThreadsCount, 1) - const projectPools = new WeakMap() const ensurePool = (project: TestProject) => { @@ -162,11 +149,14 @@ export function createBrowserPool(vitest: Vitest): ProcessPool { return 1 } - if (project.config.maxWorkers) { - return project.config.maxWorkers + const { maxWorkers, providedOptions } = project.config + if (providedOptions.maxWorkers) { + return maxWorkers } - return threadsCount + // if there are more than ~12 threads (optimistically), the main thread chokes + // https://github.com/vitest-dev/vitest/issues/7871 + return Math.min(12, maxWorkers) } return { diff --git a/packages/vitest/src/node/projects/resolveProjects.ts b/packages/vitest/src/node/projects/resolveProjects.ts index be71efb59..ab34f1abc 100644 --- a/packages/vitest/src/node/projects/resolveProjects.ts +++ b/packages/vitest/src/node/projects/resolveProjects.ts @@ -24,6 +24,7 @@ import { configFiles as defaultConfigFiles } from '../../constants' import { wildcardPatternToRegExp } from '../../utils/base' import { createDebugger } from '../../utils/debugger' import { limitConcurrency } from '../../utils/limit-concurrency' +import { resolveMaxWorkers } from '../../utils/workers' import { CaptureRawTestConfig, isExcludedByProjectFilter, @@ -338,7 +339,7 @@ async function resolveDeclaredProjectEntries( definitions, ) - const concurrent = limitConcurrency(os.availableParallelism?.() || os.cpus().length || 5) + const concurrent = limitConcurrency(os.availableParallelism() || 5) const fileProjects = [...configFiles, ...nonConfigDirectories] const promises: Promise[] = [] @@ -1177,6 +1178,7 @@ function cloneProjectConfigForBrowserInstance( screenshotDirectory, screenshotFailures, fileParallelism, + maxWorkers, // @ts-expect-error remove just in case browser: _browser, name, @@ -1188,7 +1190,12 @@ function cloneProjectConfigForBrowserInstance( return mergeConfig( { ...clonedConfig, - maxWorkers: config.fileParallelism === false ? 1 : clonedConfig.maxWorkers, + maxWorkers: + fileParallelism === false + ? 1 + : maxWorkers != null + ? resolveMaxWorkers(maxWorkers) + : clonedConfig.maxWorkers, browser: { ...parentConfig.browser, locators: locators diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 076305b46..5c1da68e3 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -20,7 +20,6 @@ import type { TestSuiteState, } from './reported-tasks' import { readFileSync } from 'node:fs' -import { availableParallelism } from 'node:os' import { performance } from 'node:perf_hooks' import { toArray } from '@vitest/utils/helpers' import { parseStacktrace } from '@vitest/utils/source-map' @@ -777,13 +776,6 @@ export abstract class BaseReporter implements Reporter { this.log() } - private getEffectiveMaxWorkers(): number { - const configured = this.ctx.config.maxWorkers - return typeof configured === 'number' && configured > 0 - ? configured - : Math.max(1, availableParallelism() - 1) - } - /** * Surfaces the cost of re-creating a DOM environment for every test file: * with an isolating pool, `jsdom`/`happy-dom` are imported and set up once @@ -803,7 +795,7 @@ export abstract class BaseReporter implements Reporter { } const executionTime = this.end - this.start - const maxWorkers = this.getEffectiveMaxWorkers() + const maxWorkers = this.ctx.config.maxWorkers const inputs = this.ctx.projects.map((project) => { const projectFiles = files.filter((file) => (file.projectName || '') === project.name) let environmentTime = 0 @@ -883,7 +875,7 @@ export abstract class BaseReporter implements Reporter { } const executionTime = this.end - this.start - const maxWorkers = this.getEffectiveMaxWorkers() + const maxWorkers = this.ctx.config.maxWorkers const inputs = this.ctx.projects.map((project) => { const projectFiles = files.filter((file) => (file.projectName || '') === project.name) let importTime = 0 @@ -1045,7 +1037,7 @@ export abstract class BaseReporter implements Reporter { // with `isolate: false` the same files would run in ~`parallelism` reused // workers instead of spawning a fresh worker for every file - const parallelism = Math.max(1, Math.min(numFiles, this.getEffectiveMaxWorkers())) + const parallelism = Math.max(1, Math.min(numFiles, this.ctx.config.maxWorkers)) // nothing was actually spawned per-file (e.g. a single worker handled everything) if (numWorkers <= parallelism) { diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index e71a7ca71..e56dd7816 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -1375,6 +1375,7 @@ export interface ResolvedConfig extends Omit< environment: boolean fsModuleCache: boolean silent: boolean + maxWorkers: boolean } cliOptions: CliOptions diff --git a/packages/vitest/src/node/types/coverage.ts b/packages/vitest/src/node/types/coverage.ts index 48fbdc3c9..d3780dd5e 100644 --- a/packages/vitest/src/node/types/coverage.ts +++ b/packages/vitest/src/node/types/coverage.ts @@ -249,7 +249,7 @@ export interface CoverageOptions { /** * Concurrency limit used when processing the coverage results. - * Defaults to `Math.min(20, os.availableParallelism?.() ?? os.cpus().length)` + * Defaults to `Math.min(20, os.availableParallelism())` */ processingConcurrency?: number diff --git a/packages/vitest/src/utils/memory-limit.ts b/packages/vitest/src/utils/memory-limit.ts index ceb3acfb7..c4e4d5e24 100644 --- a/packages/vitest/src/utils/memory-limit.ts +++ b/packages/vitest/src/utils/memory-limit.ts @@ -6,27 +6,15 @@ */ import type { ResolvedConfig } from '../node/types/config' -import * as nodeos from 'node:os' - -function getDefaultThreadsCount(config: Pick) { - const numCpus = - typeof nodeos.availableParallelism === 'function' - ? nodeos.availableParallelism() - : nodeos.cpus().length - - return config.watch ? Math.max(Math.floor(numCpus / 2), 1) : Math.max(numCpus - 1, 1) -} export function getWorkerMemoryLimit( - config: Pick, + config: Pick, ): string | number { if (config.vmMemoryLimit) { return config.vmMemoryLimit } - const workers = config.maxWorkers ?? getDefaultThreadsCount(config) - - return 1 / workers + return 1 / config.maxWorkers } /** diff --git a/packages/vitest/src/utils/workers.ts b/packages/vitest/src/utils/workers.ts index f03122603..20d04522a 100644 --- a/packages/vitest/src/utils/workers.ts +++ b/packages/vitest/src/utils/workers.ts @@ -1,8 +1,15 @@ -import os from 'node:os' +import { availableParallelism } from 'node:os' -export function getWorkersCountByPercentage(percent: string): number { - const maxWorkersCount = os.availableParallelism?.() ?? os.cpus().length - const workersCountByPercentage = Math.round((Number.parseInt(percent) / 100) * maxWorkersCount) +export function getDefaultMaxWorkers(watch: boolean): number { + const count = availableParallelism() + return watch ? Math.max(Math.floor(count / 2), 1) : Math.max(count - 1, 1) +} - return Math.max(1, Math.min(maxWorkersCount, workersCountByPercentage)) +export function resolveMaxWorkers(value: number | string): number { + if (typeof value === 'string' && value.trim().endsWith('%')) { + const count = availableParallelism() + const byPercentage = Math.round((Number.parseInt(value) / 100) * count) + return Math.max(1, Math.min(count, byPercentage)) + } + return Number(value) } diff --git a/test/e2e/test/config/browser-configs.test.ts b/test/e2e/test/config/browser-configs.test.ts index 7a3aee32d..d433d2e9a 100644 --- a/test/e2e/test/config/browser-configs.test.ts +++ b/test/e2e/test/config/browser-configs.test.ts @@ -778,26 +778,32 @@ test('provider options can be changed dynamically in CLI', async () => { }) test('fileParallelism on the instance works properly', async () => { - const projects = await config({ - fileParallelism: true, - browser: { - enabled: true, - provider: playwright(), - instances: [ - { - browser: 'chromium', - fileParallelism: false, - }, - { - browser: 'firefox', - fileParallelism: true, + const { test: rootConfig } = await resolveConfig( + { config: false }, + { + test: { + fileParallelism: true, + browser: { + enabled: true, + provider: playwright(), + instances: [ + { + browser: 'chromium', + fileParallelism: false, + }, + { + browser: 'firefox', + fileParallelism: true, + }, + ], }, - ], + }, }, - }) + ) + const projects = rootConfig.resolvedProjects.filter((p) => !p.hidden) expect(projects).toHaveLength(2) expect(projects[0].projectConfig.maxWorkers).toBe(1) - expect(projects[1].projectConfig.maxWorkers).toBe(undefined) // decided dynamically + expect(projects[1].projectConfig.maxWorkers).toBe(rootConfig.maxWorkers) }) test('detailsPanelPosition defaults to right', async () => { diff --git a/test/e2e/test/workers-option.test.ts b/test/e2e/test/workers-option.test.ts index 131eda9aa..e3049402a 100644 --- a/test/e2e/test/workers-option.test.ts +++ b/test/e2e/test/workers-option.test.ts @@ -1,26 +1,37 @@ import { describe, expect, test, vi } from 'vitest' import * as testUtils from '#test-utils' -import { getWorkersCountByPercentage } from '../../../packages/vitest/src/utils/workers.js' +import { + getDefaultMaxWorkers, + resolveMaxWorkers, +} from '../../../packages/vitest/src/utils/workers.js' -vi.mock(import('node:os'), async (importOriginal) => ({ - ...(await importOriginal()), - default: { - ...(await importOriginal()).default, - availableParallelism: () => 10, - }, -})) +vi.mock(import('node:os'), async (importOriginal) => { + const os = await importOriginal() + const availableParallelism = () => 10 + return { ...os, availableParallelism, default: { ...os.default, availableParallelism } } +}) describe('workers util', () => { test('percent=50% should return 5', () => { - expect(getWorkersCountByPercentage('50%')).toBe(5) + expect(resolveMaxWorkers('50%')).toBe(5) }) test('percent=-10% should return 1', () => { - expect(getWorkersCountByPercentage('-10%')).toBe(1) + expect(resolveMaxWorkers('-10%')).toBe(1) }) test('percent=110% should return 10', () => { - expect(getWorkersCountByPercentage('110%')).toBe(10) + expect(resolveMaxWorkers('110%')).toBe(10) + }) + + test('number strings and numbers are returned as is', () => { + expect(resolveMaxWorkers('3')).toBe(3) + expect(resolveMaxWorkers(4)).toBe(4) + }) + + test('default leaves one core free, half in watch mode', () => { + expect(getDefaultMaxWorkers(false)).toBe(9) + expect(getDefaultMaxWorkers(true)).toBe(5) }) }) diff --git a/test/unit/test/memory-limit.test.ts b/test/unit/test/memory-limit.test.ts index 757b24f6c..8ca63c9d2 100644 --- a/test/unit/test/memory-limit.test.ts +++ b/test/unit/test/memory-limit.test.ts @@ -3,16 +3,10 @@ import { getWorkerMemoryLimit } from '../../../packages/vitest/src/utils/memory- describe('getWorkerMemoryLimit', () => { it('should prioritize vmMemoryLimit', () => { - expect(getWorkerMemoryLimit({ vmMemoryLimit: '512MB', maxWorkers: 1, watch: false })).toBe( - '512MB', - ) + expect(getWorkerMemoryLimit({ vmMemoryLimit: '512MB', maxWorkers: 1 })).toBe('512MB') }) it('should calculate 1/maxWorkers', () => { - expect(getWorkerMemoryLimit({ maxWorkers: 4, watch: false })).toBe(1 / 4) - }) - - it('should calculate maxWorkers/2 when in watch-mode', () => { - expect(getWorkerMemoryLimit({ maxWorkers: 14, watch: true })).toBe(1 / 14) + expect(getWorkerMemoryLimit({ maxWorkers: 4 })).toBe(1 / 4) }) }) -- 2.51.2