diff --git a/packages/vitest/src/node/cli/doctor.ts b/packages/vitest/src/node/cli/doctor.ts index cfb9b5b81..13d925649 100644 --- a/packages/vitest/src/node/cli/doctor.ts +++ b/packages/vitest/src/node/cli/doctor.ts @@ -1,7 +1,7 @@ import type { CliOptions } from './cli-api' import { spawn } from 'node:child_process' import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { availableParallelism, tmpdir } from 'node:os' +import { tmpdir } from 'node:os' import { join } from 'node:path' import { performance } from 'node:perf_hooks' import { pathToFileURL } from 'node:url' @@ -199,11 +199,7 @@ export async function doctor(cliFilters: string[], options: CliOptions): Promise fsModuleCache: project.config.fsModuleCache === true, })) const fileCount = (await ctx.getRelevantTestSpecifications(cliFilters)).length - const configuredMaxWorkers = ctx.config.maxWorkers - const effectiveMaxWorkers = - typeof configuredMaxWorkers === 'number' && configuredMaxWorkers > 0 - ? configuredMaxWorkers - : Math.max(1, availableParallelism() - 1) + const effectiveMaxWorkers = ctx.config.maxWorkers await ctx.close() // the environments import 'happy-dom' relative to the vitest package (it is diff --git a/packages/vitest/src/node/config/resolveConfig.ts b/packages/vitest/src/node/config/resolveConfig.ts index 9b41728ed..97fe6d783 100644 --- a/packages/vitest/src/node/config/resolveConfig.ts +++ b/packages/vitest/src/node/config/resolveConfig.ts @@ -24,7 +24,7 @@ import { configFiles, defaultInspectPort } from '../../constants' import { benchmarkConfigDefaults, configDefaults } from '../../defaults' import { wildcardPatternToRegExp } from '../../utils/base' import { isAgent, isCI, stdProvider } from '../../utils/env' -import { getWorkersCountByPercentage } from '../../utils/workers' +import { getDefaultMaxWorkers, resolveMaxWorkers } from '../../utils/workers' import { BrowserLoaderPlugin } from '../plugins/browserLoader' import { ViteConfigPlugin } from '../plugins/config' import { VitestCorePlugin } from '../plugins/index' @@ -153,14 +153,6 @@ export function resolveApiServerConfig( return api } -function resolveInlineWorkerOption(value: string | number): number { - if (typeof value === 'string' && value.trim().endsWith('%')) { - return getWorkersCountByPercentage(value) - } else { - return Number(value) - } -} - /** * Records which options the user provided explicitly. Must be computed from * the raw user config sources BEFORE `configDefaults` is merged in - the @@ -179,6 +171,7 @@ function captureProvidedOptions( (source?.experimental as { fsModuleCache?: boolean } | undefined)?.fsModuleCache != null, ), silent: sources.some((source) => source?.silent != null), + maxWorkers: sources.some((source) => source?.maxWorkers != null), } } @@ -383,9 +376,9 @@ export function resolveTestConfig( throw new Error(`Cannot merge reports with --watch enabled`) } - if (resolved.maxWorkers) { - resolved.maxWorkers = resolveInlineWorkerOption(resolved.maxWorkers) - } + resolved.maxWorkers = resolved.maxWorkers + ? resolveMaxWorkers(resolved.maxWorkers) + : getDefaultMaxWorkers(resolved.watch) // `browser.fileParallelism` was replaced by the top-level `fileParallelism`. Map // it (only when browser is enabled, since it was a browser-only option) so @@ -716,6 +709,7 @@ export function resolveTestConfig( if (process.env.VITEST_MAX_WORKERS) { resolved.maxWorkers = Number.parseInt(process.env.VITEST_MAX_WORKERS) + resolved.providedOptions.maxWorkers = true } if (typeof resolved.diff === 'string') { diff --git a/packages/vitest/src/node/pool.ts b/packages/vitest/src/node/pool.ts index c92eeb72e..01d6dc4aa 100644 --- a/packages/vitest/src/node/pool.ts +++ b/packages/vitest/src/node/pool.ts @@ -299,27 +299,6 @@ function resolveConditions(project: TestProject) { .flatMap((c) => ['--conditions', c]) } -function resolveMaxWorkers(project: TestProject) { - if (project.config.maxWorkers) { - return project.config.maxWorkers - } - - if (project.vitest.config.maxWorkers) { - return project.vitest.config.maxWorkers - } - - const numCpus = - typeof nodeos.availableParallelism === 'function' - ? nodeos.availableParallelism() - : nodeos.cpus().length - - if (project.vitest.config.watch) { - return Math.max(Math.floor(numCpus / 2), 1) - } - - return Math.max(numCpus - 1, 1) -} - function getMemoryLimit(config: ResolvedConfig, pool: string) { if (pool !== 'vmForks' && pool !== 'vmThreads') { return null @@ -418,7 +397,7 @@ function groupSpecs( return sequential.specs.push([spec]) } - const maxWorkers = resolveMaxWorkers(spec.project) + const maxWorkers = spec.project.config.maxWorkers groups[order] ||= { specs: [], maxWorkers } // Multiple projects with different maxWorkers but same groupOrder @@ -458,7 +437,7 @@ function groupSpecs( let order = Math.max(0, ...groups.keys()) + 1 for (const projectName in typechecks) { - const maxWorkers = resolveMaxWorkers(typechecks[projectName][0].project) + const maxWorkers = typechecks[projectName][0].project.config.maxWorkers const previous = groups[order - 1] if (previous && previous.typecheck && maxWorkers !== previous.maxWorkers) { order += 1 diff --git a/packages/vitest/src/node/pools/browser.ts b/packages/vitest/src/node/pools/browser.ts index 84811b67a..5fcdabddf 100644 --- a/packages/vitest/src/node/pools/browser.ts +++ b/packages/vitest/src/node/pools/browser.ts @@ -10,7 +10,6 @@ import type { BrowserProvider, CDPSession } from '../types/browser' import crypto from 'node:crypto' import { statfsSync } from 'node:fs' import { readFile } from 'node:fs/promises' -import * as nodeos from 'node:os' import { createDefer } from '@vitest/utils/helpers' import { stringify } from 'flatted' import { createDebugger } from '../../utils/debugger' @@ -24,18 +23,6 @@ const PROVIDER_CLOSE_TIMEOUT = 10_000 export function createBrowserPool(vitest: Vitest): ProcessPool { const providers = new Set() - const numCpus = - typeof nodeos.availableParallelism === 'function' - ? nodeos.availableParallelism() - : nodeos.cpus().length - - // if there are more than ~12 threads (optimistically), the main thread chokes - // https://github.com/vitest-dev/vitest/issues/7871 - const maxThreadsCount = Math.min(12, numCpus - 1) - const threadsCount = vitest.config.watch - ? Math.max(Math.floor(maxThreadsCount / 2), 1) - : Math.max(maxThreadsCount, 1) - const projectPools = new WeakMap() const ensurePool = (project: TestProject) => { @@ -162,11 +149,14 @@ export function createBrowserPool(vitest: Vitest): ProcessPool { return 1 } - if (project.config.maxWorkers) { - return project.config.maxWorkers + const { maxWorkers, providedOptions } = project.config + if (providedOptions.maxWorkers) { + return maxWorkers } - return threadsCount + // if there are more than ~12 threads (optimistically), the main thread chokes + // https://github.com/vitest-dev/vitest/issues/7871 + return Math.min(12, maxWorkers) } return { diff --git a/packages/vitest/src/node/projects/resolveProjects.ts b/packages/vitest/src/node/projects/resolveProjects.ts index da8b63177..638b8be4e 100644 --- a/packages/vitest/src/node/projects/resolveProjects.ts +++ b/packages/vitest/src/node/projects/resolveProjects.ts @@ -24,6 +24,7 @@ import { configFiles as defaultConfigFiles } from '../../constants' import { wildcardPatternToRegExp } from '../../utils/base' import { createDebugger } from '../../utils/debugger' import { limitConcurrency } from '../../utils/limit-concurrency' +import { resolveMaxWorkers } from '../../utils/workers' import { CaptureRawTestConfig, isExcludedByProjectFilter, @@ -1170,6 +1171,7 @@ function cloneProjectConfigForBrowserInstance( screenshotDirectory, screenshotFailures, fileParallelism, + maxWorkers, // @ts-expect-error remove just in case browser: _browser, name, @@ -1181,7 +1183,12 @@ function cloneProjectConfigForBrowserInstance( return mergeConfig( { ...clonedConfig, - maxWorkers: config.fileParallelism === false ? 1 : clonedConfig.maxWorkers, + maxWorkers: + fileParallelism === false + ? 1 + : maxWorkers != null + ? resolveMaxWorkers(maxWorkers) + : clonedConfig.maxWorkers, browser: { ...parentConfig.browser, locators: locators diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 076305b46..5c1da68e3 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -20,7 +20,6 @@ import type { TestSuiteState, } from './reported-tasks' import { readFileSync } from 'node:fs' -import { availableParallelism } from 'node:os' import { performance } from 'node:perf_hooks' import { toArray } from '@vitest/utils/helpers' import { parseStacktrace } from '@vitest/utils/source-map' @@ -777,13 +776,6 @@ export abstract class BaseReporter implements Reporter { this.log() } - private getEffectiveMaxWorkers(): number { - const configured = this.ctx.config.maxWorkers - return typeof configured === 'number' && configured > 0 - ? configured - : Math.max(1, availableParallelism() - 1) - } - /** * Surfaces the cost of re-creating a DOM environment for every test file: * with an isolating pool, `jsdom`/`happy-dom` are imported and set up once @@ -803,7 +795,7 @@ export abstract class BaseReporter implements Reporter { } const executionTime = this.end - this.start - const maxWorkers = this.getEffectiveMaxWorkers() + const maxWorkers = this.ctx.config.maxWorkers const inputs = this.ctx.projects.map((project) => { const projectFiles = files.filter((file) => (file.projectName || '') === project.name) let environmentTime = 0 @@ -883,7 +875,7 @@ export abstract class BaseReporter implements Reporter { } const executionTime = this.end - this.start - const maxWorkers = this.getEffectiveMaxWorkers() + const maxWorkers = this.ctx.config.maxWorkers const inputs = this.ctx.projects.map((project) => { const projectFiles = files.filter((file) => (file.projectName || '') === project.name) let importTime = 0 @@ -1045,7 +1037,7 @@ export abstract class BaseReporter implements Reporter { // with `isolate: false` the same files would run in ~`parallelism` reused // workers instead of spawning a fresh worker for every file - const parallelism = Math.max(1, Math.min(numFiles, this.getEffectiveMaxWorkers())) + const parallelism = Math.max(1, Math.min(numFiles, this.ctx.config.maxWorkers)) // nothing was actually spawned per-file (e.g. a single worker handled everything) if (numWorkers <= parallelism) { diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index 310b23809..03d0eee87 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -1366,6 +1366,7 @@ export interface ResolvedConfig extends Omit< environment: boolean fsModuleCache: boolean silent: boolean + maxWorkers: boolean } cliOptions: CliOptions diff --git a/packages/vitest/src/utils/memory-limit.ts b/packages/vitest/src/utils/memory-limit.ts index ceb3acfb7..c4e4d5e24 100644 --- a/packages/vitest/src/utils/memory-limit.ts +++ b/packages/vitest/src/utils/memory-limit.ts @@ -6,27 +6,15 @@ */ import type { ResolvedConfig } from '../node/types/config' -import * as nodeos from 'node:os' - -function getDefaultThreadsCount(config: Pick) { - const numCpus = - typeof nodeos.availableParallelism === 'function' - ? nodeos.availableParallelism() - : nodeos.cpus().length - - return config.watch ? Math.max(Math.floor(numCpus / 2), 1) : Math.max(numCpus - 1, 1) -} export function getWorkerMemoryLimit( - config: Pick, + config: Pick, ): string | number { if (config.vmMemoryLimit) { return config.vmMemoryLimit } - const workers = config.maxWorkers ?? getDefaultThreadsCount(config) - - return 1 / workers + return 1 / config.maxWorkers } /** diff --git a/packages/vitest/src/utils/workers.ts b/packages/vitest/src/utils/workers.ts index f03122603..2b1228944 100644 --- a/packages/vitest/src/utils/workers.ts +++ b/packages/vitest/src/utils/workers.ts @@ -1,8 +1,19 @@ import os from 'node:os' -export function getWorkersCountByPercentage(percent: string): number { - const maxWorkersCount = os.availableParallelism?.() ?? os.cpus().length - const workersCountByPercentage = Math.round((Number.parseInt(percent) / 100) * maxWorkersCount) +function getAvailableParallelism(): number { + return os.availableParallelism?.() ?? os.cpus().length +} + +export function getDefaultMaxWorkers(watch: boolean): number { + const count = getAvailableParallelism() + return watch ? Math.max(Math.floor(count / 2), 1) : Math.max(count - 1, 1) +} - return Math.max(1, Math.min(maxWorkersCount, workersCountByPercentage)) +export function resolveMaxWorkers(value: number | string): number { + if (typeof value === 'string' && value.trim().endsWith('%')) { + const count = getAvailableParallelism() + const byPercentage = Math.round((Number.parseInt(value) / 100) * count) + return Math.max(1, Math.min(count, byPercentage)) + } + return Number(value) } diff --git a/test/e2e/test/workers-option.test.ts b/test/e2e/test/workers-option.test.ts index 131eda9aa..6bb6060fc 100644 --- a/test/e2e/test/workers-option.test.ts +++ b/test/e2e/test/workers-option.test.ts @@ -1,6 +1,9 @@ import { describe, expect, test, vi } from 'vitest' import * as testUtils from '#test-utils' -import { getWorkersCountByPercentage } from '../../../packages/vitest/src/utils/workers.js' +import { + getDefaultMaxWorkers, + resolveMaxWorkers, +} from '../../../packages/vitest/src/utils/workers.js' vi.mock(import('node:os'), async (importOriginal) => ({ ...(await importOriginal()), @@ -12,15 +15,25 @@ vi.mock(import('node:os'), async (importOriginal) => ({ describe('workers util', () => { test('percent=50% should return 5', () => { - expect(getWorkersCountByPercentage('50%')).toBe(5) + expect(resolveMaxWorkers('50%')).toBe(5) }) test('percent=-10% should return 1', () => { - expect(getWorkersCountByPercentage('-10%')).toBe(1) + expect(resolveMaxWorkers('-10%')).toBe(1) }) test('percent=110% should return 10', () => { - expect(getWorkersCountByPercentage('110%')).toBe(10) + expect(resolveMaxWorkers('110%')).toBe(10) + }) + + test('number strings and numbers are returned as is', () => { + expect(resolveMaxWorkers('3')).toBe(3) + expect(resolveMaxWorkers(4)).toBe(4) + }) + + test('default leaves one core free, half in watch mode', () => { + expect(getDefaultMaxWorkers(false)).toBe(9) + expect(getDefaultMaxWorkers(true)).toBe(5) }) }) diff --git a/test/unit/test/memory-limit.test.ts b/test/unit/test/memory-limit.test.ts index 757b24f6c..8ca63c9d2 100644 --- a/test/unit/test/memory-limit.test.ts +++ b/test/unit/test/memory-limit.test.ts @@ -3,16 +3,10 @@ import { getWorkerMemoryLimit } from '../../../packages/vitest/src/utils/memory- describe('getWorkerMemoryLimit', () => { it('should prioritize vmMemoryLimit', () => { - expect(getWorkerMemoryLimit({ vmMemoryLimit: '512MB', maxWorkers: 1, watch: false })).toBe( - '512MB', - ) + expect(getWorkerMemoryLimit({ vmMemoryLimit: '512MB', maxWorkers: 1 })).toBe('512MB') }) it('should calculate 1/maxWorkers', () => { - expect(getWorkerMemoryLimit({ maxWorkers: 4, watch: false })).toBe(1 / 4) - }) - - it('should calculate maxWorkers/2 when in watch-mode', () => { - expect(getWorkerMemoryLimit({ maxWorkers: 14, watch: true })).toBe(1 / 14) + expect(getWorkerMemoryLimit({ maxWorkers: 4 })).toBe(1 / 4) }) })