diff --git a/docs/config/maxworkers.md b/docs/config/maxworkers.md index acdc4e67b..41525518d 100644 --- a/docs/config/maxworkers.md +++ b/docs/config/maxworkers.md @@ -58,3 +58,9 @@ vitest --maxWorkers=50% ::: Vitest uses [`os.availableParallelism`](https://nodejs.org/api/os.html#osavailableparallelism) to know the maximum amount of parallelism available. + +## Browser Mode + +In [Browser Mode](/guide/browser/), every browser instance (a [project](/guide/projects) with `browser.enabled`, or an entry in [`browser.instances`](/config/browser/instances)) is a worker, so at most `maxWorkers` instances are open at the same time. The same number is the budget of pages shared by the open instances: a single instance runs up to `maxWorkers` test files in parallel in its own pages, and several instances split the budget between them, with at least one page each. + +An instance runs a single test file at a time when the browser is not headless, or when the provider does not support parallelism. diff --git a/packages/browser-playwright/src/playwright.ts b/packages/browser-playwright/src/playwright.ts index b945a5da2..570daca3d 100644 --- a/packages/browser-playwright/src/playwright.ts +++ b/packages/browser-playwright/src/playwright.ts @@ -102,6 +102,7 @@ export function playwright( } interface WarmBrowser { + config: LaunchContext['config'] promise: Promise launchOptionsJson: string pending: Set @@ -140,15 +141,7 @@ function prewarmBrowser(project: LaunchContext, options: PlaywrightProviderOptio if (warmBrowsers.has(project.config)) { return } - let pending = pendingWarmBrowsers.get(project.vitest) - if (!pending) { - const pendingBrowsers = new Set() - pending = pendingBrowsers - pendingWarmBrowsers.set(project.vitest, pendingBrowsers) - // Browsers whose projects never initialize a provider (they have no test - // files to run) are cleaned up when Vitest closes. - project.vitest.onClose(() => closeWarmBrowsers(pendingBrowsers)) - } + const pending = getPendingWarmBrowsers(project.vitest) const launchOptions = resolveLaunchOptions( project.config.browser, project.vitest.config.inspector, @@ -156,7 +149,8 @@ function prewarmBrowser(project: LaunchContext, options: PlaywrightProviderOptio browserName, ) const entry: WarmBrowser = { - launchOptionsJson: JSON.stringify(launchOptions), + config: project.config, + launchOptionsJson: getWarmBrowserKey(browserName, launchOptions), pending, promise: (async () => { debug?.('[%s] prewarming the browser', browserName) @@ -176,6 +170,41 @@ function prewarmBrowser(project: LaunchContext, options: PlaywrightProviderOptio warmBrowsers.set(project.config, entry) } +function getWarmBrowserKey(browserName: string, launchOptions: LaunchOptions): string { + return JSON.stringify([browserName, launchOptions]) +} + +function getPendingWarmBrowsers(vitest: LaunchContext['vitest']): Set { + let pending = pendingWarmBrowsers.get(vitest) + if (!pending) { + const pendingBrowsers = new Set() + pending = pendingBrowsers + pendingWarmBrowsers.set(vitest, pendingBrowsers) + // Browsers whose projects never initialize a provider (they have no test + // files to run) are cleaned up when Vitest closes. + vitest.onClose(() => closeWarmBrowsers(pendingBrowsers)) + } + return pending +} + +// the pool closes an idle instance to start another project; the browser +// process stays warm so the next project adopts it instead of launching one +function keepWarmBrowser( + project: LaunchContext, + browser: Promise, + launchOptionsJson: string, +) { + const pending = getPendingWarmBrowsers(project.vitest) + const entry: WarmBrowser = { + config: project.config, + launchOptionsJson, + pending, + promise: browser, + } + pending.add(entry) + warmBrowsers.set(project.config, entry) +} + function takeWarmBrowser(config: LaunchContext['config']): WarmBrowser | undefined { const warm = warmBrowsers.get(config) if (warm) { @@ -185,6 +214,26 @@ function takeWarmBrowser(config: LaunchContext['config']): WarmBrowser | undefin return warm } +// fewer browsers are prewarmed than there are projects and the pool starts +// projects in test order, so a browser prepared for another project is +// adopted when it is the same browser launched with the same options +function takeAnyWarmBrowser( + vitest: LaunchContext['vitest'], + launchOptionsJson: string, +): WarmBrowser | undefined { + const pending = pendingWarmBrowsers.get(vitest) + if (!pending) { + return + } + for (const warm of pending) { + if (warm.launchOptionsJson === launchOptionsJson) { + pending.delete(warm) + warmBrowsers.delete(warm.config) + return warm + } + } +} + async function closeWarmBrowsers(pending: Set): Promise { const closing = Array.from(pending, (warm) => warm.promise.then((browser) => browser.close()).catch(() => {}), @@ -245,6 +294,8 @@ export class PlaywrightBrowserProvider implements BrowserProvider { public browserName: PlaywrightBrowser private browserPromise: Promise | null = null + // set for a locally launched browser, which can be kept warm for another project + private launchOptionsJson: string | null = null private closing = false private armedContexts = new WeakMap>() @@ -369,20 +420,26 @@ export class PlaywrightBrowserProvider implements BrowserProvider { ) this.browser = this.persistentContext.browser()! } else { - const warm = takeWarmBrowser(this.project.config) - if (warm && warm.launchOptionsJson === JSON.stringify(launchOptions)) { + const launchOptionsJson = getWarmBrowserKey(this.browserName, launchOptions) + let warm = takeWarmBrowser(this.project.config) + if (warm && warm.launchOptionsJson !== launchOptionsJson) { + debug?.('[%s] discarding the prewarmed browser, launch options changed', this.browserName) + void warm.promise.then((browser) => browser.close()).catch(() => {}) + warm = undefined + } + warm ??= takeAnyWarmBrowser(this.project.vitest, launchOptionsJson) + if (warm) { const browser = await warm.promise.catch(() => null) if (browser?.isConnected()) { debug?.('[%s] adopting the prewarmed browser', this.browserName) this.browser = browser this.browserPromise = null + this.launchOptionsJson = launchOptionsJson return this.browser } - } else if (warm) { - debug?.('[%s] discarding the prewarmed browser, launch options changed', this.browserName) - void warm.promise.then((browser) => browser.close()).catch(() => {}) } this.browser = await playwright[this.browserName].launch(launchOptions) + this.launchOptionsJson = launchOptionsJson } this.browserPromise = null return this.browser @@ -768,6 +825,28 @@ export class PlaywrightBrowserProvider implements BrowserProvider { } const browser = this.browser this.browser = null + const closed = this.closeContexts() + if (browser && this.launchOptionsJson && browser.isConnected()) { + // registered before the contexts are closed, so the project that takes + // over the pool slot finds the browser right away and waits for it + const kept = closed.then( + () => browser, + async (error) => { + await browser.close().catch(() => {}) + throw error + }, + ) + keepWarmBrowser(this.project, kept, this.launchOptionsJson) + await kept + debug?.('[%s] provider is closed, the browser is kept warm', this.browserName) + return + } + await closed + await browser?.close() + debug?.('[%s] provider is closed', this.browserName) + } + + private async closeContexts(): Promise { await Promise.all(Array.from(this.pages.values(), (p) => p.close())) this.pages.clear() if (this.persistentContext) { @@ -776,8 +855,6 @@ export class PlaywrightBrowserProvider implements BrowserProvider { await Promise.all(Array.from(this.contexts.values(), (c) => c.close())) } this.contexts.clear() - await browser?.close() - debug?.('[%s] provider is closed', this.browserName) } } diff --git a/packages/browser/src/node/project.ts b/packages/browser/src/node/project.ts index 2b71edada..89cdf7c89 100644 --- a/packages/browser/src/node/project.ts +++ b/packages/browser/src/node/project.ts @@ -107,6 +107,16 @@ export class ProjectBrowser implements IProjectBrowser { } } + async closeBrowserProvider(): Promise { + const provider = this.provider + if (!provider) { + return + } + // detach first so the next run can create a new provider while this one closes + this.provider = undefined as any + await provider.close() + } + public parseErrorStacktrace(e: TestError, options: StackTraceParserOptions = {}): ParsedStack[] { return this.parent.parseErrorStacktrace(e, options) } diff --git a/packages/vitest/src/node/config/pluginHarness.ts b/packages/vitest/src/node/config/pluginHarness.ts index da989bed6..d4d016d30 100644 --- a/packages/vitest/src/node/config/pluginHarness.ts +++ b/packages/vitest/src/node/config/pluginHarness.ts @@ -13,6 +13,10 @@ export class PluginHarness { * @internal */ public _browserLastPort = defaultBrowserPort + /** + * @internal + */ + public _prewarmedBrowsers = 0 constructor( public logger: Logger = new Logger(), diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index 294ae1513..a264c6b53 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -291,6 +291,7 @@ export class Vitest { // reuse the same browser ports as the previous run instead of letting the // reused harness keep incrementing them this._harness._browserLastPort = defaultBrowserPort + this._harness._prewarmedBrowsers = 0 // harness mimics `vitest` access like in `node/create.ts` this._harness.setVitest(undefined) const config = await resolveConfig( diff --git a/packages/vitest/src/node/plugins/browserLoader.ts b/packages/vitest/src/node/plugins/browserLoader.ts index 533560af1..89e42486f 100644 --- a/packages/vitest/src/node/plugins/browserLoader.ts +++ b/packages/vitest/src/node/plugins/browserLoader.ts @@ -185,6 +185,7 @@ export async function createClusterServer( // Start browser launches now so their latency overlaps Vite server creation. // Entries that cannot run browser tests are skipped because they will never // initialize a provider that could adopt and close the prepared browser. + const harness = vitest._harness for (const child of children) { if ( child.hidden || @@ -196,7 +197,14 @@ export async function createClusterServer( // The Vite server is shared, but each child carries its own resolved // provider and browser options, so it must be prewarmed independently. const projectConfig = child.projectConfig - projectConfig.browser.provider?.prewarm?.({ config: projectConfig, vitest }) + const prewarm = projectConfig.browser.provider?.prewarm + // the pool runs at most `maxWorkers` browser instances at a time, + // the rest would only sit idle until their project gets a turn + if (!prewarm || harness._prewarmedBrowsers >= projectConfig.maxWorkers) { + continue + } + harness._prewarmedBrowsers++ + prewarm({ config: projectConfig, vitest }) } const server = await createViteServer(viteConfig) diff --git a/packages/vitest/src/node/pool.ts b/packages/vitest/src/node/pool.ts index 01d6dc4aa..b8607c699 100644 --- a/packages/vitest/src/node/pool.ts +++ b/packages/vitest/src/node/pool.ts @@ -1,6 +1,7 @@ import type { Awaitable } from '@vitest/utils' import type { ContextTestEnvironment } from '../types/worker' import type { Vitest } from './core' +import type { BrowserPool } from './pools/browser' import type { PoolTask } from './pools/types' import type { TestProject } from './project' import type { TestSpecification } from './test-specification' @@ -55,7 +56,7 @@ export function createPool(ctx: Vitest): ProcessPool { const Sequencer = ctx.config.sequence.sequencer const sequencer = new Sequencer(ctx) - let browserPool: ProcessPool | undefined + let browserPool: BrowserPool | undefined async function executeTests( method: 'run' | 'collect', @@ -199,11 +200,7 @@ export function createPool(ctx: Vitest): ProcessPool { if (browserSpecs.length) { browserPool ??= createBrowserPool(ctx) - if (method === 'collect') { - promises.push(browserPool.collectTests(browserSpecs)) - } else { - promises.push(browserPool.runTests(browserSpecs)) - } + promises.push(browserPool.runTests(method, browserSpecs, { maxWorkers })) } const groupResults = await Promise.allSettled(promises) @@ -230,7 +227,7 @@ export function createPool(ctx: Vitest): ProcessPool { async close() { await Promise.all([ pool.close(), - browserPool?.close?.(), + browserPool?.close(), ...ctx.projects.map((project) => project.typechecker?.stop()), ]) }, diff --git a/packages/vitest/src/node/pools/browser.ts b/packages/vitest/src/node/pools/browser.ts index 5fcdabddf..1283c48b7 100644 --- a/packages/vitest/src/node/pools/browser.ts +++ b/packages/vitest/src/node/pools/browser.ts @@ -3,10 +3,9 @@ import type { DeferPromise } from '@vitest/utils/helpers' import type { FileSpecification } from '../../runtime/runner/types' import type { Traces } from '../../utils/traces' import type { Vitest } from '../core' -import type { ProcessPool } from '../pool' import type { TestProject } from '../project' import type { TestSpecification } from '../test-specification' -import type { BrowserProvider, CDPSession } from '../types/browser' +import type { CDPSession } from '../types/browser' import crypto from 'node:crypto' import { statfsSync } from 'node:fs' import { readFile } from 'node:fs/promises' @@ -20,191 +19,321 @@ const debug = createDebugger('vitest:browser:pool') const PROVIDER_CLOSE_TIMEOUT = 10_000 -export function createBrowserPool(vitest: Vitest): ProcessPool { - const providers = new Set() +export interface BrowserPool { + runTests: ( + method: 'run' | 'collect', + specs: TestSpecification[], + options: { maxWorkers: number }, + ) => Promise + close: () => Promise +} + +interface BrowserTask { + project: TestProject + files: FileSpecification[] + method: 'run' | 'collect' + // providers without a per-session mocker share a single mock registry + // and some of them cannot drive several pages, so the instance runs alone + exclusive: boolean + result: DeferPromise +} - const projectPools = new WeakMap() +export function createBrowserPool(vitest: Vitest): BrowserPool { + // every browser instance is a worker; at most `maxWorkers` of them are open + const workers = new Map() + const activeWorkers = new Set() + const queue: BrowserTask[] = [] + const exitPromises: Promise[] = [] + let maxWorkers = 1 + let exclusiveRunning = false - const ensurePool = (project: TestProject) => { - if (projectPools.has(project)) { - return projectPools.get(project)! + function getThreadsCount(project: TestProject) { + const config = project.config.browser + if (!config.headless || !project.browser!.provider.supportsParallelism) { + return 1 } - debug?.('creating pool for project %s', project.name) + const { maxWorkers, providedOptions } = project.config + if (providedOptions.maxWorkers) { + return maxWorkers + } - const pool: BrowserPool = new BrowserPool(project, { - maxWorkers: getThreadsCount(project), - }) - projectPools.set(project, pool) - vitest.onCancel(() => { - pool.cancel() - }) + // if there are more than ~12 threads (optimistically), the main thread chokes + // https://github.com/vitest-dev/vitest/issues/7871 + return Math.min(12, maxWorkers) + } - return pool + // `maxWorkers` is also the budget of pages shared by the active instances: + // every instance gets a fair share, and pages above the core count only + // add contention and memory + function getPageAllowance(worker: BrowserWorker): number { + let openPages = 0 + for (const active of activeWorkers) { + openPages += active.pageCount + } + const share = Math.ceil(maxWorkers / Math.max(activeWorkers.size, 1)) + const free = Math.max(0, maxWorkers - openPages) + return Math.max(1, Math.min(worker.maxPages, share, worker.pageCount + free)) } - const runWorkspaceTests = async (method: 'run' | 'collect', specs: TestSpecification[]) => { - const groupedFiles = new Map() - const testFilesCode = new Map() - const testFileTags = new WeakMap() - - await Promise.all( - specs.map(async (spec) => { - let code = testFilesCode.get(spec.moduleId) - // TODO: this really should be done only once when collecting specifications - if (code == null) { - code = await readFile(spec.moduleId, 'utf-8').catch(() => '') - testFilesCode.set(spec.moduleId, code) - } - const { tags } = detectCodeBlock(code) - testFileTags.set(spec, tags) + function stopWorker(worker: BrowserWorker) { + workers.delete(worker.project) + exitPromises.push( + worker.stop().catch((error) => { + vitest.logger.error( + `Failed to close the browser for the "${worker.project.name}" project.`, + error, + ) }), ) + } - // to keep the sorting, we need to iterate over specs separately - for (const spec of specs) { - const { project, moduleId, testLines, testIds, testNamePattern, testTagsFilter } = spec - const files = groupedFiles.get(project) || [] - files.push({ - filepath: moduleId, - testLocations: testLines, - testIds, - testNamePattern, - testTagsFilter, - fileTags: testFileTags.get(spec), - }) - groupedFiles.set(project, files) + function evictIdleWorker(except: BrowserWorker | undefined): boolean { + // the map keeps the least recently used worker first + for (const worker of workers.values()) { + if (worker !== except && !activeWorkers.has(worker)) { + debug?.('closing the idle browser of %s to free a worker slot', worker.project.name) + stopWorker(worker) + return true + } } + return false + } + + function schedule(): void { + while (queue.length) { + const task = queue[0] + if (exclusiveRunning || activeWorkers.size >= maxWorkers) { + return + } + if (task.exclusive && activeWorkers.size > 0) { + return + } + + let worker = workers.get(task.project) + if (worker && activeWorkers.has(worker)) { + return + } + + const required = worker ? 0 : 1 + while (workers.size + required > maxWorkers) { + if (!evictIdleWorker(worker)) { + break + } + } + + if (!worker) { + debug?.('creating a worker for project %s', task.project.name) + worker = new BrowserWorker(task.project, { + maxPages: getThreadsCount(task.project), + getPageAllowance, + }) + workers.set(task.project, worker) + } + + queue.shift() + activeWorkers.add(worker) + exclusiveRunning = task.exclusive + + const current = worker + worker + .runTests(task.method, task.files) + .then( + () => task.result.resolve(), + (error) => task.result.reject(error), + ) + .finally(() => { + activeWorkers.delete(current) + exclusiveRunning = false + workers.delete(current.project) + workers.set(current.project, current) + schedule() + // the finished instance released its share of the page budget + activeWorkers.forEach((active) => active.grow()) + }) + } + } + + function cancel() { + for (const task of queue.splice(0)) { + task.result.resolve() + } + workers.forEach((worker) => worker.cancel()) + } + + async function runTests( + method: 'run' | 'collect', + specs: TestSpecification[], + options: { maxWorkers: number }, + ): Promise { + maxWorkers = options.maxWorkers let isCancelled = false vitest.onCancel(() => { isCancelled = true + cancel() }) - const initialisedPools = await Promise.all( - Array.from(groupedFiles.entries(), async ([project, files]) => { - await project._initBrowserProvider() + const groupedFiles = await groupSpecifications(specs) - if (!project.browser) { - throw new TypeError( - `The browser server was not initialized${project.name ? ` for the "${project.name}" project` : ''}. This is a bug in Vitest. Please, open a new issue with reproduction.`, - ) - } + const tasks = await Promise.all( + Array.from( + groupedFiles.entries(), + async ([project, files]): Promise => { + await project._initBrowserProvider() - if (isCancelled) { - return - } + if (!project.browser) { + throw new TypeError( + `The browser server was not initialized${project.name ? ` for the "${project.name}" project` : ''}. This is a bug in Vitest. Please, open a new issue with reproduction.`, + ) + } - debug?.('provider is ready for %s project', project.name) + if (isCancelled) { + return + } - const pool = ensurePool(project) - vitest.state.clearFiles( - project, - files.map((f) => f.filepath), - ) - providers.add(project.browser!.provider) + debug?.('provider is ready for %s project', project.name) - return { - pool, - provider: project.browser!.provider, - runTests: () => pool.runTests(method, files), - } - }), + vitest.state.clearFiles( + project, + files.map((f) => f.filepath), + ) + + const provider = project.browser.provider + return { + project, + files, + method, + exclusive: !provider.mocker || !provider.supportsParallelism, + result: createDefer(), + } + }, + ), ) if (isCancelled) { return } - const parallelPools: (() => Promise)[] = [] - const nonParallelPools: (() => Promise)[] = [] - - for (const pool of initialisedPools) { - if (!pool) { - // this means it was cancelled + const parallelTasks: BrowserTask[] = [] + const exclusiveTasks: BrowserTask[] = [] + for (const task of tasks) { + if (!task) { return } - - if (pool.provider.mocker && pool.provider.supportsParallelism) { - parallelPools.push(pool.runTests) + if (task.exclusive) { + exclusiveTasks.push(task) } else { - nonParallelPools.push(pool.runTests) + parallelTasks.push(task) } } - await Promise.all(parallelPools.map((runTests) => runTests())) + // exclusive instances go last so they don't stall the parallel ones + const ordered = [...parallelTasks, ...exclusiveTasks] + queue.push(...ordered) + schedule() - for (const runTests of nonParallelPools) { - if (isCancelled) { - return - } - - await runTests() - } + await Promise.all(ordered.map((task) => task.result)) } - function getThreadsCount(project: TestProject) { - const config = project.config.browser - if (!config.headless || !project.browser!.provider.supportsParallelism) { - return 1 + async function close() { + for (const task of queue.splice(0)) { + task.result.resolve() } + const stopping = Array.from(workers.values(), (worker) => worker.stop()) + workers.clear() + activeWorkers.clear() + await Promise.all([...stopping, ...exitPromises.splice(0)]) + // pages can be opened outside of the pool (`vitest.standalone`) + await Promise.all(vitest.projects.map((project) => closeProvider(project))) + vitest._browserSessions.sessionIds.clear() + vitest.projects.forEach((project) => { + project.browser?.state.orchestrators.forEach((orchestrator) => { + orchestrator.$close() + }) + }) + debug?.('browser pool closed all providers') + } - const { maxWorkers, providedOptions } = project.config - if (providedOptions.maxWorkers) { - return maxWorkers - } + return { runTests, close } +} - // if there are more than ~12 threads (optimistically), the main thread chokes - // https://github.com/vitest-dev/vitest/issues/7871 - return Math.min(12, maxWorkers) +async function groupSpecifications( + specs: TestSpecification[], +): Promise> { + const groupedFiles = new Map() + const testFilesCode = new Map() + const testFileTags = new WeakMap() + + await Promise.all( + specs.map(async (spec) => { + let code = testFilesCode.get(spec.moduleId) + // TODO: this really should be done only once when collecting specifications + if (code == null) { + code = await readFile(spec.moduleId, 'utf-8').catch(() => '') + testFilesCode.set(spec.moduleId, code) + } + const { tags } = detectCodeBlock(code) + testFileTags.set(spec, tags) + }), + ) + + // to keep the sorting, we need to iterate over specs separately + for (const spec of specs) { + const { project, moduleId, testLines, testIds, testNamePattern, testTagsFilter } = spec + const files = groupedFiles.get(project) || [] + files.push({ + filepath: moduleId, + testLocations: testLines, + testIds, + testNamePattern, + testTagsFilter, + fileTags: testFileTags.get(spec), + }) + groupedFiles.set(project, files) } - return { - name: 'browser', - async close() { - // a frozen or crashed browser never answers the close message; - // don't wait for it forever, the browser process is killed - // when this process exits anyway - await Promise.all( - Array.from(providers, (provider) => { - let timer: ReturnType - return Promise.race([ - Promise.resolve(provider.close()).finally(() => clearTimeout(timer)), - new Promise((resolve) => { - timer = setTimeout(() => { - vitest.logger.warn( - `The browser did not close within ${PROVIDER_CLOSE_TIMEOUT}ms. The browser process will be killed when the process exits.`, - ) - resolve() - }, PROVIDER_CLOSE_TIMEOUT) - timer.unref() - }), - ]) - }), - ) - vitest._browserSessions.sessionIds.clear() - providers.clear() - vitest.projects.forEach((project) => { - project.browser?.state.orchestrators.forEach((orchestrator) => { - orchestrator.$close() - }) - }) - debug?.('browser pool closed all providers') - }, - runTests: (files) => runWorkspaceTests('run', files), - collectTests: (files) => runWorkspaceTests('collect', files), + return groupedFiles +} + +async function closeProvider(project: TestProject): Promise { + const browser = project.browser + if (!browser?.provider) { + return } + debug?.('closing the browser provider of %s', project.name) + // a frozen or crashed browser never answers the close message; + // don't wait for it forever, the browser process is killed + // when this process exits anyway + let timer: ReturnType + await Promise.race([ + browser.closeBrowserProvider().finally(() => clearTimeout(timer)), + new Promise((resolve) => { + timer = setTimeout(() => { + project.vitest.logger.warn( + `The browser did not close within ${PROVIDER_CLOSE_TIMEOUT}ms. The browser process will be killed when the process exits.`, + ) + resolve() + }, PROVIDER_CLOSE_TIMEOUT) + timer.unref() + }), + ]) } function escapePathToRegexp(path: string): string { return path.replace(/[/\\.?*()^${}|[\]+]/g, '\\$&') } -class BrowserPool { +class BrowserWorker { private _queue: FileSpecification[] = [] + private _method: 'run' | 'collect' = 'run' private _promise: DeferPromise | undefined private _providedContext: string | undefined + // every page this worker opened or adopted, busy ones are running a file + private sessions = new Set() + private busySessions = new Set() + private openingPages = 0 private readySessions: Set private _traces: Traces @@ -214,9 +343,10 @@ class BrowserPool { } constructor( - private project: TestProject, + public project: TestProject, private options: { - maxWorkers: number + maxPages: number + getPageAllowance: (worker: BrowserWorker) => number }, ) { this._traces = project.vitest._traces @@ -241,6 +371,7 @@ class BrowserPool { this._promise?.reject(error) } this._promise = undefined + this.busySessions.clear() this.cancel() } @@ -248,6 +379,14 @@ class BrowserPool { return this.project.browser!.state.orchestrators } + get pageCount(): number { + return this.sessions.size + } + + get maxPages(): number { + return this.options.maxPages + } + async runTests(method: 'run' | 'collect', files: FileSpecification[]): Promise { this._promise ??= createDefer() @@ -257,30 +396,57 @@ class BrowserPool { return this._promise } + this._method = method this._providedContext = stringify(this.project.getProvidedContext()) this._queue.push(...files) - this.readySessions.forEach((sessionId) => { - if (this._queue.length) { - this.readySessions.delete(sessionId) - this.runNextTest(method, sessionId) + for (const sessionId of [...this.readySessions]) { + if (!this._queue.length) { + break } - }) + this.readySessions.delete(sessionId) + // the page was closed while it was idle + if (!this.orchestrators.has(sessionId)) { + this.sessions.delete(sessionId) + continue + } + this.sessions.add(sessionId) + this.runNextTest(method, sessionId) + } - if (this.orchestrators.size >= this.options.maxWorkers) { - debug?.('all orchestrators are ready, not creating more') - return this._promise + await this.openPages() + debug?.('all sessions are created') + return this._promise + } + + // opens more pages when the budget allows it, e.g. after another instance finished + grow(): void { + if (!this._promise || !this._queue.length) { + return } + this.openPages().catch((error) => this.reject(error)) + } + private async openPages(): Promise { + const method = this._method // open the minimum amount of tabs // if there is only 1 file running, we don't need 8 tabs running - const workerCount = Math.min(this.options.maxWorkers - this.orchestrators.size, files.length) + const allowance = this.options.getPageAllowance(this) + // pages that are still opening will take the next queued files + const unassigned = this._queue.length - this.openingPages + const count = Math.min(allowance - this.sessions.size, unassigned) + if (count <= 0) { + debug?.('all pages are open, not creating more') + return + } + const parallel = this.sessions.size + count > 1 const promises: Promise[] = [] - for (let i = 0; i < workerCount; i++) { + for (let i = 0; i < count; i++) { const sessionId = crypto.randomUUID() - this.project.vitest._browserSessions.sessionIds.add(sessionId) + this.sessions.add(sessionId) + this.openingPages++ const project = this.project.name debug?.('[%s] creating session for %s', sessionId, project) const page = this._traces @@ -292,17 +458,39 @@ class BrowserPool { 'vitest.browser.session_id': sessionId, }, }, - () => this.openPage(sessionId, { parallel: workerCount > 1 }), + () => this.openPage(sessionId, { parallel }), + ) + .then( + () => { + this.openingPages-- + // start running tests on the page when it's ready + this.runNextTest(method, sessionId) + }, + (error) => { + this.openingPages-- + this.sessions.delete(sessionId) + throw error + }, ) - .then(() => { - // start running tests on the page when it's ready - this.runNextTest(method, sessionId) - }) promises.push(page) } await Promise.all(promises) - debug?.('all sessions are created') - return this._promise + } + + async stop(): Promise { + this.cancel() + const sessions = this.project.vitest._browserSessions + const orchestrators = Array.from(this.sessions, (sessionId) => { + const orchestrator = this.orchestrators.get(sessionId) + this.orchestrators.delete(sessionId) + sessions.destroySession(sessionId) + return orchestrator + }) + this.sessions.clear() + this.busySessions.clear() + this.readySessions.clear() + await closeProvider(this.project) + orchestrators.forEach((orchestrator) => orchestrator?.$close()) } private async openPage(sessionId: string, options: { parallel: boolean }): Promise { @@ -323,7 +511,7 @@ class BrowserPool { return session.concurrencyId } const used = new Set() - for (const id of this.orchestrators.keys()) { + for (const id of this.sessions) { const concurrencyId = sessions.getSession(id)?.concurrencyId if (concurrencyId) { used.add(concurrencyId) @@ -350,19 +538,20 @@ class BrowserPool { } private finishSession(sessionId: string): void { + this.busySessions.delete(sessionId) this.readySessions.add(sessionId) - // the last worker finished running tests - if (this.readySessions.size === this.orchestrators.size) { + // the last page finished running tests + if (!this.busySessions.size) { this._otel.span.end() this._promise?.resolve() this._promise = undefined debug?.('[%s] all tests finished running', sessionId) } else { debug?.( - `did not finish sessions for ${sessionId}: |ready - %s| |overall - %s|`, - [...this.readySessions].join(', '), - [...this.orchestrators.keys()].join(', '), + `did not finish sessions for ${sessionId}: |busy - %s| |overall - %s|`, + [...this.busySessions].join(', '), + [...this.sessions].join(', '), ) } } @@ -395,6 +584,7 @@ class BrowserPool { } const orchestrator = this.getOrchestrator(sessionId) + this.busySessions.add(sessionId) debug?.('[%s] run test %s', sessionId, file) // warm the transform cache while the iframe is booting so the test @@ -445,6 +635,7 @@ class BrowserPool { this.cancel() this._promise?.resolve() this._promise = undefined + this.busySessions.clear() debug?.('[%s] browser connection was closed', sessionId) return } @@ -492,7 +683,6 @@ function shouldIgnoreDebugger(provider: string, browser: string) { } return browser !== 'chromium' } - // Best-effort workaround for chromium/playwright bug // https://issues.chromium.org/issues/530892387 diff --git a/packages/vitest/src/node/types/browser.ts b/packages/vitest/src/node/types/browser.ts index bc72a54ca..0e3bba8e9 100644 --- a/packages/vitest/src/node/types/browser.ts +++ b/packages/vitest/src/node/types/browser.ts @@ -459,6 +459,7 @@ export interface ProjectBrowser { provider: BrowserProvider close: () => Promise initBrowserProvider: (project: TestProject) => Promise + closeBrowserProvider: () => Promise parseStacktrace: (stack: string) => ParsedStack[] parseErrorStacktrace: (error: TestError, options?: StackTraceParserOptions) => ParsedStack[] registerCommand: ( diff --git a/test/browser/specs/instance-concurrency.test.ts b/test/browser/specs/instance-concurrency.test.ts new file mode 100644 index 000000000..e90e3991e --- /dev/null +++ b/test/browser/specs/instance-concurrency.test.ts @@ -0,0 +1,145 @@ +import type { TestModule, Vitest } from 'vitest/node' +import { expect, test } from 'vitest' +import { instances, provider, runInlineBrowserTests } from './utils' + +const [{ browser }] = instances +const names = ['first', 'second', 'third'] + +function countOpenInstances(vitest: Vitest) { + return vitest.projects.filter((project) => project.browser!.state.orchestrators.size > 0).length +} + +function countOpenPages(project: TestModule['project']) { + return project.browser!.state.orchestrators.size +} + +function createFiles(body: string) { + return Object.fromEntries( + names.map((name) => [ + `${name}.test.ts`, + ` + import { test } from 'vitest' + test('runs in ${name}', async () => { + ${body} + }) + `, + ]), + ) +} + +function createInstances() { + return names.map((name) => ({ browser, name, include: [`${name}.test.ts`] })) +} + +test.runIf(provider.name === 'playwright')( + 'runs one browser instance at a time when maxWorkers is 1', + async () => { + const events: string[] = [] + const openInstances: number[] = [] + + const { stderr, testTree } = await runInlineBrowserTests( + createFiles('await new Promise(resolve => setTimeout(resolve, 100))'), + { + maxWorkers: 1, + browser: { instances: createInstances() }, + reporters: [ + { + onTestModuleStart(module: TestModule) { + events.push(`start ${module.project.name}`) + openInstances.push(countOpenInstances(module.project.vitest)) + }, + onTestModuleEnd(module: TestModule) { + events.push(`end ${module.project.name}`) + }, + }, + ], + }, + ) + + expect(stderr).toBe('') + expect(testTree()).toMatchInlineSnapshot(` + { + "first.test.ts": { + "runs in first": "passed", + }, + "second.test.ts": { + "runs in second": "passed", + }, + "third.test.ts": { + "runs in third": "passed", + }, + } + `) + + // every instance finishes before the next one opens + expect(events).toHaveLength(names.length * 2) + for (let i = 0; i < events.length; i += 2) { + const name = events[i].replace('start ', '') + expect(events[i + 1]).toBe(`end ${name}`) + } + expect(openInstances).toEqual([1, 1, 1]) + }, +) + +test.runIf(provider.name === 'playwright')( + 'runs up to maxWorkers browser instances at the same time', + async () => { + const openInstances: number[] = [] + const openPages: number[] = [] + + const { stderr, testTree } = await runInlineBrowserTests( + { + ...createFiles(` + const { commands } = await import('vitest/browser') + await vi.waitFor(async () => { + expect(await commands.openInstances()).toBeGreaterThanOrEqual(2) + }, { timeout: 10_000 }) + `), + // the third instance only runs after one of the others is finished + 'third.test.ts': ` + import { test } from 'vitest' + test('runs in third', () => {}) + `, + }, + { + maxWorkers: 2, + globals: true, + browser: { + instances: createInstances(), + commands: { + openInstances(context) { + return countOpenInstances(context.project.vitest) + }, + }, + }, + reporters: [ + { + onTestModuleStart(module: TestModule) { + openInstances.push(countOpenInstances(module.project.vitest)) + openPages.push(countOpenPages(module.project)) + }, + }, + ], + }, + ) + + expect(stderr).toBe('') + expect(testTree()).toMatchInlineSnapshot(` + { + "first.test.ts": { + "runs in first": "passed", + }, + "second.test.ts": { + "runs in second": "passed", + }, + "third.test.ts": { + "runs in third": "passed", + }, + } + `) + expect(Math.max(...openInstances)).toBe(2) + // the running instances split the budget of 2 pages, and the last + // instance has a single file, so none of them opens a second page + expect(openPages).toEqual([1, 1, 1]) + }, +) diff --git a/test/browser/specs/prewarm.test.ts b/test/browser/specs/prewarm.test.ts index 52d701335..fdf250de4 100644 --- a/test/browser/specs/prewarm.test.ts +++ b/test/browser/specs/prewarm.test.ts @@ -105,6 +105,27 @@ test('prewarm uses the provider from the resolved instance project', async () => expect(prewarmed).toEqual([{ browser: target, name: target }]) }) +test('prewarms at most maxWorkers instances', async () => { + const { prewarmed, spyProvider } = spyOnPrewarm() + const browser = instances[0].browser! + + const result = await runInlineBrowserTests( + { + 'basic.test.ts': basicTest, + }, + { + maxWorkers: 2, + browser: { + provider: spyProvider, + instances: ['first', 'second', 'third'].map((name) => ({ browser, name })), + }, + }, + ) + + expect(result.stderr).toBe('') + expect(prewarmed).toEqual(['first', 'second']) +}) + test('does not prewarm a project without test files', async () => { const { prewarmed, spyProvider } = spyOnPrewarm() let projectNames: string[] = [] diff --git a/test/browser/specs/utils.ts b/test/browser/specs/utils.ts index c4d899c6e..59993dc5d 100644 --- a/test/browser/specs/utils.ts +++ b/test/browser/specs/utils.ts @@ -6,6 +6,12 @@ import { instances, provider } from '../settings' export { instances, provider } from '../settings' +// every browser instance is a worker, so `runVitest`'s default of a single +// worker would run the instances one after another +function getMaxWorkers(instanceCount: number) { + return Math.max(instanceCount, 1) +} + export async function runInlineBrowserTests( structure: TestFsStructure, config?: RunVitestConfig, @@ -16,6 +22,7 @@ export async function runInlineBrowserTests( { watch: false, reporters: 'none', + maxWorkers: getMaxWorkers(config?.browser?.instances?.length ?? instances.length), ...config, browser: { enabled: true, @@ -39,6 +46,7 @@ export async function runBrowserTests( { watch: false, reporters: 'none', + maxWorkers: getMaxWorkers(config?.browser?.instances?.length ?? instances.length), ...config, browser: { headless: true, ...config?.browser }, $viteConfig: viteOverrides,