From bbcdb4d97df214a67a43bf583ca7fbc524f40f57 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 22 Jul 2026 10:37:18 +0200 Subject: [PATCH] refactor(benchmark): simplify provider contract to a default export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback on the pluggable benchmark provider API. The initial design mirrored the coverage provider (a `getProvider` factory plus a `BenchmarkProviderModule` wrapper), which was a special case that benchmarks — running only in tests — don't need. - Drop `BenchmarkProviderModule` and `getProvider`: a custom provider module now default-exports a `BenchmarkProvider` directly. - Reuse the existing `TestModuleRunner` interface instead of the bespoke `BenchmarkProviderLoader`. - Remove the `TestRunner.runBenchmarks` override hook, now redundant with the provider API, and inline `tinybench.run()` in the default provider. --- packages/vitest/src/node/types/benchmark.ts | 2 +- packages/vitest/src/public/index.ts | 1 - packages/vitest/src/runtime/benchmark.ts | 32 ++++--------- .../src/runtime/benchmark/default-provider.ts | 3 +- packages/vitest/src/runtime/runners/test.ts | 10 ---- test/e2e/test/benchmarking.test.ts | 48 ++++++++----------- 6 files changed, 32 insertions(+), 64 deletions(-) diff --git a/packages/vitest/src/node/types/benchmark.ts b/packages/vitest/src/node/types/benchmark.ts index 60118a2d7..a220cf1bb 100644 --- a/packages/vitest/src/node/types/benchmark.ts +++ b/packages/vitest/src/node/types/benchmark.ts @@ -32,7 +32,7 @@ export interface BenchmarkUserOptions { * The benchmark provider that executes registered benchmarks and produces * their results. Either the built-in `'default'` provider (backed by * tinybench) or a path to a module whose default export implements - * `BenchmarkProviderModule`. The path is resolved relative to the project + * `BenchmarkProvider`. The path is resolved relative to the project * root. * * @default 'default' diff --git a/packages/vitest/src/public/index.ts b/packages/vitest/src/public/index.ts index c560dcd5c..fb71cc5fc 100644 --- a/packages/vitest/src/public/index.ts +++ b/packages/vitest/src/public/index.ts @@ -44,7 +44,6 @@ export type { BenchHooks, BenchmarkGroup, BenchmarkProvider, - BenchmarkProviderModule, BenchRegistration, BenchRegistrationInput, BenchResult, diff --git a/packages/vitest/src/runtime/benchmark.ts b/packages/vitest/src/runtime/benchmark.ts index 541a3bdfc..204a658d2 100644 --- a/packages/vitest/src/runtime/benchmark.ts +++ b/packages/vitest/src/runtime/benchmark.ts @@ -7,6 +7,7 @@ import type { TaskResultTimestampProviderInfo, } from 'tinybench' import type { SerializedConfig } from './config' +import type { TestModuleRunner } from './moduleRunner/testModuleRunner' import type { BaselineData, Test, TestBenchmark, TestBenchmarkTask } from './runner/types' import { isAbsolute, relative } from 'pathe' import c from 'tinyrainbow' @@ -81,6 +82,8 @@ export interface BenchmarkGroup { /** * Executes the benchmarks of a single test and returns their results. * + * A custom provider module must default-export an object of this shape. + * * The returned array is the sole source of results: it's what `bench().run()` * resolves to and what the reporter serializes. Results are matched to * registrations by `name`; return one {@link BenchResult} per registration. @@ -91,30 +94,15 @@ export interface BenchmarkProvider { run: (group: BenchmarkGroup) => Promise } -/** - * The module contract a custom benchmark provider must satisfy. The module's - * default export must be an object of this shape. - * - * @experimental - */ -export interface BenchmarkProviderModule { - /** Factory that creates the provider used to run benchmarks. */ - getProvider: () => BenchmarkProvider | Promise -} - -interface BenchmarkProviderLoader { - import: (id: string) => Promise> -} - let cachedProvider: Promise | undefined async function loadProviderModule( provider: string, - loader: BenchmarkProviderLoader, -): Promise { + moduleRunner: TestModuleRunner, +): Promise { let mod: Record try { - mod = await loader.import(provider) + mod = await moduleRunner.import(provider) } catch (error) { throw new Error( @@ -137,13 +125,13 @@ async function loadProviderModule( */ export function resolveBenchmarkProvider( config: SerializedConfig, - loader: BenchmarkProviderLoader, + moduleRunner: TestModuleRunner, ): Promise { if (!cachedProvider) { const provider = config.benchmark.provider cachedProvider = provider === 'default' || !provider ? Promise.resolve(createDefaultBenchmarkProvider(config)) - : Promise.resolve(loadProviderModule(provider, loader)).then(mod => mod.getProvider()) + : loadProviderModule(provider, moduleRunner) } return cachedProvider } @@ -240,7 +228,7 @@ function substitutePath(template: string, projectName: string | undefined): stri export function createBench( test: Test, config: SerializedConfig, - loader: BenchmarkProviderLoader, + moduleRunner: TestModuleRunner, ): Bench { const pending = new Set>() @@ -352,7 +340,7 @@ export function createBench( const getterTracker = workerState.getterTracker getterTracker?.resetInvocations() try { - const provider = await resolveBenchmarkProvider(config, loader) + const provider = await resolveBenchmarkProvider(config, moduleRunner) const results = await provider.run({ test, config: config.benchmark, registrations, options }) const byName = new Map() for (const result of results) { diff --git a/packages/vitest/src/runtime/benchmark/default-provider.ts b/packages/vitest/src/runtime/benchmark/default-provider.ts index a4dd112e6..f5fb3f57a 100644 --- a/packages/vitest/src/runtime/benchmark/default-provider.ts +++ b/packages/vitest/src/runtime/benchmark/default-provider.ts @@ -2,7 +2,6 @@ import type { Task as TinybenchTask } from 'tinybench' import type { BenchmarkGroup, BenchmarkProvider, BenchResult } from '../benchmark' import type { SerializedConfig } from '../config' import { Bench as Tinybench } from 'tinybench' -import { TestRunner } from '../runners/test' const now = globalThis.performance ? globalThis.performance.now.bind(globalThis.performance) @@ -27,7 +26,7 @@ export function createDefaultBenchmarkProvider(config: SerializedConfig): Benchm for (const { name, fn, fnOpts } of registrations) { tinybench.add(name, fn, fnOpts) } - await TestRunner.runBenchmarks(tinybench) + await tinybench.run() const errors = tinybench.tasks .filter(task => task.result.state === 'errored') diff --git a/packages/vitest/src/runtime/runners/test.ts b/packages/vitest/src/runtime/runners/test.ts index dde78002b..0d5313c64 100644 --- a/packages/vitest/src/runtime/runners/test.ts +++ b/packages/vitest/src/runtime/runners/test.ts @@ -1,6 +1,5 @@ import type { SpanOptions } from '@opentelemetry/api' import type { ExpectStatic } from '@vitest/expect' -import type { Bench as Tinybench, Task as TinybenchTask } from 'tinybench' import type { ModuleRunner } from 'vite/module-runner' import type { Traces } from '../../utils/traces' import type { Bench } from '../benchmark' @@ -301,15 +300,6 @@ export class TestRunner implements VitestTestRunner { static setTestFn: typeof getFn = getFn static matchesTags: typeof matchesTags = matchesTags static createFileTask: typeof createFileTask = createFileTask - - /** - * @experimental - * A function that runs tinybench tasks. - * Can be overriden to run tasks in a special environment. - */ - static async runBenchmarks(tinybench: Tinybench): Promise { - return await tinybench.run() - } } function clearModuleMocks(config: SerializedConfig) { diff --git a/test/e2e/test/benchmarking.test.ts b/test/e2e/test/benchmarking.test.ts index f82d63318..646e31da8 100644 --- a/test/e2e/test/benchmarking.test.ts +++ b/test/e2e/test/benchmarking.test.ts @@ -1339,19 +1339,15 @@ test('`benchmark.provider` runs a custom provider whose returned results are aut } } export default { - getProvider() { - return { - async run({ registrations }) { - return registrations.map((reg, i) => ({ - name: reg.name, - state: 'completed', - latency: stats((i + 1) * 100), - throughput: stats(1 / ((i + 1) * 100)), - period: (i + 1) * 100, - totalTime: (i + 1) * 1000, - })) - }, - } + async run({ registrations }) { + return registrations.map((reg, i) => ({ + name: reg.name, + state: 'completed', + latency: stats((i + 1) * 100), + throughput: stats(1 / ((i + 1) * 100)), + period: (i + 1) * 100, + totalTime: (i + 1) * 1000, + })) }, } `, @@ -1399,22 +1395,18 @@ test('`benchmark.provider` receives the registrations with raw fn and fnOpts', a } } export default { - getProvider() { - return { - async run({ registrations }) { - const out = [] - for (const reg of registrations) { - await reg.fnOpts?.beforeAll?.() - await reg.fn() - await reg.fnOpts?.afterAll?.() - out.push({ - name: reg.name, state: 'completed', - latency: stats(1), throughput: stats(1), period: 1, totalTime: 1, - }) - } - return out - }, + async run({ registrations }) { + const out = [] + for (const reg of registrations) { + await reg.fnOpts?.beforeAll?.() + await reg.fn() + await reg.fnOpts?.afterAll?.() + out.push({ + name: reg.name, state: 'completed', + latency: stats(1), throughput: stats(1), period: 1, totalTime: 1, + }) } + return out }, } `, -- 2.51.2