diff --git a/.gitignore b/.gitignore index 8ca3bb427..f35a6decc 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ docs/public/sponsors docs/.vitepress/cache/ !test/cli/fixtures/dotted-files/**/.cache test/**/__screenshots__/**/* +test/**/__traces__/**/* test/browser/fixtures/update-snapshot/basic.test.ts test/cli/fixtures/browser-multiple/basic-* .vitest-reports diff --git a/.vscode/settings.json b/.vscode/settings.json index 1a220e193..14991480b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,7 +23,7 @@ // ], "vitest.ignoreWorkspace": true, - "vitest.configSearchPatternInclude": "test/{core,cli,config}/{vitest,vite}.config.ts", + "vitest.configSearchPatternInclude": "test/{core,cli,config,browser}/{vitest,vite}.{config.ts,config.unit.mts}", "testing.automaticallyOpenTestResults": "neverOpen", // Enable eslint for all supported languages diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index a811ccac2..72b358a9f 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -305,6 +305,11 @@ export default ({ mode }: { mode: string }) => { link: '/guide/browser/visual-regression-testing', docFooterText: 'Visual Regression Testing | Browser Mode', }, + { + text: 'Trace Viewer', + link: '/guide/browser/trace-viewer', + docFooterText: 'Trace Viewer | Browser Mode', + }, ], }, { diff --git a/docs/advanced/runner.md b/docs/advanced/runner.md index 52b1d342d..91d6bdcb5 100644 --- a/docs/advanced/runner.md +++ b/docs/advanced/runner.md @@ -27,19 +27,24 @@ export interface VitestRunner { /** * Called before running a single test. Doesn't have "result" yet. */ - onBeforeRunTask?: (test: TaskPopulated) => unknown + onBeforeRunTask?: (test: Test) => unknown /** * Called before actually running the test function. Already has "result" with "state" and "startTime". */ - onBeforeTryTask?: (test: TaskPopulated, options: { retry: number; repeats: number }) => unknown + onBeforeTryTask?: (test: Test, options: { retry: number; repeats: number }) => unknown /** * Called after result and state are set. */ - onAfterRunTask?: (test: TaskPopulated) => unknown + onAfterRunTask?: (test: Test) => unknown /** * Called right after running the test function. Doesn't have new state yet. Will not be called, if the test function throws. */ - onAfterTryTask?: (test: TaskPopulated, options: { retry: number; repeats: number }) => unknown + onAfterTryTask?: (test: Test, options: { retry: number; repeats: number }) => unknown + /** + * Called after the retry resolution happend. Unlike `onAfterTryTask`, the test now has a new state. + * All `after` hooks were also called by this point. + */ + onAfterRetryTask?: (test: Test, options: { retry: number; repeats: number }) => unknown /** * Called before running a single suite. Doesn't have "result" yet. diff --git a/docs/config/index.md b/docs/config/index.md index 0f615b6d8..f49f1e201 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -2439,7 +2439,7 @@ Polling timeout in milliseconds Always print console traces when calling any `console` method. This is useful for debugging. -### attachmentsDir 3.2.0 +### attachmentsDir 3.2.0 {#attachmentsdir} - **Type:** `string` - **Default:** `'.vitest-attachments'` diff --git a/docs/guide/browser/config.md b/docs/guide/browser/config.md index d5ea370cb..8db0cb7e1 100644 --- a/docs/guide/browser/config.md +++ b/docs/guide/browser/config.md @@ -128,7 +128,7 @@ A path to the HTML entry point. Can be relative to the root of the project. This Configure options for Vite server that serves code in the browser. Does not affect [`test.api`](#api) option. By default, Vitest assigns port `63315` to avoid conflicts with the development server, allowing you to run both in parallel. -## browser.provider advanced {#browser-provider} +## browser.provider {#browser-provider} - **Type:** `BrowserProviderOption` - **Default:** `'preview'` @@ -154,7 +154,7 @@ export default defineConfig({ To configure how provider initializes the browser, you can pass down options to the factory function: -```ts{7-15,22-27} +```ts{7-13,20-26} import { playwright } from '@vitest/browser/providers/playwright' export default defineConfig({ @@ -188,7 +188,7 @@ export default defineConfig({ }) ``` -### Custom Provider +### Custom Provider advanced ::: danger ADVANCED API The custom provider API is highly experimental and can change between patches. If you just need to run tests in a browser, use the [`browser.instances`](#browser-instances) option instead. @@ -307,6 +307,50 @@ The timeout in milliseconds. If connection to the browser takes longer, the test This is the time it should take for the browser to establish the WebSocket connection with the Vitest server. In normal circumstances, this timeout should never be reached. ::: +## browser.trace + +- **Type:** `'on' | 'off' | 'on-first-retry' | 'on-all-retries' | 'retain-on-failure' | object` +- **CLI:** `--browser.trace=on`, `--browser.trace=retain-on-failure` +- **Default:** `'off'` + +Capture a trace of your browser test runs. You can preview traces with [Playwright Trace Viewer](https://trace.playwright.dev/). + +This options supports the following values: + +- `'on'` - capture trace for all tests. (not recommended as it's performance heavy) +- `'off'` - do not capture traces. +- `'on-first-retry'` - capture trace only when retrying the test for the first time. +- `'on-all-retries'` - capture trace on every retry of the test. +- `'retain-on-failure'` - capture trace only for tests that fail. This will automatically delete traces for tests that pass. +- `object` - an object with the following shape: + +```ts +interface TraceOptions { + mode: 'on' | 'off' | 'on-first-retry' | 'on-all-retries' | 'retain-on-failure' + /** + * The directory where all traces will be stored. By default, Vitest + * stores all traces in `__traces__` folder close to the test file. + */ + tracesDir?: string + /** + * Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. + * @default true + */ + screenshots?: boolean + /** + * If this option is true tracing will + * - capture DOM snapshot on every action + * - record network activity + * @default true + */ + snapshots?: boolean +} +``` + +::: danger WARNING +This option is supported only by the [**playwright**](/guide/browser/playwright) provider. +::: + ## browser.trackUnhandledErrors - **Type:** `boolean` diff --git a/docs/guide/browser/playwright.md b/docs/guide/browser/playwright.md index c261f5fad..05f746724 100644 --- a/docs/guide/browser/playwright.md +++ b/docs/guide/browser/playwright.md @@ -18,7 +18,7 @@ export default defineConfig({ Vitest opens a single page to run all tests in the same file. You can configure the `launch`, `connect` and `context` when calling `playwright` at the top level or inside instances: -```ts{7-15,22-27} [vitest.config.js] +```ts{7-14,21-26} [vitest.config.js] import { playwright } from '@vitest/browser/providers/playwright' import { defineConfig } from 'vitest/config' diff --git a/docs/guide/browser/trace-viewer.md b/docs/guide/browser/trace-viewer.md new file mode 100644 index 000000000..2e8984dd3 --- /dev/null +++ b/docs/guide/browser/trace-viewer.md @@ -0,0 +1,74 @@ +# Trace Viewer + +Vitest Browser Mode supports generating Playwright's [trace files](https://playwright.dev/docs/trace-viewer#viewing-remote-traces). To enable tracing, you need to set the [`trace`](/guide/browser/config#browser-trace) option in the `test.browser` configuration. + +::: warning +Generating trace files is only available when using the [Playwright provider](/guide/browser/playwright). +::: + +::: code-group +```ts [vitest.config.js] +import { defineConfig } from 'vitest/config' +import { playwright } from '@vitest/browser/providers/playwright' + +export default defineConfig({ + test: { + browser: { + provider: playwright(), + trace: 'on', + }, + }, +}) +``` +```bash [CLI] +vitest --browser.trace=on +``` +::: + +By default, Vitest will generate a trace file for each test. You can also configure it to only generate traces on test failures by setting `trace` to `'on-first-retry'`, `'on-all-retries'` or `'retain-on-failure'`. The files will be saved in `__traces__` folder next to your test files. The name of the trace includes the project name, the test name, the [`repeats` count and `retry` count](/api/#test-api-reference): + +``` +chromium-my-test-0-0.trace.zip +^^^^^^^^ project name + ^^^^^^ test name + ^ repeat count + ^ retry count +``` + +To change the output directory, you can set the `tracesDir` option in the `test.browser.trace` configuration. This way all traces will be stored in the same directory, grouped by the test file. + +```ts [vitest.config.js] +import { defineConfig } from 'vitest/config' +import { playwright } from '@vitest/browser/providers/playwright' + +export default defineConfig({ + test: { + browser: { + provider: playwright(), + trace: { + mode: 'on', + // the path is relative to the root of the project + tracesDir: './playwright-traces', + }, + }, + }, +}) +``` + +The traces are available in reporters as [annotations](/guide/test-annotations). For example, in the HTML reporter, you can find the link to the trace file in the test details. + +## Preview + +To open the trace file, you can use the Playwright Trace Viewer. Run the following command in your terminal: + +```bash +npx playwright show-trace "path-to-trace-file" +``` + +This will start the Trace Viewer and load the specified trace file. + +Alternatively, you can open the Trace Viewer in your browser at https://trace.playwright.dev and upload the trace file there. + +## Limitations + +At the moment, Vitest cannot populate the "Sources" tab in the Trace Viewer. This means that while you can see the actions and screenshots captured during the test, you won't be able to view the source code of your tests directly within the Trace Viewer. You will need to refer back to your code editor to see the test implementation. diff --git a/docs/guide/cli-generated.md b/docs/guide/cli-generated.md index 994418d7a..12430a3b8 100644 --- a/docs/guide/cli-generated.md +++ b/docs/guide/cli-generated.md @@ -376,6 +376,13 @@ If connection to the browser takes longer, the test suite will fail (default: `6 Control if Vitest catches uncaught exceptions so they can be reported (default: `true`) +### browser.trace + +- **CLI:** `--browser.trace ` +- **Config:** [browser.trace](/guide/browser/config#browser-trace) + +Enable trace view mode. Supported: "on", "off", "on-first-retry", "on-all-retries", "retain-on-failure". + ### pool - **CLI:** `--pool ` diff --git a/packages/browser/src/client/tester/runner.ts b/packages/browser/src/client/tester/runner.ts index 2357ac279..b3dfd2c61 100644 --- a/packages/browser/src/client/tester/runner.ts +++ b/packages/browser/src/client/tester/runner.ts @@ -1,8 +1,20 @@ -import type { CancelReason, File, Suite, Task, TaskEventPack, TaskResultPack, Test, TestAnnotation, VitestRunner } from '@vitest/runner' +import type { + CancelReason, + File, + Suite, + Task, + TaskEventPack, + TaskResultPack, + Test, + TestAnnotation, + VitestRunner, +} from '@vitest/runner' import type { SerializedConfig, TestExecutionMethod, WorkerGlobalState } from 'vitest' import type { VitestBrowserClientMocker } from './mocker' +import type { CommandsManager } from './utils' import { globalChannel, onCancel } from '@vitest/browser/client' import { page, userEvent } from '@vitest/browser/context' +import { getTestName } from '@vitest/runner/utils' import { DecodedMap, getOriginalPosition, @@ -12,7 +24,7 @@ import { } from 'vitest/internal/browser' import { NodeBenchmarkRunner, VitestTestRunner } from 'vitest/runners' import { createStackString, parseStacktrace } from '../../../../utils/src/source-map' -import { getWorkerState, moduleRunner } from '../utils' +import { getBrowserState, getWorkerState, moduleRunner } from '../utils' import { rpc } from './rpc' import { VitestBrowserSnapshotEnvironment } from './snapshot' @@ -43,23 +55,91 @@ export function createBrowserRunner( hashMap = browserHashMap public sourceMapCache = new Map() public method = 'run' as TestExecutionMethod + private commands: CommandsManager constructor(options: BrowserRunnerOptions) { super(options.config) this.config = options.config + this.commands = getBrowserState().commands } setMethod(method: TestExecutionMethod) { this.method = method } + private traces = new Map() + onBeforeTryTask: VitestRunner['onBeforeTryTask'] = async (...args) => { await userEvent.cleanup() await super.onBeforeTryTask?.(...args) + const trace = this.config.browser.trace + const test = args[0] + if (trace === 'off') { + return + } + const { retry, repeats } = args[1] + if (trace === 'on-all-retries' && retry === 0) { + return + } + if (trace === 'on-first-retry' && retry !== 1) { + return + } + let title = getTestName(test) + if (retry) { + title += ` (retry x${retry})` + } + if (repeats) { + title += ` (repeat x${repeats})` + } + + const name = getTraceName(test, retry, repeats) + await this.commands.triggerCommand( + '__vitest_startChunkTrace', + [{ name, title }], + ) + } + + onAfterRetryTask = async (test: Test, { retry, repeats }: { retry: number; repeats: number }) => { + const trace = this.config.browser.trace + if (trace === 'off') { + return + } + if (trace === 'on-all-retries' && retry === 0) { + return + } + if (trace === 'on-first-retry' && retry !== 1) { + return + } + const name = getTraceName(test, retry, repeats) + if (!this.traces.has(test.id)) { + this.traces.set(test.id, []) + } + const traces = this.traces.get(test.id)! + const { tracePath } = await this.commands.triggerCommand( + '__vitest_stopChunkTrace', + [{ name }], + ) as { tracePath: string } + traces.push(tracePath) } onAfterRunTask = async (task: Test) => { await super.onAfterRunTask?.(task) + const trace = this.config.browser.trace + const traces = this.traces.get(task.id) || [] + if (traces.length) { + if (trace === 'retain-on-failure' && task.result?.state === 'pass') { + await this.commands.triggerCommand( + '__vitest_deleteTracing', + [{ traces }], + ) + } + else { + await this.commands.triggerCommand( + '__vitest_annotateTraces', + [{ testId: task.id, traces }], + ) + } + } if (this.config.bail && task.result?.state === 'fail') { const previousFailures = await rpc().getCountOfFailedTests() @@ -94,13 +174,14 @@ export function createBrowserRunner( await Promise.all([ super.onBeforeRunSuite?.(suite), (async () => { - if ('filepath' in suite) { - const map = await rpc().getBrowserFileSourceMap(suite.filepath) - this.sourceMapCache.set(suite.filepath, map) - const snapshotEnvironment = this.config.snapshotOptions.snapshotEnvironment - if (snapshotEnvironment instanceof VitestBrowserSnapshotEnvironment) { - snapshotEnvironment.addSourceMap(suite.filepath, map) - } + if (!('filepath' in suite)) { + return + } + const map = await rpc().getBrowserFileSourceMap(suite.filepath) + this.sourceMapCache.set(suite.filepath, map) + const snapshotEnvironment = this.config.snapshotOptions.snapshotEnvironment + if (snapshotEnvironment instanceof VitestBrowserSnapshotEnvironment) { + snapshotEnvironment.addSourceMap(suite.filepath, map) } })(), ]) @@ -174,7 +255,7 @@ export function createBrowserRunner( return rpc().onTaskUpdate(this.method, task, events) } - importFile = async (filepath: string) => { + importFile = async (filepath: string, mode: 'collect' | 'setup') => { let hash = this.hashMap.get(filepath) if (!hash) { hash = Date.now().toString() @@ -185,6 +266,11 @@ export function createBrowserRunner( const prefix = `/${/^\w:/.test(filepath) ? '@fs/' : ''}` const query = `browserv=${hash}` const importpath = `${prefix}${filepath}?${query}`.replace(/\/+/g, '/') + // start tracing before the test file is imported + const trace = this.config.browser.trace + if (mode === 'collect' && trace !== 'off') { + await this.commands.triggerCommand('__vitest_startTracing', []) + } try { await import(/* @vite-ignore */ importpath) } @@ -279,3 +365,8 @@ async function updateTestFilesLocations(files: File[], sourceMaps: Map = async ({ context, project, provider, sessionId }) => { + if (provider instanceof PlaywrightBrowserProvider) { + if (provider.tracingContexts.has(sessionId)) { + return + } + + provider.tracingContexts.add(sessionId) + const options = project.config.browser!.trace + await context.tracing.start({ + screenshots: options.screenshots ?? true, + snapshots: options.snapshots ?? true, + // currently, PW shows sources in private methods + sources: false, + }).catch(() => { + provider.tracingContexts.delete(sessionId) + }) + return + } + throw new TypeError(`The ${provider.name} provider does not support tracing.`) +} + +export const startChunkTrace: BrowserCommand<[{ name: string; title: string }]> = async ( + command, + { name, title }, +) => { + const { provider, sessionId, testPath, context } = command + if (!testPath) { + throw new Error(`stopChunkTrace cannot be called outside of the test file.`) + } + if (provider instanceof PlaywrightBrowserProvider) { + if (!provider.tracingContexts.has(sessionId)) { + await startTracing(command) + } + const path = resolveTracesPath(command, name) + provider.pendingTraces.set(path, sessionId) + await context.tracing.startChunk({ name, title }) + return + } + throw new TypeError(`The ${provider.name} provider does not support tracing.`) +} + +export const stopChunkTrace: BrowserCommand<[{ name: string }]> = async ( + context, + { name }, +) => { + if (context.provider instanceof PlaywrightBrowserProvider) { + const path = resolveTracesPath(context, name) + context.provider.pendingTraces.delete(path) + await context.context.tracing.stopChunk({ path }) + return { tracePath: path } + } + throw new TypeError(`The ${context.provider.name} provider does not support tracing.`) +} + +function resolveTracesPath({ testPath, project }: BrowserCommandContext, name: string) { + if (!testPath) { + throw new Error(`This command can only be called inside a test file.`) + } + const options = project.config.browser!.trace + const sanitizedName = `${project.name.replace(/[^a-z0-9]/gi, '-')}-${name}.trace.zip` + if (options.tracesDir) { + return resolve(options.tracesDir, sanitizedName) + } + const dir = dirname(testPath) + const base = basename(testPath) + return resolve( + dir, + '__traces__', + base, + `${project.name.replace(/[^a-z0-9]/gi, '-')}-${name}.trace.zip`, + ) +} + +export const deleteTracing: BrowserCommand<[{ traces: string[] }]> = async ( + context, + { traces }, +) => { + if (!context.testPath) { + throw new Error(`stopChunkTrace cannot be called outside of the test file.`) + } + if (context.provider instanceof PlaywrightBrowserProvider) { + return Promise.all( + traces.map(trace => unlink(trace).catch((err) => { + if (err.code === 'ENOENT') { + // Ignore the error if the file doesn't exist + return + } + // Re-throw other errors + throw err + })), + ) + } + + throw new Error(`provider ${context.provider.name} is not supported`) +} + +export const annotateTraces: BrowserCommand<[{ traces: string[]; testId: string }]> = async ( + { project }, + { testId, traces }, +) => { + const vitest = project.vitest + await Promise.all(traces.map((trace) => { + const entity = vitest.state.getReportedEntityById(testId) + return vitest._testRun.annotate(testId, { + message: relative(project.config.root, trace), + type: 'traces', + attachment: { + path: trace, + contentType: 'application/octet-stream', + }, + location: entity?.location + ? { + file: entity.module.moduleId, + line: entity.location.line, + column: entity.location.column, + } + : undefined, + }) + })) +} diff --git a/packages/browser/src/node/providers/playwright.ts b/packages/browser/src/node/providers/playwright.ts index 71befe530..5dcb59ffc 100644 --- a/packages/browser/src/node/providers/playwright.ts +++ b/packages/browser/src/node/providers/playwright.ts @@ -39,7 +39,10 @@ export interface PlaywrightProviderOptions { * The options passed down to [`playwright.connect`](https://playwright.dev/docs/api/class-browsertype#browser-type-launch) method. * @see {@link https://playwright.dev/docs/api/class-browsertype#browser-type-launch} */ - launchOptions?: LaunchOptions + launchOptions?: Omit< + LaunchOptions, + 'tracesDir' + > /** * The options passed down to [`playwright.connect`](https://playwright.dev/docs/api/class-browsertype#browser-type-connect) method. * @@ -91,12 +94,30 @@ export class PlaywrightBrowserProvider implements BrowserProvider { private browserPromise: Promise | null = null private closing = false + public tracingContexts: Set = new Set() + public pendingTraces: Map = new Map() + constructor( private project: TestProject, private options: PlaywrightProviderOptions, ) { this.browserName = project.config.browser.name as PlaywrightBrowser this.mocker = this.createMocker() + + // make sure the traces are finished if the test hangs + process.on('SIGTERM', () => { + if (!this.browser) { + return + } + const promises = [] + for (const [trace, contextId] of this.pendingTraces.entries()) { + promises.push(() => { + const context = this.contexts.get(contextId) + return context?.tracing.stopChunk({ path: trace }) + }) + } + return Promise.allSettled(promises) + }) } private async openBrowser() { @@ -131,10 +152,14 @@ export class PlaywrightBrowserProvider implements BrowserProvider { return this.browser } - const launchOptions = { + const launchOptions: LaunchOptions = { ...this.options.launchOptions, headless: options.headless, - } satisfies LaunchOptions + } + + if (typeof options.trace === 'object' && options.trace.tracesDir) { + launchOptions.tracesDir = options.trace?.tracesDir + } if (this.project.config.inspector.enabled) { // NodeJS equivalent defaults: https://nodejs.org/en/learn/getting-started/debugging#enable-inspector diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index fe00ee81a..4adf8f671 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -382,6 +382,11 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { test.onFailed = undefined test.onFinished = undefined + await runner.onAfterRetryTask?.(test, { + retry: retryCount, + repeats: repeatCount, + }) + // skipped with new PendingError if (test.result?.pending || test.result?.state === 'skip') { test.mode = 'skip' diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index 6ed48ea4f..895253e27 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -106,6 +106,14 @@ export interface VitestRunner { test: Test, options: { retry: number; repeats: number } ) => unknown + /** + * Called after the retry resolution happend. Unlike `onAfterTryTask`, the test now has a new state. + * All `after` hooks were also called by this point. + */ + onAfterRetryTask?: ( + test: Test, + options: { retry: number; repeats: number } + ) => unknown /** * Called before running a single suite. Doesn't have "result" yet. diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index 9f52ccb53..fe5de3737 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -418,6 +418,14 @@ export const cliOptionsConfig: VitestCLIOptions = { trackUnhandledErrors: { description: 'Control if Vitest catches uncaught exceptions so they can be reported (default: `true`)', }, + trace: { + description: 'Enable trace view mode. Supported: "on", "off", "on-first-retry", "on-all-retries", "retain-on-failure".', + argument: '', + subcommands: null, // don't support subcommands + transform(value) { + return { mode: value } + }, + }, orchestratorScripts: null, commands: null, viewport: null, diff --git a/packages/vitest/src/node/config/resolveConfig.ts b/packages/vitest/src/node/config/resolveConfig.ts index 864c8950b..9c2a3d0d2 100644 --- a/packages/vitest/src/node/config/resolveConfig.ts +++ b/packages/vitest/src/node/config/resolveConfig.ts @@ -786,6 +786,16 @@ export function resolveConfig( resolved.includeTaskLocation ??= true } + if (typeof resolved.browser.trace === 'string' || !resolved.browser.trace) { + resolved.browser.trace = { mode: resolved.browser.trace || 'off' } + } + if (resolved.browser.trace.tracesDir != null) { + resolved.browser.trace.tracesDir = resolvePath( + resolved.browser.trace.tracesDir, + resolved.root, + ) + } + const htmlReporter = toArray(resolved.reporters).some((reporter) => { if (Array.isArray(reporter)) { return reporter[0] === 'html' diff --git a/packages/vitest/src/node/config/serializeConfig.ts b/packages/vitest/src/node/config/serializeConfig.ts index 60d6cd738..9e956e079 100644 --- a/packages/vitest/src/node/config/serializeConfig.ts +++ b/packages/vitest/src/node/config/serializeConfig.ts @@ -156,6 +156,7 @@ export function serializeConfig(project: TestProject): SerializedConfig { } : {}, trackUnhandledErrors: browser.trackUnhandledErrors ?? true, + trace: browser.trace.mode, } })(config.browser), standalone: config.standalone, diff --git a/packages/vitest/src/node/printError.ts b/packages/vitest/src/node/printError.ts index da0131e76..4d829eeab 100644 --- a/packages/vitest/src/node/printError.ts +++ b/packages/vitest/src/node/printError.ts @@ -164,9 +164,10 @@ function printErrorInner( } printErrorMessage(e, logger) if (options.screenshotPaths?.length) { - const length = options.screenshotPaths.length + const uniqueScreenshots = Array.from(new Set(options.screenshotPaths)) + const length = uniqueScreenshots.length logger.error(`\nFailure screenshot${length > 1 ? 's' : ''}:`) - logger.error(options.screenshotPaths.map(p => ` - ${c.dim(relative(process.cwd(), p))}`).join('\n')) + logger.error(uniqueScreenshots.map(p => ` - ${c.dim(relative(process.cwd(), p))}`).join('\n')) if (!e.diff) { logger.error() } diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index aba68597c..b7c88409f 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -1,4 +1,4 @@ -import type { File, Task } from '@vitest/runner' +import type { File, Task, TestAnnotation } from '@vitest/runner' import type { SerializedError } from '@vitest/utils' import type { TestError, UserConsoleLog } from '../../types/general' import type { Vitest } from '../core' @@ -275,16 +275,29 @@ export abstract class BaseReporter implements Reporter { const PADDING = ' '.repeat(padding) - annotations.forEach(({ location, type, message }) => { + const groupedAnnotations: Record = {} + + annotations.forEach((annotation) => { + const { location, type } = annotation + let group: string if (location) { const file = relative(test.project.config.root, location.file) - this[console](`${PADDING}${c.blue(F_POINTER)} ${c.gray(`${file}:${location.line}:${location.column}`)} ${c.bold(type)}`) + group = `${c.gray(`${file}:${location.line}:${location.column}`)} ${c.bold(type)}` } else { - this[console](`${PADDING}${c.blue(F_POINTER)} ${c.bold(type)}`) + group = c.bold(type) } - this[console](`${PADDING} ${c.blue(F_DOWN_RIGHT)} ${message}`) + + groupedAnnotations[group] ??= [] + groupedAnnotations[group].push(annotation) }) + + for (const group in groupedAnnotations) { + this[console](`${PADDING}${c.blue(F_POINTER)} ${group}`) + groupedAnnotations[group].forEach(({ message }) => { + this[console](`${PADDING} ${c.blue(F_DOWN_RIGHT)} ${message}`) + }) + } } protected getEntityPrefix(entity: TestCase | TestModule | TestSuite): string { diff --git a/packages/vitest/src/node/state.ts b/packages/vitest/src/node/state.ts index 7a68fdfc0..a08044a26 100644 --- a/packages/vitest/src/node/state.ts +++ b/packages/vitest/src/node/state.ts @@ -211,6 +211,11 @@ export class StateManager { return this.reportedTasksMap.get(task) } + getReportedEntityById(taskId: string): TestModule | TestCase | TestSuite | undefined { + const task = this.idMap.get(taskId) + return task ? this.reportedTasksMap.get(task) : undefined + } + updateTasks(packs: TaskResultPack[]): void { for (const [id, result, meta] of packs) { const task = this.idMap.get(id) diff --git a/packages/vitest/src/node/test-run.ts b/packages/vitest/src/node/test-run.ts index f797457a2..a4a5b2f14 100644 --- a/packages/vitest/src/node/test-run.ts +++ b/packages/vitest/src/node/test-run.ts @@ -16,12 +16,13 @@ import type { TestSpecification } from './spec' import type { TestRunEndReason } from './types/reporter' import assert from 'node:assert' import { createHash } from 'node:crypto' +import { existsSync } from 'node:fs' import { copyFile, mkdir } from 'node:fs/promises' import { isPrimitive } from '@vitest/utils/helpers' import { serializeValue } from '@vitest/utils/serialize' import { parseErrorStacktrace } from '@vitest/utils/source-map' import mime from 'mime/lite' -import { basename, dirname, extname, resolve } from 'pathe' +import { basename, extname, resolve } from 'pathe' export class TestRun { constructor(private vitest: Vitest) {} @@ -226,7 +227,9 @@ export class TestRun { project.config.attachmentsDir, `${sanitizeFilePath(annotation.message)}-${hash}${extname(currentPath)}`, ) - await mkdir(dirname(newPath), { recursive: true }) + if (!existsSync(project.config.attachmentsDir)) { + await mkdir(project.config.attachmentsDir, { recursive: true }) + } await copyFile(currentPath, newPath) attachment.path = newPath diff --git a/packages/vitest/src/node/types/browser.ts b/packages/vitest/src/node/types/browser.ts index 65e881599..4a497c544 100644 --- a/packages/vitest/src/node/types/browser.ts +++ b/packages/vitest/src/node/types/browser.ts @@ -3,6 +3,7 @@ import type { CancelReason } from '@vitest/runner' import type { Awaitable, ParsedStack, TestError } from '@vitest/utils' import type { StackTraceParserOptions } from '@vitest/utils/source-map' import type { ViteDevServer } from 'vite' +import type { BrowserTraceViewMode } from '../../runtime/config' import type { BrowserTesterOptions } from '../../types/browser' import type { TestProject } from '../project' import type { ApiConfig, ProjectConfig } from './config' @@ -168,6 +169,32 @@ export interface BrowserConfigOptions { testIdAttribute?: string } + /** + * Generate traces that can be viewed on https://trace.playwright.dev/ + * + * This option is supported only by **playwright** provider. + */ + trace?: BrowserTraceViewMode | { + mode: BrowserTraceViewMode + /** + * The directory where all traces will be stored. By default, Vitest + * stores all traces in `__traces__` folder close to the test file. + */ + tracesDir?: string + /** + * Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. + * @default true + */ + screenshots?: boolean + /** + * If this option is true tracing will + * - capture DOM snapshot on every action + * - record network activity + * @default true + */ + snapshots?: boolean + } + /** * Directory where screenshots will be saved when page.screenshot() is called * If not set, all screenshots are saved to __screenshots__ directory in the same folder as the test file. @@ -269,7 +296,7 @@ export interface ProjectBrowser { parseErrorStacktrace: (error: TestError, options?: StackTraceParserOptions) => ParsedStack[] } -export interface BrowserCommand { +export interface BrowserCommand { (context: BrowserCommandContext, ...payload: Payload): Awaitable } @@ -318,6 +345,14 @@ export interface ResolvedBrowserOptions extends BrowserConfigOptions { locators: { testIdAttribute: string } + trace: { + mode: BrowserTraceViewMode + tracesDir?: string + screenshots?: boolean + snapshots?: boolean + // TODO: map locations to test ones + // sources?: boolean + } } type ToMatchScreenshotResolvePath = (data: { diff --git a/packages/vitest/src/runtime/config.ts b/packages/vitest/src/runtime/config.ts index 98acd25e3..8cfbb3918 100644 --- a/packages/vitest/src/runtime/config.ts +++ b/packages/vitest/src/runtime/config.ts @@ -122,6 +122,7 @@ export interface SerializedConfig { // for playwright actionTimeout?: number } + trace: BrowserTraceViewMode trackUnhandledErrors: boolean } standalone: boolean @@ -162,3 +163,4 @@ export type RuntimeConfig = Pick< } export type RuntimeOptions = Partial +export type BrowserTraceViewMode = 'on' | 'off' | 'on-first-retry' | 'on-all-retries' | 'retain-on-failure' diff --git a/test/browser/fixtures/trace-view/basic.test.ts b/test/browser/fixtures/trace-view/basic.test.ts new file mode 100644 index 000000000..8bcd3e188 --- /dev/null +++ b/test/browser/fixtures/trace-view/basic.test.ts @@ -0,0 +1,31 @@ +import { describe, test } from 'vitest'; + +test('a single test', () => { + // ... +}) + +test('repeated test', { repeats: 2 }, () => { + // ... +}) + +test('retried test', { retry: 2 }, ({ task }) => { + if (task.result?.retryCount !== 2) { + throw new Error('failed test') + } +}) + +test('repeated retried tests', { repeats: 2, retry: 2 }, ({ task }) => { + if (task.result?.retryCount !== 2) { + throw new Error('failed test') + } +}) + +describe('nested suite', () => { + test('suite test', () => { + // ... + }) + + test.skip('skipped test', () => { + // ... + }) +}) diff --git a/test/browser/fixtures/trace-view/failing.special.ts b/test/browser/fixtures/trace-view/failing.special.ts new file mode 100644 index 000000000..3d0a29042 --- /dev/null +++ b/test/browser/fixtures/trace-view/failing.special.ts @@ -0,0 +1,13 @@ +import { test } from 'vitest'; + +test('fail', () => { + throw new Error('fail') +}) + +test('retried fail', { retry: 2 }, () => { + throw new Error('fail') +}) + +test('repeated fail', { repeats: 2 }, () => { + throw new Error('fail') +}) diff --git a/test/browser/fixtures/trace-view/vitest.config.ts b/test/browser/fixtures/trace-view/vitest.config.ts new file mode 100644 index 000000000..e7a19d7a1 --- /dev/null +++ b/test/browser/fixtures/trace-view/vitest.config.ts @@ -0,0 +1,20 @@ +import { fileURLToPath } from 'node:url' +import { defineConfig } from 'vitest/config' +import { playwright } from '@vitest/browser/providers/playwright' + +export default defineConfig({ + cacheDir: fileURLToPath(new URL("./node_modules/.vite", import.meta.url)), + test: { + browser: { + enabled: true, + provider: playwright(), + instances: [ + { browser: 'chromium' }, + { browser: 'firefox' }, + { browser: 'webkit' }, + ], + headless: true, + trace: 'off', + }, + }, +}) diff --git a/test/browser/specs/playwright-trace.test.ts b/test/browser/specs/playwright-trace.test.ts new file mode 100644 index 000000000..7932b056a --- /dev/null +++ b/test/browser/specs/playwright-trace.test.ts @@ -0,0 +1,191 @@ +import { readdirSync, rmSync } from 'node:fs' +import { resolve } from 'pathe' +import { afterEach, describe, expect, test } from 'vitest' +import { provider, runBrowserTests } from './utils' + +const tracesFolder = resolve(import.meta.dirname, '../fixtures/trace-view/__traces__') +const basicTestTracesFolder = resolve(tracesFolder, 'basic.test.ts') + +describe.runIf(provider.name === 'playwright')('playwright tracing', () => { + afterEach(() => { + rmSync(tracesFolder, { recursive: true, force: true }) + }) + + test('vitest generates trace files when running with `on`', async () => { + const { stderr, ctx } = await runBrowserTests({ + root: './fixtures/trace-view', + browser: { + trace: 'on', + }, + includeTaskLocation: true, + }) + + expect(stderr).toBe('') + expect(readdirSync(tracesFolder)).toEqual(['basic.test.ts']) + expect(readdirSync(basicTestTracesFolder).sort()).toMatchInlineSnapshot(` + [ + "chromium-a-single-test-0-0.trace.zip", + "chromium-nested-suite-suite-test-0-0.trace.zip", + "chromium-repeated-retried-tests-0-0.trace.zip", + "chromium-repeated-retried-tests-0-1.trace.zip", + "chromium-repeated-retried-tests-0-2.trace.zip", + "chromium-repeated-retried-tests-1-0.trace.zip", + "chromium-repeated-retried-tests-2-0.trace.zip", + "chromium-repeated-test-0-0.trace.zip", + "chromium-repeated-test-1-0.trace.zip", + "chromium-repeated-test-2-0.trace.zip", + "chromium-retried-test-0-0.trace.zip", + "chromium-retried-test-0-1.trace.zip", + "chromium-retried-test-0-2.trace.zip", + "firefox-a-single-test-0-0.trace.zip", + "firefox-nested-suite-suite-test-0-0.trace.zip", + "firefox-repeated-retried-tests-0-0.trace.zip", + "firefox-repeated-retried-tests-0-1.trace.zip", + "firefox-repeated-retried-tests-0-2.trace.zip", + "firefox-repeated-retried-tests-1-0.trace.zip", + "firefox-repeated-retried-tests-2-0.trace.zip", + "firefox-repeated-test-0-0.trace.zip", + "firefox-repeated-test-1-0.trace.zip", + "firefox-repeated-test-2-0.trace.zip", + "firefox-retried-test-0-0.trace.zip", + "firefox-retried-test-0-1.trace.zip", + "firefox-retried-test-0-2.trace.zip", + "webkit-a-single-test-0-0.trace.zip", + "webkit-nested-suite-suite-test-0-0.trace.zip", + "webkit-repeated-retried-tests-0-0.trace.zip", + "webkit-repeated-retried-tests-0-1.trace.zip", + "webkit-repeated-retried-tests-0-2.trace.zip", + "webkit-repeated-retried-tests-1-0.trace.zip", + "webkit-repeated-retried-tests-2-0.trace.zip", + "webkit-repeated-test-0-0.trace.zip", + "webkit-repeated-test-1-0.trace.zip", + "webkit-repeated-test-2-0.trace.zip", + "webkit-retried-test-0-0.trace.zip", + "webkit-retried-test-0-1.trace.zip", + "webkit-retried-test-0-2.trace.zip", + ] + `) + + // traces are also stored in attachments so they are visible in all reporters + const testModules = ctx.state.getTestModules() + expect(testModules).toHaveLength(3) + testModules.forEach((testModule) => { + for (const test of testModule.children.allTests()) { + if (test.result().state === 'skipped') { + continue + } + + const annotations = test.annotations() + expect(annotations.length).toBeGreaterThan(0) + + annotations.forEach((annotation) => { + expect(annotation.message).toContain('basic.test.ts/') + expect(annotation.type).toBe('traces') + expect(annotation.attachment!.contentType).toBe('application/octet-stream') + expect(annotation.location).toEqual({ + file: test.module.moduleId, + line: test.location.line, + column: test.location.column, + }) + }) + } + }) + }) + + test('vitest generates trace files when running with `on-all-retries`', async () => { + const { stderr } = await runBrowserTests({ + root: './fixtures/trace-view', + browser: { + trace: 'on-all-retries', + }, + }) + + expect(stderr).toBe('') + expect(readdirSync(tracesFolder)).toEqual(['basic.test.ts']) + expect(readdirSync(basicTestTracesFolder).sort()).toMatchInlineSnapshot(` + [ + "chromium-repeated-retried-tests-0-1.trace.zip", + "chromium-repeated-retried-tests-0-2.trace.zip", + "chromium-retried-test-0-1.trace.zip", + "chromium-retried-test-0-2.trace.zip", + "firefox-repeated-retried-tests-0-1.trace.zip", + "firefox-repeated-retried-tests-0-2.trace.zip", + "firefox-retried-test-0-1.trace.zip", + "firefox-retried-test-0-2.trace.zip", + "webkit-repeated-retried-tests-0-1.trace.zip", + "webkit-repeated-retried-tests-0-2.trace.zip", + "webkit-retried-test-0-1.trace.zip", + "webkit-retried-test-0-2.trace.zip", + ] + `) + }) + + test('vitest generates trace files when running with `on-first-retries`', async () => { + const { stderr } = await runBrowserTests({ + root: './fixtures/trace-view', + browser: { + trace: 'on-first-retry', + }, + }) + + expect(stderr).toBe('') + expect(readdirSync(tracesFolder)).toEqual(['basic.test.ts']) + expect(readdirSync(basicTestTracesFolder).sort()).toMatchInlineSnapshot(` + [ + "chromium-repeated-retried-tests-0-1.trace.zip", + "chromium-retried-test-0-1.trace.zip", + "firefox-repeated-retried-tests-0-1.trace.zip", + "firefox-retried-test-0-1.trace.zip", + "webkit-repeated-retried-tests-0-1.trace.zip", + "webkit-retried-test-0-1.trace.zip", + ] + `) + }) + + test('vitest generates trace files when running with `retain-on-failure`', async () => { + const { stderr } = await runBrowserTests({ + root: './fixtures/trace-view', + include: ['./*.test.ts', './*.special.ts'], + browser: { + trace: 'retain-on-failure', + }, + }) + + const failingTestTracesFolder = resolve(tracesFolder, 'failing.special.ts') + + expect(readdirSync(tracesFolder)).toEqual([ + 'basic.test.ts', + 'failing.special.ts', + ]) + expect(readdirSync(basicTestTracesFolder).sort()).toMatchInlineSnapshot(`[]`) + expect(readdirSync(failingTestTracesFolder).sort()).toMatchInlineSnapshot(` + [ + "chromium-fail-0-0.trace.zip", + "chromium-repeated-fail-0-0.trace.zip", + "chromium-repeated-fail-1-0.trace.zip", + "chromium-repeated-fail-2-0.trace.zip", + "chromium-retried-fail-0-0.trace.zip", + "chromium-retried-fail-0-1.trace.zip", + "chromium-retried-fail-0-2.trace.zip", + "firefox-fail-0-0.trace.zip", + "firefox-repeated-fail-0-0.trace.zip", + "firefox-repeated-fail-1-0.trace.zip", + "firefox-repeated-fail-2-0.trace.zip", + "firefox-retried-fail-0-0.trace.zip", + "firefox-retried-fail-0-1.trace.zip", + "firefox-retried-fail-0-2.trace.zip", + "webkit-fail-0-0.trace.zip", + "webkit-repeated-fail-0-0.trace.zip", + "webkit-repeated-fail-1-0.trace.zip", + "webkit-repeated-fail-2-0.trace.zip", + "webkit-retried-fail-0-0.trace.zip", + "webkit-retried-fail-0-1.trace.zip", + "webkit-retried-fail-0-2.trace.zip", + ] + `) + + // the default reporter outputs attachments + expect(stderr).toContain('❯ traces') + expect(stderr).toContain('↳ __traces__/failing.special.ts/') + }) +})