diff --git a/docs/config/browser/traceview.md b/docs/config/browser/traceview.md index a186eded7..fb8b38b1c 100644 --- a/docs/config/browser/traceview.md +++ b/docs/config/browser/traceview.md @@ -9,7 +9,7 @@ outline: deep - **CLI:** `--browser.traceView` - **Default:** `false` -Enable trace-view collection for browser tests. Vitest captures DOM snapshots for browser interactions and can show them in the browser UI, Vitest UI, or HTML reporter when those surfaces are enabled — no external tools required. +Enable [trace view](/guide/browser/trace-view) collection for browser tests. Vitest captures DOM snapshots for browser interactions and can show them in the browser UI, Vitest UI, or HTML reporter when those surfaces are enabled — no external tools required. When a test fails, the terminal output also includes a [timeline of its recorded steps](/guide/browser/trace-view#failure-output). ```ts export default defineConfig({ diff --git a/docs/guide/browser/snippets/trace-failure.ansi b/docs/guide/browser/snippets/trace-failure.ansi new file mode 100644 index 000000000..eebc87e5c --- /dev/null +++ b/docs/guide/browser/snippets/trace-failure.ansi @@ -0,0 +1,24 @@ + FAIL src/sign-in.test.ts > signs in +Error: expect(element).toHaveTextContent() + +Expected element to have text content: + Welcome +Received: + Sign in + + ❯ src/sign-in.test.ts:11:51 +  9| }) +  10| await page.getByRole('button', { name: 'Sign in' }).click() +  11| await expect.element(page.getByRole('heading')).toHaveTextContent('Welcome') +  | ^ +  12| }) + +Caused by: Error: Matcher did not succeed in time. + +Trace: + step locator start duration location + fill form -  +1ms 58ms src/sign-in.test.ts:6:14 + ├── fill getByLabel('Email')  +2ms 31ms src/sign-in.test.ts:7:40 + └── fill getByLabel('Password')  +34ms 24ms src/sign-in.test.ts:8:43 + click getByRole('button', { name: 'Sign in' }) +61ms 18ms src/sign-in.test.ts:10:55 + toHaveTextContent [ERROR] getByRole('heading')  +80ms 1003ms src/sign-in.test.ts:11:16 diff --git a/docs/guide/browser/trace-view.md b/docs/guide/browser/trace-view.md index e4863f101..bbe0a3ee7 100644 --- a/docs/guide/browser/trace-view.md +++ b/docs/guide/browser/trace-view.md @@ -2,7 +2,7 @@ `browser.traceView` records browser interactions as DOM snapshots and lets you replay them step by step in Vitest's built-in trace viewer. It is useful when the live browser view is not enough: you can inspect earlier tests, failed retries, screenshots, assertions, and user actions after the browser has already moved on. -Trace view is additive to the current browser testing workflow. Enabling it does not force a single debugging mode. You can use it with the normal local browser UI, with a headless browser and Vitest UI, or with the HTML reporter in CI. +Trace view is additive to the current browser testing workflow. Enabling it does not force a single debugging mode. You can use it with the normal local browser UI, with a headless browser and Vitest UI, or with the HTML reporter in CI. When a test fails, the terminal output also includes a text timeline of its recorded steps. ::: tip Trace view, browser UI, and HTML reports @@ -44,7 +44,7 @@ vitest --browser.traceView ::: -When `browser.traceView` is enabled, tests with recorded traces can be opened in the trace viewer from the [browser UI](/config/browser/ui), [Vitest UI](/guide/ui), and [HTML reporter](/guide/reporters#html-reporter). +When `browser.traceView` is enabled, tests with recorded traces can be opened in the trace viewer from the [browser UI](/config/browser/ui), [Vitest UI](/guide/ui), and [HTML reporter](/guide/reporters#html-reporter). Failed tests also print their recorded steps in the terminal, as described in [Failure Output](#failure-output). With the HTML reporter enabled, the [Vitest VS Code extension](https://github.com/vitest-dev/vscode#trace-view) adds an **Open Trace View** action to tests in the Testing view and editor gutter. @@ -81,6 +81,7 @@ TODO: The browser UI / Vitest UI / browser driver combinations are not specific | Debug locally with a headless browser | `vitest --browser.traceView --browser.headless --ui` | The browser runs headless, while Vitest UI shows recorded trace steps and snapshots. | | Debug locally with a visible browser window and Vitest UI | `vitest --browser.traceView --browser.headless=false --browser.ui=false --ui` | Vitest UI shows recorded trace steps and snapshots, while tests run in a separate headed browser window. | | Generate a static report for CI or run mode | `vitest run --browser.traceView --reporter=html` | The HTML report includes the trace viewer for recorded tests. | +| See failure steps in CI logs or from a coding agent | `vitest run --browser.traceView` | The failure summary in the terminal includes a timeline of the steps that led to each failure. | ## Relation to Playwright Traces @@ -107,6 +108,14 @@ Each entry captures the DOM state at that point, along with timing information, In Vitest UI, trace entries are streamed as the test runs, so you can inspect recorded steps before the test finishes. Long-running actions, `expect.element(...)` assertions, and callback `page.mark()` entries appear as in-progress steps first, then update with their final status and duration. +## Failure Output + +When a test fails, the error in the terminal is followed by a timeline of its recorded steps: + +<<< ./snippets/trace-failure.ansi [Error report] + +These are the same steps the trace viewer lists, without DOM snapshots. Only the last 10 steps are printed, so open the trace viewer to see the full trace. + ## Custom Trace Entries You can insert your own named entries with `page.mark()` and `locator.mark()`: diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 97c739b9e..d790443bf 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -44,6 +44,7 @@ import { renderBenchmarkRow, } from './renderers/benchmark-table' import { F_CHECK, F_DOWN_RIGHT, F_POINTER } from './renderers/figures' +import { formatTraceTimeline } from './renderers/trace-timeline' import { countTestErrors, divider, @@ -1529,6 +1530,16 @@ export abstract class BaseReporter implements Reporter { screenshotPaths, }) + // TODO: print the trace of the attempt that produced this error instead of the last attempt, + // using `task.result.attempts` (https://github.com/vitest-dev/vitest/pull/11232) + for (const task of tasks) { + const timeline = + task.type === 'test' && formatTraceTimeline(task.artifacts, this.ctx.config.root) + if (timeline) { + this.error(`${timeline}\n`) + } + } + if (tasks[0].type === 'test' && tasks[0].annotations.length) { const test = this.ctx.state.getReportedEntity(tasks[0]) as TestCase this.printAnnotations(test, 'error', 1) diff --git a/packages/vitest/src/node/reporters/renderers/trace-timeline.ts b/packages/vitest/src/node/reporters/renderers/trace-timeline.ts new file mode 100644 index 000000000..a8680b9b8 --- /dev/null +++ b/packages/vitest/src/node/reporters/renderers/trace-timeline.ts @@ -0,0 +1,207 @@ +import type { TestArtifact } from '../../../runtime/runner/types' +import { relative } from 'pathe' +import c from 'tinyrainbow' +import { F_TREE_NODE_END, F_TREE_NODE_MIDDLE } from './figures' + +interface TimelineRow { + cells: [name: string, locator: string, start: string, duration: string, location: string] + status?: 'pass' | 'fail' +} + +const MAX_STEPS = 10 + +export function formatTraceTimeline(artifacts: TestArtifact[], root: string): string | undefined { + const attempt = [...getTraceAttemptMap(artifacts).values()].at(-1) + if (!attempt) { + return + } + + // the test end is already shown by the error location + const visible = attempt.entries.filter((step) => step.name !== 'vitest:onAfterRetryTask') + if (!visible.length) { + return + } + const shown = visible.slice(-MAX_STEPS) + const prefixes = getTreePrefixes(shown) + const headerRow: TimelineRow = { cells: ['step', 'locator', 'start', 'duration', 'location'] } + const rows = [headerRow, ...shown.map((step, i) => formatStep(step, prefixes[i], root))] + + const widths = headerRow.cells.map((_, i) => Math.max(...rows.map((row) => row.cells[i].length))) + const lines = rows.map((row) => { + // the last column is left unpadded so rows have no trailing spaces inside its color codes + const [name, locator, start, duration, location] = row.cells.map((cell, i) => + i === 4 ? cell : i === 2 || i === 3 ? cell.padStart(widths[i]) : cell.padEnd(widths[i]), + ) + if (row === headerRow) { + return c.dim(` ${name} ${locator} ${start} ${duration} ${location}`) + } + const text = ` ${name} ${colorCell(locator, c.cyan)} ${start} ${colorCell(duration)} ${colorCell(location, c.dim)}` + return row.status === 'fail' ? c.red(text) : text + }) + const omitted = visible.length - shown.length + if (omitted) { + lines.splice(1, 0, c.dim(` ... ${omitted} earlier ${omitted === 1 ? 'step' : 'steps'}`)) + } + + const header: string[] = [] + if (attempt.retry) { + header.push(`retry ${attempt.retry}`) + } + if (attempt.repeats) { + header.push(`repeat ${attempt.repeats}`) + } + const title = header.length ? `Trace (${header.join(', ')}):` : 'Trace:' + return [`\n${title}`, ...lines].join('\n') +} + +function getTreePrefixes(steps: NormalizedBrowserTraceEntry[]): string[] { + const hasNextSibling = steps.map((step, i) => { + const next = steps.slice(i + 1).find((other) => other.depth <= step.depth) + return next?.depth === step.depth + }) + const lastAtDepth: number[] = [] + return steps.map((step, i) => { + lastAtDepth[step.depth] = i + if (step.depth === 0) { + return '' + } + let prefix = '' + for (let level = 1; level < step.depth; level++) { + prefix += hasNextSibling[lastAtDepth[level]] ? '│ ' : ' ' + } + return `${prefix}${hasNextSibling[i] ? F_TREE_NODE_MIDDLE : F_TREE_NODE_END} ` + }) +} + +function formatStep(step: NormalizedBrowserTraceEntry, prefix: string, root: string): TimelineRow { + const location = + step.location && + `${relative(root, step.location.file)}:${step.location.line}:${step.location.column}` + return { + cells: [ + `${prefix}${formatStepName(step.name)}`, + step.element?.locator || '-', + `+${formatMs(step.startTime)}`, + step.duration == null ? '-' : formatMs(step.duration), + location || '-', + ], + status: step.status, + } +} + +function colorCell(cell: string, format = (text: string) => text): string { + return cell.trim() === '-' ? c.dim(cell) : format(cell) +} + +function formatMs(ms: number): string { + return `${Math.round(ms)}ms` +} + +function formatStepName(name: string): string { + return name.startsWith('vitest:') ? name.slice('vitest:'.length) : name +} + +// TODO: everything below is copied from `BrowserTraceData`/`BrowserTraceEntry` in +// packages/browser/src/client/tester/trace.ts (only the fields used here) and +// `getTraceAttemptMap` in packages/ui/client/composables/trace-view.ts. +// share them instead (https://github.com/vitest-dev/vitest/pull/11604) + +interface BrowserTraceData { + retry: number + repeats: number + entries: BrowserTraceEntry[] +} + +interface BrowserTraceEntry { + name: string + range?: { id: string; phase: 'start' | 'end' } + status?: 'pass' | 'fail' + startTime: number + duration?: number + location?: { file: string; line: number; column: number } + element?: { locator: string } +} + +interface NormalizedBrowserTraceData extends BrowserTraceData { + entries: NormalizedBrowserTraceEntry[] +} + +interface NormalizedBrowserTraceEntry extends BrowserTraceEntry { + depth: number +} + +function getTraceAttemptKey(trace: BrowserTraceData): string { + return `${trace.repeats}:${trace.retry}` +} + +function normalizeTraceEntries(entries: BrowserTraceEntry[]): NormalizedBrowserTraceEntry[] { + const merged: NormalizedBrowserTraceEntry[] = [] + const startMap = new Map() + const parents: string[] = [] + + for (const entry of entries) { + const range = entry.range + if (!range) { + merged.push({ + ...entry, + depth: parents.length, + }) + continue + } + + if (range.phase === 'start') { + startMap.set(range.id, merged.length) + merged.push({ + ...entry, + depth: parents.length, + }) + parents.push(range.id) + continue + } + + // when range.phase === 'end' + const index = startMap.get(range.id) + if (index == null) { + // unpaired range shouldn't happen but just leave it there + merged.push({ ...entry, depth: 0 }) + continue + } + + // Keep the start timestamp for positioning, but derive display duration + // from the end event timestamp. + const start = merged[index] + merged[index] = { + ...start, + ...entry, + startTime: start.startTime, + duration: entry.startTime - start.startTime, + } + parents.pop() + } + + return merged +} + +function getTraceAttemptMap(artifacts: TestArtifact[]): Map { + const grouped: Record = {} + for (const artifact of artifacts) { + if (artifact.type !== 'internal:browserTrace') { + continue + } + const trace = artifact.data as BrowserTraceData + const key = getTraceAttemptKey(trace) + grouped[key] ??= [] + grouped[key].push(trace) + } + + const merged = new Map() + for (const [key, traces] of Object.entries(grouped)) { + const trace = traces[0] + const entries = traces.flatMap((trace) => trace.entries) + merged.set(key, { + ...trace, + entries: normalizeTraceEntries(entries), + }) + } + return merged +} diff --git a/test/browser/specs/trace-timeline.test.ts b/test/browser/specs/trace-timeline.test.ts new file mode 100644 index 000000000..2d75d79eb --- /dev/null +++ b/test/browser/specs/trace-timeline.test.ts @@ -0,0 +1,164 @@ +import type { TestFsStructure } from '../../test-utils' +import { expect, test } from 'vitest' +import { instances, runInlineBrowserTests } from './utils' + +async function runBrowserTests(structure: TestFsStructure) { + return runInlineBrowserTests(structure, { + reporters: 'default', + browser: { + ui: false, + instances: instances.slice(0, 1), + traceView: true, + screenshotFailures: false, + }, + }) +} + +function extractTimelines(stderr: string) { + return [...stderr.matchAll(/\nTrace\b.*?\n\n/gs)].map((match) => { + const [title, ...lines] = match[0].trim().split('\n') + const rows = lines.map((line) => + line + .trim() + .replace(/\d+(\.\d+)?m?s\b/g, '