diff --git a/packages/browser/src/client/tester/runner.ts b/packages/browser/src/client/tester/runner.ts index fef76125f..2198039de 100644 --- a/packages/browser/src/client/tester/runner.ts +++ b/packages/browser/src/client/tester/runner.ts @@ -76,10 +76,12 @@ function createBrowserRunner( } private traces = new Map() + private testAttemptIndexMap = new Map() async onBeforeTryTask(test: Test, options: TestTryOptions) { await userEvent.cleanup() super.onBeforeTryTask?.(test, options) + this.testAttemptIndexMap.set(test.id, options) const trace = this.config.browser.trace const { retry, repeats } = options const shouldTrace = @@ -120,6 +122,7 @@ function createBrowserRunner( test: Test, { retry, repeats }: { retry: number; repeats: number }, ) => { + this.testAttemptIndexMap.delete(test.id) const hasActiveTraceView = getBrowserState().browserTraceAttempts.has(test.id) if (hasActiveTraceView) { const status = test.result?.state @@ -192,6 +195,7 @@ function createBrowserRunner( Reflect.get(lastErrorContext, 'meta')?.outcome !== 'unstable-screenshot' ) ) { + const { retry, repeats } = this.testAttemptIndexMap.get(task.id) ?? { retry: 0, repeats: 0 } const screenshot = await page .screenshot({ timeout: this.config.browser.providerOptions?.actionTimeout ?? 5_000, @@ -199,7 +203,7 @@ function createBrowserRunner( this.config.attachmentsDir, 'failure-screenshots', basename(task.file.filepath), - `${task.fullTestName.replace(/\W/g, '-')}.png`, + `${getFailureScreenshotName(task, retry, repeats)}.png`, ), }) .catch((err) => { @@ -209,6 +213,8 @@ function createBrowserRunner( await recordArtifact(task, { type: 'internal:failureScreenshot', attachments: [{ contentType: 'image/png', path: screenshot, originalPath: screenshot }], + retryIndex: retry, + repeatIndex: repeats, } as const) } } @@ -456,6 +462,16 @@ async function updateTestFilesLocations(files: File[], sourceMaps: Map--' suffix only when test enables retry or repeat + if (!maxRetries && !test.repeats) { + return name + } + return `${name}-${repeats}-${retry}` +} + function getTraceName(task: Task, retryCount: number, repeatsCount: number) { const name = getTestName(task, '-').replace(/[^a-z0-9]/gi, '-') return `${name}-${repeatsCount}-${retryCount}` diff --git a/packages/ui/client/components/FailureScreenshot.vue b/packages/ui/client/components/FailureScreenshot.vue index d69e88c51..594204868 100644 --- a/packages/ui/client/components/FailureScreenshot.vue +++ b/packages/ui/client/components/FailureScreenshot.vue @@ -14,7 +14,8 @@ const { task } = defineProps<{ const showScreenshot = ref(false) const artifact = computed(() => { if (task.type === 'test') { - const artifact = task.artifacts.find( + // TODO: show each attempt's screenshot next to that attempt's errors, not only the last one + const artifact = task.artifacts.findLast( (artifact) => artifact.type === 'internal:failureScreenshot', ) diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index d790443bf..199cef44d 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -2,6 +2,7 @@ import type { ParsedStack, SerializedError } from '@vitest/utils' import type { File, Task, + TaskResultAttempt, TestAnnotation, TestBenchmark, TestBenchmarkTask, @@ -1459,24 +1460,30 @@ export abstract class BaseReporter implements Reporter { } private printTaskErrors(tasks: Task[], errorDivider: () => void) { - const errorsQueue: [error: TestError | undefined, tests: Task[]][] = [] + interface ErrorsQueueItem { + error: TestError + // `undefined` for tasks without attempts (suites, files) + sources: { task: Task; attempt?: TaskResultAttempt }[] + } + const errorsQueue: ErrorsQueueItem[] = [] for (const task of tasks) { // Merge identical errors - task.result?.errors?.forEach((error) => { - let previous + for (const { error, attempt } of getErrorsWithAttempt(task)) { + let previous: ErrorsQueueItem | undefined - if (error?.stack) { - previous = errorsQueue.find((i) => { - if (i[0]?.stack !== error.stack || i[0]?.diff !== error.diff) { + if (error.stack) { + previous = errorsQueue.find((item) => { + if (item.error.stack !== error.stack || item.error.diff !== error.diff) { return false } + const itemTask = item.sources[0].task const currentProjectName = (task as File)?.projectName || task.file?.projectName || '' - const projectName = (i[1][0] as File)?.projectName || i[1][0].file?.projectName || '' + const projectName = (itemTask as File)?.projectName || itemTask.file?.projectName || '' const currentAnnotations = task.type === 'test' && task.annotations - const itemAnnotations = i[1][0].type === 'test' && i[1][0].annotations + const itemAnnotations = itemTask.type === 'test' && itemTask.annotations return ( projectName === currentProjectName && deepEqual(currentAnnotations, itemAnnotations) @@ -1485,14 +1492,24 @@ export abstract class BaseReporter implements Reporter { } if (previous) { - previous[1].push(task) + previous.sources.push({ task, attempt }) } else { - errorsQueue.push([error, [task]]) + errorsQueue.push({ error, sources: [{ task, attempt }] }) } - }) + } } - for (const [error, tasks] of errorsQueue) { + for (const { error, sources } of errorsQueue) { + const tasks = sources.map(({ task }) => task) + const attemptsByTask = new Map>() + for (const { task, attempt } of sources) { + attemptsByTask.set(task, (attemptsByTask.get(task) ?? new Set()).add(attempt)) + } + const screenshotPaths = [...attemptsByTask].flatMap(([task, attempts]) => + [...attempts].flatMap((attempt) => + attempt ? getFailureScreenshotPaths(task, attempt) : [], + ), + ) for (const task of tasks) { const filepath = (task as File)?.filepath || '' const projectName = (task as File)?.projectName || task.file?.projectName || '' @@ -1510,20 +1527,6 @@ export abstract class BaseReporter implements Reporter { ) } - const screenshotPaths = tasks.reduce((paths, t) => { - if (t.type === 'test') { - for (const artifact of t.artifacts) { - if (artifact.type === 'internal:failureScreenshot') { - if (artifact.attachments.length) { - paths.push(artifact.attachments[0].originalPath) - } - } - } - } - - return paths - }, []) - this.ctx.logger.printError(error, { project: this.ctx.getProjectByName(tasks[0].file.projectName || ''), verbose: this.verbose, @@ -1551,6 +1554,32 @@ export abstract class BaseReporter implements Reporter { } } +function getErrorsWithAttempt(task: Task): { error: TestError; attempt?: TaskResultAttempt }[] { + const attempts = task.result?.attempts + if (!attempts?.length) { + return (task.result?.errors ?? []).map((error) => ({ error })) + } + return attempts.flatMap((attempt) => (attempt.errors ?? []).map((error) => ({ error, attempt }))) +} + +function getFailureScreenshotPaths(task: Task, attempt: TaskResultAttempt): string[] { + if (task.type !== 'test') { + return [] + } + const paths: string[] = [] + for (const artifact of task.artifacts) { + if ( + artifact.type === 'internal:failureScreenshot' && + artifact.attachments.length && + artifact.retryIndex === attempt.retryIndex && + artifact.repeatIndex === attempt.repeatIndex + ) { + paths.push(artifact.attachments[0].originalPath) + } + } + return paths +} + function deepEqual(a: any, b: any): boolean { if (a === b) { return true diff --git a/packages/vitest/src/runtime/runner/types.ts b/packages/vitest/src/runtime/runner/types.ts index 0a3164601..481497def 100644 --- a/packages/vitest/src/runtime/runner/types.ts +++ b/packages/vitest/src/runtime/runner/types.ts @@ -1517,6 +1517,10 @@ interface FailureScreenshotArtifactAttachment extends TestAttachment { export interface FailureScreenshotArtifact extends TestArtifactBase { type: 'internal:failureScreenshot' attachments: [FailureScreenshotArtifactAttachment] | [] + /** The zero-based retry index of the attempt that took the screenshot. */ + retryIndex?: number + /** The zero-based repeat index of the attempt that took the screenshot. */ + repeatIndex?: number } /** diff --git a/test/browser/specs/failure-screenshot.test.ts b/test/browser/specs/failure-screenshot.test.ts index 5520a1eb0..c80a75da5 100644 --- a/test/browser/specs/failure-screenshot.test.ts +++ b/test/browser/specs/failure-screenshot.test.ts @@ -1,4 +1,6 @@ import type { TestFsStructure } from '../../test-utils' +import { readFileSync } from 'node:fs' +import { resolve } from 'pathe' import { describe, expect, test } from 'vitest' import { runInlineTests } from '../../test-utils' import utilsContent from '../fixtures/expect-dom/utils?raw' @@ -105,4 +107,86 @@ describe('failure screenshots', () => { ) }) }) + + describe('retry and repeats', () => { + function extractFailureLines(stderr: string, root: string) { + return stderr + .split('\n') + .filter((line) => /FAIL|AssertionError|Failure screenshot|\.png/.test(line)) + .map((line) => line.replace(root.split('/').at(-1)!, '')) + .join('\n') + } + + function readReportedScreenshots(stderr: string) { + const paths = [...new Set(stderr.match(/vitest-test-\S+?\.png/g))] + return paths.map((path) => readFileSync(resolve(path)).toString('base64')) + } + + test('saves and reports one screenshot per attempt', async () => { + const { stderr, root } = await runBrowserTests({ + [testFilename]: /* ts */ ` + import { expect, test } from 'vitest' + import { page } from 'vitest/browser' + + let attempt = 0 + test('repeated', { repeats: 1, retry: 1 }, async () => { + attempt++ + document.body.innerHTML = \`\` + await page.getByText(\`attempt \${attempt}\`).click() + expect(attempt).toBe(10) + }) + `, + }) + + expect(extractFailureLines(stderr, root)).toMatchInlineSnapshot(` + " FAIL |chromium| basic.test.ts > repeated + AssertionError: expected 1 to be 10 // Object.is equality + Failure screenshot: + - /.vitest/attachments/failure-screenshots/basic.test.ts/repeated-0-0.png + FAIL |chromium| basic.test.ts > repeated + AssertionError: expected 2 to be 10 // Object.is equality + Failure screenshot: + - /.vitest/attachments/failure-screenshots/basic.test.ts/repeated-0-1.png + FAIL |chromium| basic.test.ts > repeated + AssertionError: expected 3 to be 10 // Object.is equality + Failure screenshot: + - /.vitest/attachments/failure-screenshots/basic.test.ts/repeated-1-0.png + FAIL |chromium| basic.test.ts > repeated + AssertionError: expected 4 to be 10 // Object.is equality + Failure screenshot: + - /.vitest/attachments/failure-screenshots/basic.test.ts/repeated-1-1.png" + `) + expect(new Set(readReportedScreenshots(stderr)).size).toBe(4) + }) + + test('lists screenshots of every attempt that produced the error', async () => { + const { stderr, root } = await runBrowserTests({ + [testFilename]: /* ts */ ` + import { expect, test } from 'vitest' + + test('soft', { retry: 1 }, () => { + document.body.innerHTML = '' + expect.soft(1).toBe(2) + expect.soft(2).toBe(3) + }) + `, + }) + + // screenshots are taken per attempt, so each soft error lists the same ones + expect(extractFailureLines(stderr, root)).toMatchInlineSnapshot(` + " FAIL |chromium| basic.test.ts > soft + FAIL |chromium| basic.test.ts > soft + AssertionError: expected 1 to be 2 // Object.is equality + Failure screenshots: + - /.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-0.png + - /.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-1.png + FAIL |chromium| basic.test.ts > soft + FAIL |chromium| basic.test.ts > soft + AssertionError: expected 2 to be 3 // Object.is equality + Failure screenshots: + - /.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-0.png + - /.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-1.png" + `) + }) + }) })