diff --git a/docs/api/advanced/runner.md b/docs/api/advanced/runner.md index f4f6561ad..d3c6fd690 100644 --- a/docs/api/advanced/runner.md +++ b/docs/api/advanced/runner.md @@ -268,6 +268,37 @@ export interface TaskResult { * The zero-based index of the current repeat. */ repeatCount?: number + /** + * Individual results for every retry and repeat attempt. + */ + attempts?: TaskResultAttempt[] +} + +export interface TaskResultAttempt { + /** + * The state after applying the test's expected failure option. + */ + state: 'pass' | 'fail' | 'skip' + /** + * Errors produced by this attempt. + */ + errors?: TestError[] + /** + * How long in milliseconds the attempt took to run. + */ + duration: number + /** + * Time in milliseconds when the attempt started running. + */ + startTime: number + /** + * The zero-based retry index within the repeat. + */ + retryIndex: number + /** + * The zero-based repeat index. + */ + repeatIndex: number } ``` diff --git a/docs/api/advanced/test-case.md b/docs/api/advanced/test-case.md index e6d6a824f..c07e47fb6 100644 --- a/docs/api/advanced/test-case.md +++ b/docs/api/advanced/test-case.md @@ -228,7 +228,7 @@ interface TestResultPassed { ``` ::: warning -Note that the test with `passed` state can still have errors attached - this can happen if `retry` was triggered at least once. +Note that the test with `passed` state can still have errors attached - this can happen if `retry` was triggered at least once. Use [`attempts()`](#attempts) to see which run produced each error. ::: ## diagnostic @@ -278,6 +278,58 @@ interface TestDiagnostic { `diagnostic()` will return `undefined` if the test was not scheduled to run yet. ::: +## attempts 5.1.0 {#attempts} + +```ts +function attempts(): ReadonlyArray +``` + +Results of every run of the test, in execution order. A test runs more than once when [`retry`](/config/retry) or [`repeats`](/api/test#repeats) is configured, and each run produces one entry. Unlike `result()`, which only reports the final outcome and merges errors from all runs, each attempt keeps its own state, errors and timing. + +```ts +interface TestAttempt { + /** + * The state of this attempt. `fails` tests are already inverted, + * so an expected failure is reported as `passed`. + */ + readonly state: 'passed' | 'failed' | 'skipped' + /** + * Errors thrown during this attempt only. + */ + readonly errors: ReadonlyArray | undefined + /** + * How long in milliseconds the attempt took to run. + */ + readonly duration: number + /** + * Time in milliseconds when the attempt started. + */ + readonly startTime: number + /** + * The zero-based retry index within the current repeat. + */ + readonly retryIndex: number + /** + * The zero-based repeat index. + */ + readonly repeatIndex: number +} +``` + +For example, a test with `retry: 2` that passes on its third run returns: + +```ts +[ + { state: 'failed', retryIndex: 0, repeatIndex: 0, errors: [/* ... */] }, + { state: 'failed', retryIndex: 1, repeatIndex: 0, errors: [/* ... */] }, + { state: 'passed', retryIndex: 2, repeatIndex: 0, errors: undefined }, +] +``` + +::: info +`attempts()` returns an empty array if the test has not run yet. +::: + ## annotations ```ts diff --git a/packages/vitest/src/node/reporters/reported-tasks.ts b/packages/vitest/src/node/reporters/reported-tasks.ts index 6cdac5d63..00c632b34 100644 --- a/packages/vitest/src/node/reporters/reported-tasks.ts +++ b/packages/vitest/src/node/reporters/reported-tasks.ts @@ -194,6 +194,20 @@ export class TestCase extends ReportedTaskImplementation { } satisfies TestResultFailed } + /** + * Individual results for every retry and repeat attempt. + */ + public attempts(): ReadonlyArray { + return (this.task.result?.attempts || []).map((attempt) => ({ + state: attempt.state === 'pass' ? 'passed' : attempt.state === 'fail' ? 'failed' : 'skipped', + errors: attempt.errors as TestError[] | undefined, + duration: attempt.duration, + startTime: attempt.startTime, + retryIndex: attempt.retryIndex, + repeatIndex: attempt.repeatIndex, + })) + } + /** * Test annotations added via the `task.annotate` API during the test execution. */ @@ -609,10 +623,29 @@ function buildOptions(task: RunnerTestCase | RunnerTestSuite): TaskOptions { } } -export type TestSuiteState = 'skipped' | 'pending' | 'failed' | 'passed' +export type TestAttemptState = 'skipped' | 'failed' | 'passed' +export type TestSuiteState = TestAttemptState | 'pending' export type TestModuleState = TestSuiteState | 'queued' export type TestState = TestResult['state'] +/** + * The result of an individual test attempt. + */ +export interface TestAttempt { + /** The state after applying the test's expected failure option. */ + readonly state: TestAttemptState + /** Errors produced by this attempt. */ + readonly errors: ReadonlyArray | undefined + /** How long in milliseconds the attempt took to run. */ + readonly duration: number + /** Time in milliseconds when the attempt started running. */ + readonly startTime: number + /** The zero-based retry index within the repeat. */ + readonly retryIndex: number + /** The zero-based repeat index. */ + readonly repeatIndex: number +} + export type TestResult = TestResultPassed | TestResultFailed | TestResultSkipped | TestResultPending export interface TestResultPending { diff --git a/packages/vitest/src/public/index.ts b/packages/vitest/src/public/index.ts index 5a9869cf9..13ad975fe 100644 --- a/packages/vitest/src/public/index.ts +++ b/packages/vitest/src/public/index.ts @@ -87,6 +87,7 @@ export type { TaskBase as RunnerTaskBase, TaskEventPack as RunnerTaskEventPack, TaskResult as RunnerTaskResult, + TaskResultAttempt as RunnerTaskResultAttempt, TaskResultPack as RunnerTaskResultPack, Test as RunnerTestCase, File as RunnerTestFile, diff --git a/packages/vitest/src/public/node.ts b/packages/vitest/src/public/node.ts index 28806e9fd..59a1e58a2 100644 --- a/packages/vitest/src/public/node.ts +++ b/packages/vitest/src/public/node.ts @@ -89,6 +89,8 @@ export type { Report } from '../node/reporters/report' export type { ModuleDiagnostic, TaskOptions, + TestAttempt, + TestAttemptState, TestCase, TestCollection, TestDiagnostic, diff --git a/packages/vitest/src/runtime/runner/run.ts b/packages/vitest/src/runtime/runner/run.ts index 257c093b9..22e30b3c4 100644 --- a/packages/vitest/src/runtime/runner/run.ts +++ b/packages/vitest/src/runtime/runner/run.ts @@ -12,6 +12,7 @@ import type { Task, TaskMeta, TaskResult, + TaskResultAttempt, TaskResultPack, TaskState, TaskUpdateEvent, @@ -599,6 +600,7 @@ async function runTest(test: Test, runner: VitestRunner): Promise { state: 'run', startTime: unixNow(), retryCount: 0, + attempts: [], } updateTask('test-prepare', test, runner) @@ -615,7 +617,20 @@ async function runTest(test: Test, runner: VitestRunner): Promise { test.result.state = 'run' as TaskState const retry = getRetryCount(test.retry) for (let retryCount = 0; retryCount <= retry; retryCount++) { + const attemptStart = now() + const attemptStartTime = unixNow() const attemptErrorsStart = test.result.errors?.length ?? 0 + const recordAttempt = (state: TaskResultAttempt['state']) => { + const errors = test.result!.errors?.slice(attemptErrorsStart) + test.result!.attempts!.push({ + state, + errors: errors?.length ? errors : undefined, + duration: now() - attemptStart, + startTime: attemptStartTime, + retryIndex: retryCount, + repeatIndex: repeatCount, + }) + } const beforeEachCleanups: unknown[] = [] // fixtureCheckpoint is passed by callAroundEachHooks - it represents the count // of fixture cleanup functions AFTER all aroundEach fixtures have been resolved @@ -714,12 +729,14 @@ async function runTest(test: Test, runner: VitestRunner): Promise { // skipped with new PendingError if (test.result?.pending || test.result?.state === 'skip') { + recordAttempt('skip') test.mode = 'skip' test.result = { state: 'skip', note: test.result?.note, pending: true, duration: now() - start, + attempts: test.result.attempts, } updateTask('test-finished', test, runner) setCurrentTest(undefined) @@ -744,6 +761,7 @@ async function runTest(test: Test, runner: VitestRunner): Promise { } } } + recordAttempt(test.result.state === 'pass' ? 'pass' : 'fail') if (test.result.state === 'pass') { break } diff --git a/packages/vitest/src/runtime/runner/types.ts b/packages/vitest/src/runtime/runner/types.ts index 5e4c1a1ff..0a3164601 100644 --- a/packages/vitest/src/runtime/runner/types.ts +++ b/packages/vitest/src/runtime/runner/types.ts @@ -158,6 +158,24 @@ export interface TaskMeta { __vitest_label__?: string } +/** + * The result of an individual test attempt. + */ +export interface TaskResultAttempt { + /** The state after applying the test's expected failure option. */ + state: 'pass' | 'fail' | 'skip' + /** Errors produced by this attempt. */ + errors?: TestError[] + /** How long in milliseconds the attempt took to run. */ + duration: number + /** Time in milliseconds when the attempt started running. */ + startTime: number + /** The zero-based retry index within the repeat. */ + retryIndex: number + /** The zero-based repeat index. */ + repeatIndex: number +} + /** * The result of calling a task. */ @@ -174,6 +192,10 @@ export interface TaskResult { * if `expect.soft()` failed multiple times or `retry` was triggered. */ errors?: TestError[] + /** + * Individual results for every retry and repeat attempt. + */ + attempts?: TaskResultAttempt[] /** * How long in milliseconds the task took to run. */ diff --git a/test/e2e/test/reported-tasks.test.ts b/test/e2e/test/reported-tasks.test.ts index d3145c3ff..4c23193fe 100644 --- a/test/e2e/test/reported-tasks.test.ts +++ b/test/e2e/test/reported-tasks.test.ts @@ -276,6 +276,39 @@ it('correctly reports flaky tests', ({ testModule }) => { const result = testFlaky.result()! expect(result.state).toBe('passed') expect(result.errors).toHaveLength(2) + expect( + testFlaky.attempts().map((attempt) => ({ + state: attempt.state, + errors: attempt.errors?.map((error) => error.message), + retryIndex: attempt.retryIndex, + repeatIndex: attempt.repeatIndex, + })), + ).toMatchInlineSnapshot(` + [ + { + "errors": [ + "expected +0 to be 2 // Object.is equality", + ], + "repeatIndex": 0, + "retryIndex": 0, + "state": "failed", + }, + { + "errors": [ + "expected 1 to be 2 // Object.is equality", + ], + "repeatIndex": 0, + "retryIndex": 1, + "state": "failed", + }, + { + "errors": undefined, + "repeatIndex": 0, + "retryIndex": 2, + "state": "passed", + }, + ] + `) }) it('correctly reports repeated tests', ({ testModule }) => { @@ -287,6 +320,65 @@ it('correctly reports repeated tests', ({ testModule }) => { const result = testRepeated.result()! expect(result.state).toBe('failed') expect(result.errors).toHaveLength(6) + expect( + testRepeated.attempts().map((attempt) => ({ + state: attempt.state, + errors: attempt.errors?.map((error) => error.message), + retryIndex: attempt.retryIndex, + repeatIndex: attempt.repeatIndex, + })), + ).toMatchInlineSnapshot(` + [ + { + "errors": [ + "expected 1 to be 2 // Object.is equality", + ], + "repeatIndex": 0, + "retryIndex": 0, + "state": "failed", + }, + { + "errors": [ + "expected 1 to be 2 // Object.is equality", + ], + "repeatIndex": 1, + "retryIndex": 0, + "state": "failed", + }, + { + "errors": [ + "expected 1 to be 2 // Object.is equality", + ], + "repeatIndex": 2, + "retryIndex": 0, + "state": "failed", + }, + { + "errors": [ + "expected 1 to be 2 // Object.is equality", + ], + "repeatIndex": 3, + "retryIndex": 0, + "state": "failed", + }, + { + "errors": [ + "expected 1 to be 2 // Object.is equality", + ], + "repeatIndex": 4, + "retryIndex": 0, + "state": "failed", + }, + { + "errors": [ + "expected 1 to be 2 // Object.is equality", + ], + "repeatIndex": 5, + "retryIndex": 0, + "state": "failed", + }, + ] + `) }) it('correctly passed down metadata', ({ testModule }) => { diff --git a/test/e2e/test/retry.test.ts b/test/e2e/test/retry.test.ts index 17a29fc44..566a7e8a0 100644 --- a/test/e2e/test/retry.test.ts +++ b/test/e2e/test/retry.test.ts @@ -131,7 +131,7 @@ test('expected failures exhaust retries in every repeat when assertions pass', a }) test('expected failures can recover through a retry in every repeat', async () => { - const { stderr, errorTree } = await runInlineTests({ + const { stderr, errorTree, results } = await runInlineTests({ 'repeats.test.js': ` import { afterAll, expect, it } from 'vitest' @@ -157,6 +157,161 @@ test('expected failures can recover through a retry in every repeat', async () = }, } `) + + const [test] = results[0].children.allTests() + const attempts = test.attempts() + expect( + attempts.map((attempt) => ({ + state: attempt.state, + errors: attempt.errors?.map((error) => error.message) || [], + retryIndex: attempt.retryIndex, + repeatIndex: attempt.repeatIndex, + })), + ).toMatchInlineSnapshot(` + [ + { + "errors": [ + "Expect test to fail", + ], + "repeatIndex": 0, + "retryIndex": 0, + "state": "failed", + }, + { + "errors": [], + "repeatIndex": 0, + "retryIndex": 1, + "state": "passed", + }, + { + "errors": [ + "Expect test to fail", + ], + "repeatIndex": 1, + "retryIndex": 0, + "state": "failed", + }, + { + "errors": [], + "repeatIndex": 1, + "retryIndex": 1, + "state": "passed", + }, + { + "errors": [ + "Expect test to fail", + ], + "repeatIndex": 2, + "retryIndex": 0, + "state": "failed", + }, + { + "errors": [], + "repeatIndex": 2, + "retryIndex": 1, + "state": "passed", + }, + ] + `) + attempts.forEach((attempt) => { + expect(attempt.duration).toBeGreaterThanOrEqual(0) + expect(attempt.startTime).toBeGreaterThan(0) + }) +}) + +test('attempts grow while the test runs', async () => { + const { stderr, errorTree } = await runInlineTests({ + 'attempts.test.js': ` + import { afterAll, afterEach, expect, it } from 'vitest' + + const seen = [] + let runs = 0 + + // the current attempt is recorded after afterEach, so only previous attempts are visible + afterEach(({ task }) => { + seen.push(task.result.attempts.map(({ repeatIndex, retryIndex, state }) => ({ repeatIndex, retryIndex, state }))) + }) + + // | repeat 0 | repeat 1 | + // | retry 0 -> retry 1 | retry 0 -> retry 1 | + // | fail -> pass | fail -> pass | + it('flaky', { retry: 1, repeats: 1 }, () => { + expect(++runs % 2).toBe(0) + }) + + afterAll(() => { + expect(seen).toEqual([ + [], + [ + { repeatIndex: 0, retryIndex: 0, state: 'fail' }, + ], + [ + { repeatIndex: 0, retryIndex: 0, state: 'fail' }, + { repeatIndex: 0, retryIndex: 1, state: 'pass' }, + ], + [ + { repeatIndex: 0, retryIndex: 0, state: 'fail' }, + { repeatIndex: 0, retryIndex: 1, state: 'pass' }, + { repeatIndex: 1, retryIndex: 0, state: 'fail' }, + ], + ]) + }) + `, + }) + + expect(stderr).toBe('') + expect(errorTree()).toMatchInlineSnapshot(` + { + "attempts.test.js": { + "flaky": "passed", + }, + } + `) +}) + +test('skipping during a retry ends attempts with a skipped attempt', async () => { + const { stderr, results } = await runInlineTests({ + 'skip.test.js': ` + import { expect, it } from 'vitest' + + let runs = 0 + + // | repeat 0 | repeat 1 | + // | retry 0 -> retry 1 -> retry 2 | | + // | fail -> skip -> (not run) | (not run) | + it('skips on retry', { retry: 2, repeats: 1 }, ({ skip }) => { + if (++runs === 2) { + skip() + } + expect(1).toBe(2) + }) + + it.skip('skipped statically', () => {}) + `, + }) + + expect(stderr).toBe('') + const [skipsOnRetry, skippedStatically] = results[0].children.allTests() + expect(skipsOnRetry.result().state).toBe('skipped') + expect( + skipsOnRetry + .attempts() + .map(({ repeatIndex, retryIndex, state }) => ({ repeatIndex, retryIndex, state })), + ).toMatchInlineSnapshot(` + [ + { + "repeatIndex": 0, + "retryIndex": 0, + "state": "failed", + }, + { + "repeatIndex": 0, + "retryIndex": 1, + "state": "skipped", + }, + ] + `) + expect(skippedStatically.attempts()).toEqual([]) }) test('syntax errors remain failures after successful repeats', async () => {