diff --git a/docs/api/advanced/runner.md b/docs/api/advanced/runner.md
index f4f6561ad..d3c6fd690 100644
--- a/docs/api/advanced/runner.md
+++ b/docs/api/advanced/runner.md
@@ -268,6 +268,37 @@ export interface TaskResult {
* The zero-based index of the current repeat.
*/
repeatCount?: number
+ /**
+ * Individual results for every retry and repeat attempt.
+ */
+ attempts?: TaskResultAttempt[]
+}
+
+export interface TaskResultAttempt {
+ /**
+ * The state after applying the test's expected failure option.
+ */
+ state: 'pass' | 'fail' | 'skip'
+ /**
+ * Errors produced by this attempt.
+ */
+ errors?: TestError[]
+ /**
+ * How long in milliseconds the attempt took to run.
+ */
+ duration: number
+ /**
+ * Time in milliseconds when the attempt started running.
+ */
+ startTime: number
+ /**
+ * The zero-based retry index within the repeat.
+ */
+ retryIndex: number
+ /**
+ * The zero-based repeat index.
+ */
+ repeatIndex: number
}
```
diff --git a/docs/api/advanced/test-case.md b/docs/api/advanced/test-case.md
index e6d6a824f..c07e47fb6 100644
--- a/docs/api/advanced/test-case.md
+++ b/docs/api/advanced/test-case.md
@@ -228,7 +228,7 @@ interface TestResultPassed {
```
::: warning
-Note that the test with `passed` state can still have errors attached - this can happen if `retry` was triggered at least once.
+Note that the test with `passed` state can still have errors attached - this can happen if `retry` was triggered at least once. Use [`attempts()`](#attempts) to see which run produced each error.
:::
## diagnostic
@@ -278,6 +278,58 @@ interface TestDiagnostic {
`diagnostic()` will return `undefined` if the test was not scheduled to run yet.
:::
+## attempts 5.1.0 {#attempts}
+
+```ts
+function attempts(): ReadonlyArray
+```
+
+Results of every run of the test, in execution order. A test runs more than once when [`retry`](/config/retry) or [`repeats`](/api/test#repeats) is configured, and each run produces one entry. Unlike `result()`, which only reports the final outcome and merges errors from all runs, each attempt keeps its own state, errors and timing.
+
+```ts
+interface TestAttempt {
+ /**
+ * The state of this attempt. `fails` tests are already inverted,
+ * so an expected failure is reported as `passed`.
+ */
+ readonly state: 'passed' | 'failed' | 'skipped'
+ /**
+ * Errors thrown during this attempt only.
+ */
+ readonly errors: ReadonlyArray | undefined
+ /**
+ * How long in milliseconds the attempt took to run.
+ */
+ readonly duration: number
+ /**
+ * Time in milliseconds when the attempt started.
+ */
+ readonly startTime: number
+ /**
+ * The zero-based retry index within the current repeat.
+ */
+ readonly retryIndex: number
+ /**
+ * The zero-based repeat index.
+ */
+ readonly repeatIndex: number
+}
+```
+
+For example, a test with `retry: 2` that passes on its third run returns:
+
+```ts
+[
+ { state: 'failed', retryIndex: 0, repeatIndex: 0, errors: [/* ... */] },
+ { state: 'failed', retryIndex: 1, repeatIndex: 0, errors: [/* ... */] },
+ { state: 'passed', retryIndex: 2, repeatIndex: 0, errors: undefined },
+]
+```
+
+::: info
+`attempts()` returns an empty array if the test has not run yet.
+:::
+
## annotations
```ts
diff --git a/packages/vitest/src/node/reporters/reported-tasks.ts b/packages/vitest/src/node/reporters/reported-tasks.ts
index 6cdac5d63..00c632b34 100644
--- a/packages/vitest/src/node/reporters/reported-tasks.ts
+++ b/packages/vitest/src/node/reporters/reported-tasks.ts
@@ -194,6 +194,20 @@ export class TestCase extends ReportedTaskImplementation {
} satisfies TestResultFailed
}
+ /**
+ * Individual results for every retry and repeat attempt.
+ */
+ public attempts(): ReadonlyArray {
+ return (this.task.result?.attempts || []).map((attempt) => ({
+ state: attempt.state === 'pass' ? 'passed' : attempt.state === 'fail' ? 'failed' : 'skipped',
+ errors: attempt.errors as TestError[] | undefined,
+ duration: attempt.duration,
+ startTime: attempt.startTime,
+ retryIndex: attempt.retryIndex,
+ repeatIndex: attempt.repeatIndex,
+ }))
+ }
+
/**
* Test annotations added via the `task.annotate` API during the test execution.
*/
@@ -609,10 +623,29 @@ function buildOptions(task: RunnerTestCase | RunnerTestSuite): TaskOptions {
}
}
-export type TestSuiteState = 'skipped' | 'pending' | 'failed' | 'passed'
+export type TestAttemptState = 'skipped' | 'failed' | 'passed'
+export type TestSuiteState = TestAttemptState | 'pending'
export type TestModuleState = TestSuiteState | 'queued'
export type TestState = TestResult['state']
+/**
+ * The result of an individual test attempt.
+ */
+export interface TestAttempt {
+ /** The state after applying the test's expected failure option. */
+ readonly state: TestAttemptState
+ /** Errors produced by this attempt. */
+ readonly errors: ReadonlyArray | undefined
+ /** How long in milliseconds the attempt took to run. */
+ readonly duration: number
+ /** Time in milliseconds when the attempt started running. */
+ readonly startTime: number
+ /** The zero-based retry index within the repeat. */
+ readonly retryIndex: number
+ /** The zero-based repeat index. */
+ readonly repeatIndex: number
+}
+
export type TestResult = TestResultPassed | TestResultFailed | TestResultSkipped | TestResultPending
export interface TestResultPending {
diff --git a/packages/vitest/src/public/index.ts b/packages/vitest/src/public/index.ts
index 5a9869cf9..13ad975fe 100644
--- a/packages/vitest/src/public/index.ts
+++ b/packages/vitest/src/public/index.ts
@@ -87,6 +87,7 @@ export type {
TaskBase as RunnerTaskBase,
TaskEventPack as RunnerTaskEventPack,
TaskResult as RunnerTaskResult,
+ TaskResultAttempt as RunnerTaskResultAttempt,
TaskResultPack as RunnerTaskResultPack,
Test as RunnerTestCase,
File as RunnerTestFile,
diff --git a/packages/vitest/src/public/node.ts b/packages/vitest/src/public/node.ts
index 28806e9fd..59a1e58a2 100644
--- a/packages/vitest/src/public/node.ts
+++ b/packages/vitest/src/public/node.ts
@@ -89,6 +89,8 @@ export type { Report } from '../node/reporters/report'
export type {
ModuleDiagnostic,
TaskOptions,
+ TestAttempt,
+ TestAttemptState,
TestCase,
TestCollection,
TestDiagnostic,
diff --git a/packages/vitest/src/runtime/runner/run.ts b/packages/vitest/src/runtime/runner/run.ts
index 257c093b9..22e30b3c4 100644
--- a/packages/vitest/src/runtime/runner/run.ts
+++ b/packages/vitest/src/runtime/runner/run.ts
@@ -12,6 +12,7 @@ import type {
Task,
TaskMeta,
TaskResult,
+ TaskResultAttempt,
TaskResultPack,
TaskState,
TaskUpdateEvent,
@@ -599,6 +600,7 @@ async function runTest(test: Test, runner: VitestRunner): Promise {
state: 'run',
startTime: unixNow(),
retryCount: 0,
+ attempts: [],
}
updateTask('test-prepare', test, runner)
@@ -615,7 +617,20 @@ async function runTest(test: Test, runner: VitestRunner): Promise {
test.result.state = 'run' as TaskState
const retry = getRetryCount(test.retry)
for (let retryCount = 0; retryCount <= retry; retryCount++) {
+ const attemptStart = now()
+ const attemptStartTime = unixNow()
const attemptErrorsStart = test.result.errors?.length ?? 0
+ const recordAttempt = (state: TaskResultAttempt['state']) => {
+ const errors = test.result!.errors?.slice(attemptErrorsStart)
+ test.result!.attempts!.push({
+ state,
+ errors: errors?.length ? errors : undefined,
+ duration: now() - attemptStart,
+ startTime: attemptStartTime,
+ retryIndex: retryCount,
+ repeatIndex: repeatCount,
+ })
+ }
const beforeEachCleanups: unknown[] = []
// fixtureCheckpoint is passed by callAroundEachHooks - it represents the count
// of fixture cleanup functions AFTER all aroundEach fixtures have been resolved
@@ -714,12 +729,14 @@ async function runTest(test: Test, runner: VitestRunner): Promise {
// skipped with new PendingError
if (test.result?.pending || test.result?.state === 'skip') {
+ recordAttempt('skip')
test.mode = 'skip'
test.result = {
state: 'skip',
note: test.result?.note,
pending: true,
duration: now() - start,
+ attempts: test.result.attempts,
}
updateTask('test-finished', test, runner)
setCurrentTest(undefined)
@@ -744,6 +761,7 @@ async function runTest(test: Test, runner: VitestRunner): Promise {
}
}
}
+ recordAttempt(test.result.state === 'pass' ? 'pass' : 'fail')
if (test.result.state === 'pass') {
break
}
diff --git a/packages/vitest/src/runtime/runner/types.ts b/packages/vitest/src/runtime/runner/types.ts
index 5e4c1a1ff..0a3164601 100644
--- a/packages/vitest/src/runtime/runner/types.ts
+++ b/packages/vitest/src/runtime/runner/types.ts
@@ -158,6 +158,24 @@ export interface TaskMeta {
__vitest_label__?: string
}
+/**
+ * The result of an individual test attempt.
+ */
+export interface TaskResultAttempt {
+ /** The state after applying the test's expected failure option. */
+ state: 'pass' | 'fail' | 'skip'
+ /** Errors produced by this attempt. */
+ errors?: TestError[]
+ /** How long in milliseconds the attempt took to run. */
+ duration: number
+ /** Time in milliseconds when the attempt started running. */
+ startTime: number
+ /** The zero-based retry index within the repeat. */
+ retryIndex: number
+ /** The zero-based repeat index. */
+ repeatIndex: number
+}
+
/**
* The result of calling a task.
*/
@@ -174,6 +192,10 @@ export interface TaskResult {
* if `expect.soft()` failed multiple times or `retry` was triggered.
*/
errors?: TestError[]
+ /**
+ * Individual results for every retry and repeat attempt.
+ */
+ attempts?: TaskResultAttempt[]
/**
* How long in milliseconds the task took to run.
*/
diff --git a/test/e2e/test/reported-tasks.test.ts b/test/e2e/test/reported-tasks.test.ts
index d3145c3ff..4c23193fe 100644
--- a/test/e2e/test/reported-tasks.test.ts
+++ b/test/e2e/test/reported-tasks.test.ts
@@ -276,6 +276,39 @@ it('correctly reports flaky tests', ({ testModule }) => {
const result = testFlaky.result()!
expect(result.state).toBe('passed')
expect(result.errors).toHaveLength(2)
+ expect(
+ testFlaky.attempts().map((attempt) => ({
+ state: attempt.state,
+ errors: attempt.errors?.map((error) => error.message),
+ retryIndex: attempt.retryIndex,
+ repeatIndex: attempt.repeatIndex,
+ })),
+ ).toMatchInlineSnapshot(`
+ [
+ {
+ "errors": [
+ "expected +0 to be 2 // Object.is equality",
+ ],
+ "repeatIndex": 0,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ {
+ "errors": [
+ "expected 1 to be 2 // Object.is equality",
+ ],
+ "repeatIndex": 0,
+ "retryIndex": 1,
+ "state": "failed",
+ },
+ {
+ "errors": undefined,
+ "repeatIndex": 0,
+ "retryIndex": 2,
+ "state": "passed",
+ },
+ ]
+ `)
})
it('correctly reports repeated tests', ({ testModule }) => {
@@ -287,6 +320,65 @@ it('correctly reports repeated tests', ({ testModule }) => {
const result = testRepeated.result()!
expect(result.state).toBe('failed')
expect(result.errors).toHaveLength(6)
+ expect(
+ testRepeated.attempts().map((attempt) => ({
+ state: attempt.state,
+ errors: attempt.errors?.map((error) => error.message),
+ retryIndex: attempt.retryIndex,
+ repeatIndex: attempt.repeatIndex,
+ })),
+ ).toMatchInlineSnapshot(`
+ [
+ {
+ "errors": [
+ "expected 1 to be 2 // Object.is equality",
+ ],
+ "repeatIndex": 0,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ {
+ "errors": [
+ "expected 1 to be 2 // Object.is equality",
+ ],
+ "repeatIndex": 1,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ {
+ "errors": [
+ "expected 1 to be 2 // Object.is equality",
+ ],
+ "repeatIndex": 2,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ {
+ "errors": [
+ "expected 1 to be 2 // Object.is equality",
+ ],
+ "repeatIndex": 3,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ {
+ "errors": [
+ "expected 1 to be 2 // Object.is equality",
+ ],
+ "repeatIndex": 4,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ {
+ "errors": [
+ "expected 1 to be 2 // Object.is equality",
+ ],
+ "repeatIndex": 5,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ ]
+ `)
})
it('correctly passed down metadata', ({ testModule }) => {
diff --git a/test/e2e/test/retry.test.ts b/test/e2e/test/retry.test.ts
index 17a29fc44..566a7e8a0 100644
--- a/test/e2e/test/retry.test.ts
+++ b/test/e2e/test/retry.test.ts
@@ -131,7 +131,7 @@ test('expected failures exhaust retries in every repeat when assertions pass', a
})
test('expected failures can recover through a retry in every repeat', async () => {
- const { stderr, errorTree } = await runInlineTests({
+ const { stderr, errorTree, results } = await runInlineTests({
'repeats.test.js': `
import { afterAll, expect, it } from 'vitest'
@@ -157,6 +157,161 @@ test('expected failures can recover through a retry in every repeat', async () =
},
}
`)
+
+ const [test] = results[0].children.allTests()
+ const attempts = test.attempts()
+ expect(
+ attempts.map((attempt) => ({
+ state: attempt.state,
+ errors: attempt.errors?.map((error) => error.message) || [],
+ retryIndex: attempt.retryIndex,
+ repeatIndex: attempt.repeatIndex,
+ })),
+ ).toMatchInlineSnapshot(`
+ [
+ {
+ "errors": [
+ "Expect test to fail",
+ ],
+ "repeatIndex": 0,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ {
+ "errors": [],
+ "repeatIndex": 0,
+ "retryIndex": 1,
+ "state": "passed",
+ },
+ {
+ "errors": [
+ "Expect test to fail",
+ ],
+ "repeatIndex": 1,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ {
+ "errors": [],
+ "repeatIndex": 1,
+ "retryIndex": 1,
+ "state": "passed",
+ },
+ {
+ "errors": [
+ "Expect test to fail",
+ ],
+ "repeatIndex": 2,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ {
+ "errors": [],
+ "repeatIndex": 2,
+ "retryIndex": 1,
+ "state": "passed",
+ },
+ ]
+ `)
+ attempts.forEach((attempt) => {
+ expect(attempt.duration).toBeGreaterThanOrEqual(0)
+ expect(attempt.startTime).toBeGreaterThan(0)
+ })
+})
+
+test('attempts grow while the test runs', async () => {
+ const { stderr, errorTree } = await runInlineTests({
+ 'attempts.test.js': `
+ import { afterAll, afterEach, expect, it } from 'vitest'
+
+ const seen = []
+ let runs = 0
+
+ // the current attempt is recorded after afterEach, so only previous attempts are visible
+ afterEach(({ task }) => {
+ seen.push(task.result.attempts.map(({ repeatIndex, retryIndex, state }) => ({ repeatIndex, retryIndex, state })))
+ })
+
+ // | repeat 0 | repeat 1 |
+ // | retry 0 -> retry 1 | retry 0 -> retry 1 |
+ // | fail -> pass | fail -> pass |
+ it('flaky', { retry: 1, repeats: 1 }, () => {
+ expect(++runs % 2).toBe(0)
+ })
+
+ afterAll(() => {
+ expect(seen).toEqual([
+ [],
+ [
+ { repeatIndex: 0, retryIndex: 0, state: 'fail' },
+ ],
+ [
+ { repeatIndex: 0, retryIndex: 0, state: 'fail' },
+ { repeatIndex: 0, retryIndex: 1, state: 'pass' },
+ ],
+ [
+ { repeatIndex: 0, retryIndex: 0, state: 'fail' },
+ { repeatIndex: 0, retryIndex: 1, state: 'pass' },
+ { repeatIndex: 1, retryIndex: 0, state: 'fail' },
+ ],
+ ])
+ })
+ `,
+ })
+
+ expect(stderr).toBe('')
+ expect(errorTree()).toMatchInlineSnapshot(`
+ {
+ "attempts.test.js": {
+ "flaky": "passed",
+ },
+ }
+ `)
+})
+
+test('skipping during a retry ends attempts with a skipped attempt', async () => {
+ const { stderr, results } = await runInlineTests({
+ 'skip.test.js': `
+ import { expect, it } from 'vitest'
+
+ let runs = 0
+
+ // | repeat 0 | repeat 1 |
+ // | retry 0 -> retry 1 -> retry 2 | |
+ // | fail -> skip -> (not run) | (not run) |
+ it('skips on retry', { retry: 2, repeats: 1 }, ({ skip }) => {
+ if (++runs === 2) {
+ skip()
+ }
+ expect(1).toBe(2)
+ })
+
+ it.skip('skipped statically', () => {})
+ `,
+ })
+
+ expect(stderr).toBe('')
+ const [skipsOnRetry, skippedStatically] = results[0].children.allTests()
+ expect(skipsOnRetry.result().state).toBe('skipped')
+ expect(
+ skipsOnRetry
+ .attempts()
+ .map(({ repeatIndex, retryIndex, state }) => ({ repeatIndex, retryIndex, state })),
+ ).toMatchInlineSnapshot(`
+ [
+ {
+ "repeatIndex": 0,
+ "retryIndex": 0,
+ "state": "failed",
+ },
+ {
+ "repeatIndex": 0,
+ "retryIndex": 1,
+ "state": "skipped",
+ },
+ ]
+ `)
+ expect(skippedStatically.attempts()).toEqual([])
})
test('syntax errors remain failures after successful repeats', async () => {