diff --git a/docs/.vitepress/components.d.ts b/docs/.vitepress/components.d.ts index 6d19e0f7d..04e66fb93 100644 --- a/docs/.vitepress/components.d.ts +++ b/docs/.vitepress/components.d.ts @@ -7,6 +7,7 @@ export {} /* prettier-ignore */ declare module 'vue' { export interface GlobalComponents { + Advanced: typeof import('./components/Advanced.vue')['default'] ArrowDown: typeof import('./components/ArrowDown.vue')['default'] BlogIndex: typeof import('./components/BlogIndex.vue')['default'] Box: typeof import('./components/Box.vue')['default'] diff --git a/docs/.vitepress/components/Advanced.vue b/docs/.vitepress/components/Advanced.vue new file mode 100644 index 000000000..7bc97edeb --- /dev/null +++ b/docs/.vitepress/components/Advanced.vue @@ -0,0 +1,5 @@ + diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 10e7648b7..ee388f272 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -993,6 +993,10 @@ export default ({ mode }: { mode: string }) => { text: 'TaskMeta', link: '/api/advanced/metadata', }, + { + text: 'TestArtifact', + link: '/api/advanced/artifacts', + }, ], }, // { diff --git a/docs/api/advanced/artifacts.md b/docs/api/advanced/artifacts.md new file mode 100644 index 000000000..c5393b2b3 --- /dev/null +++ b/docs/api/advanced/artifacts.md @@ -0,0 +1,176 @@ +--- +outline: deep +title: Test Artifacts +--- + +# Test Artifacts 4.0.11 + +::: warning +This is an advanced API. As a user, you most likely want to use [test annotations](/guide/test-annotations) to add notes or context to your tests instead. This is primarily used internally and by library authors. +::: + +Test artifacts allow attaching or recording structured data, files, or metadata during test execution. This is a low-level feature primarily designed for: + +- Internal use ([`annotate`](/guide/test-annotations) is built on top of the artifact system) +- Framework authors creating custom testing tools on top of Vitest + +Each artifact includes: + +- A type discriminator which is a unique identifier for the artifact type +- Custom data, can be any relevant information +- Optional attachments, either files or inline content associated with the artifact +- A source code location indicating where the artifact was created + +Vitest automatically manages attachment serialization (files are copied to [`attachmentsDir`](/config/#attachmentsdir)) and injects source location metadata, so you can focus on the data you want to record. All artifacts **must** extend from [`TestArtifactBase`](#testartifactbase) and all attachments from [`TestAttachment`](#testattachment) to be correctly handled internally. + +## API + +### `recordArtifact` {#recordartifact} + +::: warning +`recordArtifact` is an experimental API. Breaking changes might not follow SemVer, please pin Vitest's version when using it. + +The API surface may change based on feedback. We encourage you to try it out and share your experience with the team. +::: + +```ts +function recordArtifact(task: Test, artifact: Artifact): Promise +``` + +The `recordArtifact` function records an artifact during test execution and returns it. It expects a [task](/api/advanced/runner#tasks) as the first parameter and an object assignable to [`TestArtifact`](#testartifact) as the second. + +This function has to be used within a test, and the test has to still be running. Recording after test completion will throw an error. + +When an artifact is recorded on a test, it emits an `onTestArtifactRecord` runner event and a [`onTestCaseArtifactRecord` reporter event](/api/advanced/reporters#ontestcaseartifactrecord). + +Note: annotations, [even though they're built on top of this feature](#relationship-with-annotations), won't appear in the `task.artifacts` array for backwards compatibility reasons until the next major version. + +### `TestArtifact` + +The `TestArtifact` type is a union containing all artifacts Vitest can produce, including custom ones. All artifacts extend from [`TestArtifactBase`](#testartifactbase) + +### `TestArtifactBase` {#testartifactbase} + +```ts +export interface TestArtifactBase { + /** File or data attachments associated with this artifact */ + attachments?: TestAttachment[] + /** Source location where this artifact was created */ + location?: TestArtifactLocation +} +``` + +The `TestArtifactBase` interface is the base for all test artifacts. + +Extend this interface when creating custom test artifacts. Vitest automatically manages the `attachments` array and injects the `location` property to indicate where the artifact was created in your test code. + +### `TestAttachment` + +```ts +export interface TestAttachment { + /** MIME type of the attachment (e.g., 'image/png', 'text/plain') */ + contentType?: string + /** File system path to the attachment */ + path?: string + /** Inline attachment content as a string or raw binary data */ + body?: string | Uint8Array +} +``` + +The `TestAttachment` interface represents a file or data attachment associated with a test artifact. + +Attachments can be either file-based (via `path`) or inline content (via `body`). The `contentType` helps consumers understand how to interpret the attachment data. + +### `TestArtifactLocation` + +```ts +export interface TestArtifactLocation { + /** Line number in the source file (1-indexed) */ + line: number + /** Column number in the line (1-indexed) */ + column: number + /** Path to the source file */ + file: string +} +``` + +The `TestArtifactLocation` interface represents the source code location information for a test artifact. It indicates where in the source code the artifact originated from. + +### `TestArtifactRegistry` + +The `TestArtifactRegistry` interface is a registry for custom test artifact types. + +Augmenting this interface using [TypeScript's module augmentation feature](https://typescriptlang.org/docs/handbook/declaration-merging#module-augmentation) allows registering custom artifact types that tests can produce. + +Each custom artifact should extend [`TestArtifactBase`](#testartifactbase) and include a unique `type` discriminator property. + +Here are a few guidelines or best practices to follow: + +- Try using a `Symbol` as the **registry key** to guarantee uniqueness +- The `type` property should follow the pattern `'package-name:artifact-name'`, **`'internal:'` is a reserved prefix** +- Use `attachments` to include files or data; extend [`TestAttachment`](#testattachment) for custom metadata +- `location` property is automatically injected + +## Custom Artifacts + +To use and manage artifacts in a type-safe manner, you need to create its type and register it: + +```ts +import type { TestArtifactBase, TestAttachment } from 'vitest' + +interface A11yReportAttachment extends TestAttachment { + contentType: 'text/html' + path: string +} + +interface AccessibilityArtifact extends TestArtifactBase { + type: 'a11y:report' + passed: boolean + wcagLevel: 'A' | 'AA' | 'AAA' + attachments: [A11yReportAttachment] +} + +const a11yReportKey = Symbol('report') + +declare module 'vitest' { + interface TestArtifactRegistry { + [a11yReportKey]: AccessibilityArtifact + } +} +``` + +As long as the types are assignable to their bases and don't have errors, everything should work fine and you should be able to record artifacts using [`recordArtifact`](#recordartifact): + +```ts +async function toBeAccessible( + this: MatcherState, + actual: Element, + wcagLevel: 'A' | 'AA' | 'AAA' = 'AA' +): AsyncExpectationResult { + const report = await runAccessibilityAudit(actual, wcagLevel) + + await recordArtifact(this.task, { + type: 'a11y:report', + passed: report.violations.length === 0, + wcagLevel, + attachments: [{ + contentType: 'text/html', + path: report.path, + }], + }) + + return { + pass: violations.length === 0, + message: () => `Found ${report.violations.length} accessibility violation(s)` + } +} +``` + +## Relationship with Annotations + +Test annotations are built on top of the artifact system. When using annotations in tests, they create `internal:annotation` artifacts under the hood. However, annotations are: + +- Simpler to use +- Designed for end-users, not developers + +Use annotations if you just want to add notes to your tests. Use artifacts if you need custom data. diff --git a/docs/api/advanced/reporters.md b/docs/api/advanced/reporters.md index 90f4ddc39..9060c9b8d 100644 --- a/docs/api/advanced/reporters.md +++ b/docs/api/advanced/reporters.md @@ -16,6 +16,7 @@ Vitest has its own test run lifecycle. These are represented by reporter's metho - [`onHookEnd(beforeAll)`](#onhookend) - [`onTestCaseReady`](#ontestcaseready) - [`onTestAnnotate`](#ontestannotate) 3.2.0 + - [`onTestCaseArtifactRecord`](#ontestcaseartifactrecord) 4.0.11 - [`onHookStart(beforeEach)`](#onhookstart) - [`onHookEnd(beforeEach)`](#onhookend) - [`onHookStart(afterEach)`](#onhookstart) @@ -332,3 +333,18 @@ function onTestAnnotate( The `onTestAnnotate` hook is associated with the [`context.annotate`](/guide/test-context#annotate) method. When `annotate` is invoked, Vitest serialises it and sends the same attachment to the main thread where reporter can interact with it. If the path is specified, Vitest stores it in a separate directory (configured by [`attachmentsDir`](/config/#attachmentsdir)) and modifies the `path` property to reference it. + +## onTestCaseArtifactRecord 4.0.11 {#ontestcaseartifactrecord} + +```ts +function onTestCaseArtifactRecord( + testCase: TestCase, + artifact: TestArtifact, +): Awaitable +``` + +The `onTestCaseArtifactRecord` hook is associated with the [`recordArtifact`](/api/advanced/artifacts#recordartifact) utility. When `recordArtifact` is invoked, Vitest serialises it and sends the same attachment to the main thread where reporter can interact with it. + +If the path is specified, Vitest stores it in a separate directory (configured by [`attachmentsDir`](/config/#attachmentsdir)) and modifies the `path` property to reference it. + +Note: annotations, [even though they're built on top of this feature](/api/advanced/artifacts#relationship-with-annotations), won't hit this hook and won't appear in the `task.artifacts` array for backwards compatibility reasons until the next major version. diff --git a/packages/browser-playwright/src/commands/trace.ts b/packages/browser-playwright/src/commands/trace.ts index ca54ec006..55b26453d 100644 --- a/packages/browser-playwright/src/commands/trace.ts +++ b/packages/browser-playwright/src/commands/trace.ts @@ -106,20 +106,25 @@ export const annotateTraces: BrowserCommand<[{ traces: string[]; testId: string const vitest = project.vitest await Promise.all(traces.map((trace) => { const entity = vitest.state.getReportedEntityById(testId) - return vitest._testRun.annotate(testId, { - message: relative(project.config.root, trace), - type: 'traces', - attachment: { - path: trace, - contentType: 'application/octet-stream', + const location = entity?.location + ? { + file: entity.module.moduleId, + line: entity.location.line, + column: entity.location.column, + } + : undefined + return vitest._testRun.recordArtifact(testId, { + type: 'internal:annotation', + annotation: { + message: relative(project.config.root, trace), + type: 'traces', + attachment: { + path: trace, + contentType: 'application/octet-stream', + }, + location, }, - location: entity?.location - ? { - file: entity.module.moduleId, - line: entity.location.line, - column: entity.location.column, - } - : undefined, + location, }) })) } diff --git a/packages/browser/src/client/tester/runner.ts b/packages/browser/src/client/tester/runner.ts index c328736e4..00da160aa 100644 --- a/packages/browser/src/client/tester/runner.ts +++ b/packages/browser/src/client/tester/runner.ts @@ -7,6 +7,7 @@ import type { TaskResultPack, Test, TestAnnotation, + TestArtifact, VitestRunner, } from '@vitest/runner' import type { SerializedConfig, TestExecutionMethod, WorkerGlobalState } from 'vitest' @@ -227,28 +228,41 @@ export function createBrowserRunner( } onTestAnnotate = (test: Test, annotation: TestAnnotation): Promise => { - if (annotation.location) { + const artifact: TestArtifact = { type: 'internal:annotation', annotation, location: annotation.location } + + return this.onTestArtifactRecord(test, artifact).then(({ annotation }) => annotation) + } + + onTestArtifactRecord = (test: Test, artifact: Artifact): Promise => { + if (artifact.location) { // the file should be the test file // tests from other files are not supported - const map = this.sourceMapCache.get(annotation.location.file) + const map = this.sourceMapCache.get(artifact.location.file) + if (!map) { - return rpc().onTaskAnnotate(test.id, annotation) + return rpc().onTaskArtifactRecord(test.id, artifact) } - const traceMap = new DecodedMap(map as any, annotation.location.file) - const position = getOriginalPosition(traceMap, annotation.location) + const traceMap = new DecodedMap(map as any, artifact.location.file) + const position = getOriginalPosition(traceMap, artifact.location) + if (position) { const { source, column, line } = position - const file = source || annotation.location.file - annotation.location = { + const file = source || artifact.location.file + artifact.location = { line, column: column + 1, // if the file path is on windows, we need to remove the starting slash file: file.match(/\/\w:\//) ? file.slice(1) : file, } + + if (artifact.type === 'internal:annotation') { + artifact.annotation.location = artifact.location + } } } - return rpc().onTaskAnnotate(test.id, annotation) + + return rpc().onTaskArtifactRecord(test.id, artifact) } onTaskUpdate = (task: TaskResultPack[], events: TaskEventPack[]): Promise => { diff --git a/packages/browser/src/node/rpc.ts b/packages/browser/src/node/rpc.ts index f3235dd79..5fe2b740b 100644 --- a/packages/browser/src/node/rpc.ts +++ b/packages/browser/src/node/rpc.ts @@ -150,8 +150,8 @@ export function setupBrowserRpc(globalServer: ParentBrowserProject, defaultMocke await vitest._testRun.collected(project, files) } }, - async onTaskAnnotate(id, annotation) { - return vitest._testRun.annotate(id, annotation) + async onTaskArtifactRecord(id, artifact) { + return vitest._testRun.recordArtifact(id, artifact) }, async onTaskUpdate(method, packs, events) { if (method === 'collect') { diff --git a/packages/browser/src/types.ts b/packages/browser/src/types.ts index 0b06d0d26..3265d94c8 100644 --- a/packages/browser/src/types.ts +++ b/packages/browser/src/types.ts @@ -1,5 +1,5 @@ import type { MockedModuleSerialized, ServerIdResolution, ServerMockResolution } from '@vitest/mocker' -import type { TaskEventPack, TaskResultPack, TestAnnotation } from '@vitest/runner' +import type { TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner' import type { BirpcReturn } from 'birpc' import type { AfterSuiteRunMeta, @@ -18,7 +18,7 @@ export interface WebSocketBrowserHandlers { onUnhandledError: (error: unknown, type: string) => Promise onQueued: (method: TestExecutionMethod, file: RunnerTestFile) => void onCollected: (method: TestExecutionMethod, files: RunnerTestFile[]) => Promise - onTaskAnnotate: (testId: string, annotation: TestAnnotation) => Promise + onTaskArtifactRecord: (testId: string, artifact: Artifact) => Promise onTaskUpdate: (method: TestExecutionMethod, packs: TaskResultPack[], events: TaskEventPack[]) => void onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void cancelCurrentRun: (reason: CancelReason) => void diff --git a/packages/runner/src/artifact.ts b/packages/runner/src/artifact.ts new file mode 100644 index 000000000..2b0c2b797 --- /dev/null +++ b/packages/runner/src/artifact.ts @@ -0,0 +1,184 @@ +import type { Test, TestArtifact, TestArtifactBase /* used in JSDoc */, TestAttachment } from './types' +import { finishSendTasksUpdate } from './run' +import { getRunner } from './suite' +import { findTestFileStackTrace } from './utils/collect' + +/** + * @experimental + * @advanced + * + * Records a custom test artifact during test execution. + * + * This function allows you to attach structured data, files, or metadata to a test. + * + * Vitest automatically injects the source location where the artifact was created and manages any attachments you include. + * + * @param task - The test task context, typically accessed via `this.task` in custom matchers or `context.task` in tests + * @param artifact - The artifact to record. Must extend {@linkcode TestArtifactBase} + * + * @returns A promise that resolves to the recorded artifact with location injected + * + * @throws {Error} If called after the test has finished running + * @throws {Error} If the test runner doesn't support artifacts + * + * @example + * ```ts + * // In a custom assertion + * async function toHaveValidSchema(this: MatcherState, actual: unknown) { + * const validation = validateSchema(actual) + * + * await recordArtifact(this.task, { + * type: 'my-plugin:schema-validation', + * passed: validation.valid, + * errors: validation.errors, + * }) + * + * return { pass: validation.valid, message: () => '...' } + * } + * ``` + */ +export async function recordArtifact(task: Test, artifact: Artifact): Promise { + const runner = getRunner() + + if (task.result && task.result.state !== 'run') { + throw new Error(`Cannot record a test artifact outside of the test run. The test "${task.name}" finished running with the "${task.result.state}" state already.`) + } + + const stack = findTestFileStackTrace( + task.file.filepath, + new Error('STACK_TRACE').stack!, + ) + + if (stack) { + artifact.location = { + file: stack.file, + line: stack.line, + column: stack.column, + } + + if (artifact.type === 'internal:annotation') { + artifact.annotation.location = artifact.location + } + } + + if (Array.isArray(artifact.attachments)) { + for (const attachment of artifact.attachments) { + manageArtifactAttachment(attachment) + } + } + + // annotations won't resolve as artifacts for backwards compatibility until next major + if (artifact.type === 'internal:annotation') { + return artifact + } + + if (!runner.onTestArtifactRecord) { + throw new Error(`Test runner doesn't support test artifacts.`) + } + + await finishSendTasksUpdate(runner) + + const resolvedArtifact = await runner.onTestArtifactRecord(task, artifact) + + task.artifacts.push(resolvedArtifact) + + return resolvedArtifact as typeof artifact +} + +const table: string[] = [] +for (let i = 65; i < 91; i++) { + table.push(String.fromCharCode(i)) // A-Z +} +for (let i = 97; i < 123; i++) { + table.push(String.fromCharCode(i)) // a-z +} +for (let i = 0; i < 10; i++) { + table.push(i.toString(10)) // 0-9 +} +table.push('+', '/') + +function encodeUint8Array(bytes: Uint8Array): string { + let base64 = '' + const len = bytes.byteLength + for (let i = 0; i < len; i += 3) { + if (len === i + 1) { // last 1 byte + const a = (bytes[i] & 0xFC) >> 2 + const b = ((bytes[i] & 0x03) << 4) + base64 += table[a] + base64 += table[b] + base64 += '==' + } + else if (len === i + 2) { // last 2 bytes + const a = (bytes[i] & 0xFC) >> 2 + const b = ((bytes[i] & 0x03) << 4) | ((bytes[i + 1] & 0xF0) >> 4) + const c = ((bytes[i + 1] & 0x0F) << 2) + base64 += table[a] + base64 += table[b] + base64 += table[c] + base64 += '=' + } + else { + const a = (bytes[i] & 0xFC) >> 2 + const b = ((bytes[i] & 0x03) << 4) | ((bytes[i + 1] & 0xF0) >> 4) + const c = ((bytes[i + 1] & 0x0F) << 2) | ((bytes[i + 2] & 0xC0) >> 6) + const d = bytes[i + 2] & 0x3F + base64 += table[a] + base64 += table[b] + base64 += table[c] + base64 += table[d] + } + } + return base64 +} + +/** + * Records an async operation associated with a test task. + * + * This function tracks promises that should be awaited before a test completes. + * The promise is automatically removed from the test's promise list once it settles. + */ +export function recordAsyncOperation( + test: Test, + promise: Promise, +): Promise { + // if promise is explicitly awaited, remove it from the list + promise = promise.finally(() => { + if (!test.promises) { + return + } + const index = test.promises.indexOf(promise) + if (index !== -1) { + test.promises.splice(index, 1) + } + }) + + // record promise + if (!test.promises) { + test.promises = [] + } + test.promises.push(promise) + return promise +} + +/** + * Validates and prepares a test attachment for serialization. + * + * This function ensures attachments have either `body` or `path` set (but not both), and converts `Uint8Array` bodies to base64-encoded strings for easier serialization. + * + * @param attachment - The attachment to validate and prepare + * + * @throws {TypeError} If neither `body` nor `path` is provided + * @throws {TypeError} If both `body` and `path` are provided + */ +export function manageArtifactAttachment(attachment: TestAttachment): void { + if (attachment.body == null && !attachment.path) { + throw new TypeError(`Test attachment requires "body" or "path" to be set. Both are missing.`) + } + if (attachment.body && attachment.path) { + throw new TypeError(`Test attachment requires only one of "body" or "path" to be set. Both are specified.`) + } + // convert to a string so it's easier to serialise + if (attachment.body instanceof Uint8Array) { + attachment.body = encodeUint8Array(attachment.body) + } +} diff --git a/packages/runner/src/context.ts b/packages/runner/src/context.ts index 73e421497..c42d49db7 100644 --- a/packages/runner/src/context.ts +++ b/packages/runner/src/context.ts @@ -6,16 +6,14 @@ import type { SuiteCollector, Test, TestAnnotation, - TestAnnotationLocation, - TestAttachment, TestContext, WriteableTestContext, } from './types/tasks' import { getSafeTimers } from '@vitest/utils/timers' +import { manageArtifactAttachment, recordArtifact, recordAsyncOperation } from './artifact' import { PendingError } from './errors' import { finishSendTasksUpdate } from './run' import { getRunner } from './suite' -import { findTestFileStackTrace } from './utils' const now = Date.now @@ -157,76 +155,37 @@ export function createTestContext( ) } - async function annotate( - message: string, - location?: TestAnnotationLocation, - type?: string, - attachment?: TestAttachment, - ) { - const annotation: TestAnnotation = { - message, - type: type || 'notice', - } - if (attachment) { - if (attachment.body == null && !attachment.path) { - throw new TypeError(`Test attachment requires "body" or "path" to be set. Both are missing.`) - } - if (attachment.body && attachment.path) { - throw new TypeError(`Test attachment requires only one of "body" or "path" to be set. Both are specified.`) - } - annotation.attachment = attachment - // convert to a string so it's easier to serialise - if (attachment.body instanceof Uint8Array) { - attachment.body = encodeUint8Array(attachment.body) - } - } - if (location) { - annotation.location = location - } - - if (!runner.onTestAnnotate) { - throw new Error(`Test runner doesn't support test annotations.`) - } - - await finishSendTasksUpdate(runner) - - const resolvedAnnotation = await runner.onTestAnnotate(test, annotation) - test.annotations.push(resolvedAnnotation) - return resolvedAnnotation - } - context.annotate = ((message, type, attachment) => { if (test.result && test.result.state !== 'run') { throw new Error(`Cannot annotate tests outside of the test run. The test "${test.name}" finished running with the "${test.result.state}" state already.`) } - const stack = findTestFileStackTrace( - test.file.filepath, - new Error('STACK_TRACE').stack!, - ) + const annotation: TestAnnotation = { + message, + type: typeof type === 'object' || type === undefined ? 'notice' : type, + } + const annotationAttachment = typeof type === 'object' ? type : attachment - let location: undefined | TestAnnotationLocation + if (annotationAttachment) { + annotation.attachment = annotationAttachment - if (stack) { - location = { - file: stack.file, - line: stack.line, - column: stack.column, - } + manageArtifactAttachment(annotation.attachment) } - if (typeof type === 'object') { - return recordAsyncAnnotation( - test, - annotate(message, location, undefined, type), - ) - } - else { - return recordAsyncAnnotation( - test, - annotate(message, location, type, attachment), - ) - } + return recordAsyncOperation( + test, + recordArtifact(test, { type: 'internal:annotation', annotation }).then(async ({ annotation }) => { + if (!runner.onTestAnnotate) { + throw new Error(`Test runner doesn't support test annotations.`) + } + + await finishSendTasksUpdate(runner) + + const resolvedAnnotation = await runner.onTestAnnotate(test, annotation) + test.annotations.push(resolvedAnnotation) + return resolvedAnnotation + }), + ) }) as TestContext['annotate'] context.onTestFailed = (handler, timeout) => { @@ -286,72 +245,3 @@ export function getFileContext(file: File): Record { export function setFileContext(file: File, context: Record): void { fileContexts.set(file, context) } - -const table: string[] = [] -for (let i = 65; i < 91; i++) { - table.push(String.fromCharCode(i)) // A-Z -} -for (let i = 97; i < 123; i++) { - table.push(String.fromCharCode(i)) // a-z -} -for (let i = 0; i < 10; i++) { - table.push(i.toString(10)) // 0-9 -} -table.push('+', '/') - -function encodeUint8Array(bytes: Uint8Array): string { - let base64 = '' - const len = bytes.byteLength - for (let i = 0; i < len; i += 3) { - if (len === i + 1) { // last 1 byte - const a = (bytes[i] & 0xFC) >> 2 - const b = ((bytes[i] & 0x03) << 4) - base64 += table[a] - base64 += table[b] - base64 += '==' - } - else if (len === i + 2) { // last 2 bytes - const a = (bytes[i] & 0xFC) >> 2 - const b = ((bytes[i] & 0x03) << 4) | ((bytes[i + 1] & 0xF0) >> 4) - const c = ((bytes[i + 1] & 0x0F) << 2) - base64 += table[a] - base64 += table[b] - base64 += table[c] - base64 += '=' - } - else { - const a = (bytes[i] & 0xFC) >> 2 - const b = ((bytes[i] & 0x03) << 4) | ((bytes[i + 1] & 0xF0) >> 4) - const c = ((bytes[i + 1] & 0x0F) << 2) | ((bytes[i + 2] & 0xC0) >> 6) - const d = bytes[i + 2] & 0x3F - base64 += table[a] - base64 += table[b] - base64 += table[c] - base64 += table[d] - } - } - return base64 -} - -function recordAsyncAnnotation( - test: Test, - promise: Promise, -): Promise { - // if promise is explicitly awaited, remove it from the list - promise = promise.finally(() => { - if (!test.promises) { - return - } - const index = test.promises.indexOf(promise) - if (index !== -1) { - test.promises.splice(index, 1) - } - }) - - // record promise - if (!test.promises) { - test.promises = [] - } - test.promises.push(promise) - return promise -} diff --git a/packages/runner/src/index.ts b/packages/runner/src/index.ts index 3886a4db3..39a91ee3e 100644 --- a/packages/runner/src/index.ts +++ b/packages/runner/src/index.ts @@ -1,3 +1,4 @@ +export { recordArtifact } from './artifact' export { afterAll, afterEach, diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index f8e73c7d4..601f48063 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -318,6 +318,7 @@ function createSuiteCollector( : 'run', meta: options.meta ?? Object.create(null), annotations: [], + artifacts: [], } const handler = options.handler if (task.mode === 'run' && !handler) { diff --git a/packages/runner/src/types.ts b/packages/runner/src/types.ts index 177e18521..eaccf8148 100644 --- a/packages/runner/src/types.ts +++ b/packages/runner/src/types.ts @@ -42,8 +42,13 @@ export type { TaskUpdateEvent, Test, TestAnnotation, + TestAnnotationArtifact, TestAnnotationLocation, TestAPI, + TestArtifact, + TestArtifactBase, + TestArtifactLocation, + TestArtifactRegistry, TestAttachment, TestContext, TestFunction, diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index 266de0493..51695e8c5 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -9,6 +9,7 @@ import type { TaskResultPack, Test, TestAnnotation, + TestArtifact, TestContext, } from './tasks' @@ -145,6 +146,13 @@ export interface VitestRunner { */ onTestAnnotate?: (test: Test, annotation: TestAnnotation) => Promise + /** + * @experimental + * + * Called when artifacts are recorded on tests via the `recordArtifact` utility. + */ + onTestArtifactRecord?: (test: Test, artifact: Artifact) => Promise + /** * Called before running all tests in collected paths. */ diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index d0afc9ca0..b726e0829 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -192,6 +192,7 @@ export type TaskResultPack = [ export interface TaskEventData { annotation?: TestAnnotation | undefined + artifact?: TestArtifact | undefined } export type TaskEventPack = [ @@ -222,6 +223,7 @@ export type TaskUpdateEvent | 'after-hook-start' | 'after-hook-end' | 'test-annotation' + | 'test-artifact' export interface Suite extends TaskBase { type: 'suite' @@ -283,25 +285,12 @@ export interface Test extends TaskPopulated { * An array of custom annotations. */ annotations: TestAnnotation[] -} - -export interface TestAttachment { - contentType?: string - path?: string - body?: string | Uint8Array -} - -export interface TestAnnotationLocation { - line: number - column: number - file: string -} - -export interface TestAnnotation { - message: string - type: string - location?: TestAnnotationLocation - attachment?: TestAttachment + /** + * An array of artifacts produced by the test. + * + * @experimental + */ + artifacts: TestArtifact[] } export type Task = Test | Suite | File @@ -721,3 +710,155 @@ export type SequenceSetupFiles = 'list' | 'parallel' export type WriteableTestContext = { -readonly [P in keyof TestContext]: TestContext[P] } + +// test artifacts + +/** + * Represents a file or data attachment associated with a test artifact. + * + * Attachments can be either file-based (via `path`) or inline content (via `body`). + * The `contentType` helps consumers understand how to interpret the attachment data. + */ +export interface TestAttachment { + /** MIME type of the attachment (e.g., 'image/png', 'text/plain') */ + contentType?: string + /** File system path to the attachment */ + path?: string + /** Inline attachment content as a string or raw binary data */ + body?: string | Uint8Array +} + +/** + * Source code location information for a test artifact. + * + * Indicates where in the source code the artifact originated from. + */ +export interface TestArtifactLocation { + /** Line number in the source file (1-indexed) */ + line: number + /** Column number in the line (1-indexed) */ + column: number + /** Path to the source file */ + file: string +} + +/** + * @experimental + * + * Base interface for all test artifacts. + * + * Extend this interface when creating custom test artifacts. Vitest automatically manages the `attachments` array and injects the `location` property to indicate where the artifact was created in your test code. + */ +export interface TestArtifactBase { + /** File or data attachments associated with this artifact */ + attachments?: TestAttachment[] + /** Source location where this artifact was created */ + location?: TestArtifactLocation +} + +/** + * @deprecated Use {@linkcode TestArtifactLocation} instead. + * + * Kept for backwards compatibility. + */ +export type TestAnnotationLocation = TestArtifactLocation + +export interface TestAnnotation { + message: string + type: string + location?: TestArtifactLocation + attachment?: TestAttachment +} + +/** + * @experimental + * + * Artifact type for test annotations. + */ +export interface TestAnnotationArtifact extends TestArtifactBase { + type: 'internal:annotation' + annotation: TestAnnotation +} + +/** + * @experimental + * @advanced + * + * Registry for custom test artifact types. + * + * Augment this interface to register custom artifact types that your tests can produce. + * + * Each custom artifact should extend {@linkcode TestArtifactBase} and include a unique `type` discriminator property. + * + * @remarks + * - Use a `Symbol` as the **registry key** to guarantee uniqueness + * - The `type` property should follow the pattern `'package-name:artifact-name'`, `'internal:'` is a reserved prefix + * - Use `attachments` to include files or data; extend {@linkcode TestAttachment} for custom metadata + * - `location` property is automatically injected to indicate where the artifact was created + * + * @example + * ```ts + * // Define custom attachment type for generated PDF + * interface PDFAttachment extends TestAttachment { + * contentType: 'application/pdf' + * body: Uint8Array + * pageCount: number + * fileSize: number + * } + * + * interface PDFGenerationArtifact extends TestArtifactBase { + * type: 'my-plugin:pdf-generation' + * templateName: string + * isValid: boolean + * attachments: [PDFAttachment] + * } + * + * // Use a symbol to guarantee key uniqueness + * const pdfKey = Symbol('pdf-generation') + * + * declare module 'vitest' { + * interface TestArtifactRegistry { + * [pdfKey]: PDFGenerationArtifact + * } + * } + * + * // Custom assertion for PDF generation + * async function toGenerateValidPDF( + * this: MatcherState, + * actual: PDFTemplate, + * data: Record + * ): AsyncExpectationResult { + * const pdfBuffer = await actual.render(data) + * const validation = await validatePDF(pdfBuffer) + * + * await recordArtifact(this.task, { + * type: 'my-plugin:pdf-generation', + * templateName: actual.name, + * isValid: validation.success, + * attachments: [{ + * contentType: 'application/pdf', + * body: pdfBuffer, + * pageCount: validation.pageCount, + * fileSize: pdfBuffer.byteLength + * }] + * }) + * + * return { + * pass: validation.success, + * message: () => validation.success + * ? `Generated valid PDF with ${validation.pageCount} pages` + * : `Invalid PDF: ${validation.error}` + * } + * } + * ``` + */ +export interface TestArtifactRegistry {} + +/** + * @experimental + * + * Union type of all test artifacts, including built-in and custom registered artifacts. + * + * This type automatically includes all artifacts registered via {@link TestArtifactRegistry}. + */ +export type TestArtifact = TestAnnotationArtifact | TestArtifactRegistry[keyof TestArtifactRegistry] diff --git a/packages/ui/client/composables/client/index.ts b/packages/ui/client/composables/client/index.ts index a08f86292..d29a40f41 100644 --- a/packages/ui/client/composables/client/index.ts +++ b/packages/ui/client/composables/client/index.ts @@ -34,7 +34,10 @@ export const client = (function createVitestClient() { }, handlers: { onTestAnnotate(testId: string, annotation: TestAnnotation) { - explorerTree.annotateTest(testId, annotation) + explorerTree.recordTestArtifact(testId, { type: 'internal:annotation', annotation, location: annotation.location }) + }, + onTestArtifactRecord(testId, artifact) { + explorerTree.recordTestArtifact(testId, artifact) }, onTaskUpdate(packs: RunnerTaskResultPack[], events: RunnerTaskEventPack[]) { explorerTree.resumeRun(packs, events) diff --git a/packages/ui/client/composables/explorer/collector.ts b/packages/ui/client/composables/explorer/collector.ts index e6f3898dc..87a604883 100644 --- a/packages/ui/client/composables/explorer/collector.ts +++ b/packages/ui/client/composables/explorer/collector.ts @@ -1,4 +1,4 @@ -import type { File, Task, TaskResultPack, Test, TestAnnotation } from '@vitest/runner' +import type { File, Task, TaskResultPack, Test, TestArtifact } from '@vitest/runner' import type { Arrayable } from '@vitest/utils' import type { CollectFilteredTests, CollectorInfo, Filter, FilteredTests } from '~/composables/explorer/types' import { isTestCase } from '@vitest/runner/utils' @@ -66,9 +66,9 @@ export function preparePendingTasks(packs: TaskResultPack[]) { }) } -export function annotateTest( +export function recordTestArtifact( id: string, - annotation: TestAnnotation, + artifact: TestArtifact, ) { const pending = explorerTree.pendingTasks const idMap = client.state.idMap @@ -80,7 +80,13 @@ export function annotateTest( pending.set(test.file.id, file) } file.add(test.id) - test.annotations.push(annotation) + + if (artifact.type === 'internal:annotation') { + test.annotations.push(artifact.annotation) + } + else { + test.artifacts.push(artifact) + } } } diff --git a/packages/ui/client/composables/explorer/tree.ts b/packages/ui/client/composables/explorer/tree.ts index 746710267..faa1d698c 100644 --- a/packages/ui/client/composables/explorer/tree.ts +++ b/packages/ui/client/composables/explorer/tree.ts @@ -1,4 +1,4 @@ -import type { File, TaskResultPack, TestAnnotation } from '@vitest/runner' +import type { File, TaskResultPack, TestArtifact } from '@vitest/runner' import type { RunnerTaskEventPack } from 'vitest' import type { CollectorInfo, @@ -9,7 +9,7 @@ import type { import { useRafFn } from '@vueuse/core' import { reactive } from 'vue' import { runCollapseAllTask, runCollapseNode } from '~/composables/explorer/collapse' -import { annotateTest, collectTestsTotalData, preparePendingTasks, runCollect, runLoadFiles } from '~/composables/explorer/collector' +import { collectTestsTotalData, preparePendingTasks, recordTestArtifact, runCollect, runLoadFiles } from '~/composables/explorer/collector' import { runExpandAll, runExpandNode } from '~/composables/explorer/expand' import { runFilter } from '~/composables/explorer/filter' import { @@ -83,8 +83,8 @@ export class ExplorerTree { this.collect(true, false) } - annotateTest(testId: string, annotation: TestAnnotation) { - annotateTest(testId, annotation) + recordTestArtifact(testId: string, artifact: TestArtifact) { + recordTestArtifact(testId, artifact) if (!this.onTaskUpdateCalled) { clearTimeout(this.resumeEndRunId) this.onTaskUpdateCalled = true diff --git a/packages/vitest/src/api/setup.ts b/packages/vitest/src/api/setup.ts index 8af491347..821dc0ca0 100644 --- a/packages/vitest/src/api/setup.ts +++ b/packages/vitest/src/api/setup.ts @@ -1,4 +1,4 @@ -import type { File, TaskEventPack, TaskResultPack, TestAnnotation } from '@vitest/runner' +import type { File, TaskEventPack, TaskResultPack, TestAnnotation, TestArtifact } from '@vitest/runner' import type { SerializedError } from '@vitest/utils' import type { IncomingMessage } from 'node:http' import type { ViteDevServer } from 'vite' @@ -198,6 +198,16 @@ export class WebSocketReporter implements Reporter { }) } + async onTestCaseArtifactRecord(testCase: TestCase, artifact: TestArtifact): Promise { + if (this.clients.size === 0) { + return + } + + this.clients.forEach((client) => { + client.onTestArtifactRecord?.(testCase.id, artifact)?.catch?.(noop) + }) + } + async onTaskUpdate(packs: TaskResultPack[], events: TaskEventPack[]): Promise { if (this.clients.size === 0) { return diff --git a/packages/vitest/src/api/types.ts b/packages/vitest/src/api/types.ts index e6f4503f4..2d029d365 100644 --- a/packages/vitest/src/api/types.ts +++ b/packages/vitest/src/api/types.ts @@ -1,4 +1,4 @@ -import type { File, TaskEventPack, TaskResultPack, TestAnnotation } from '@vitest/runner' +import type { File, TaskEventPack, TaskResultPack, TestAnnotation, TestArtifact } from '@vitest/runner' import type { Awaitable } from '@vitest/utils' import type { BirpcReturn } from 'birpc' import type { SerializedConfig } from '../runtime/config' @@ -61,6 +61,7 @@ export interface WebSocketEvents { executionTime?: number, ) => Awaitable onTestAnnotate?: (testId: string, annotation: TestAnnotation) => Awaitable + onTestArtifactRecord?: (testId: string, artifact: TestArtifact) => Awaitable onTaskUpdate?: (packs: TaskResultPack[], events: TaskEventPack[]) => Awaitable onUserConsoleLog?: (log: UserConsoleLog) => Awaitable onPathsCollected?: (paths?: string[]) => Awaitable diff --git a/packages/vitest/src/node/ast-collect.ts b/packages/vitest/src/node/ast-collect.ts index d8ca00882..827d9cc82 100644 --- a/packages/vitest/src/node/ast-collect.ts +++ b/packages/vitest/src/node/ast-collect.ts @@ -350,6 +350,7 @@ function createFileTask( meta: {}, timeout: 0, annotations: [], + artifacts: [], } definition.task = task latestSuite.tasks.push(task) diff --git a/packages/vitest/src/node/pools/rpc.ts b/packages/vitest/src/node/pools/rpc.ts index d9dd51c33..302345e34 100644 --- a/packages/vitest/src/node/pools/rpc.ts +++ b/packages/vitest/src/node/pools/rpc.ts @@ -120,8 +120,8 @@ export function createMethodsRPC(project: TestProject, methodsOptions: MethodsOp onAfterSuiteRun(meta) { vitest.coverageProvider?.onAfterSuiteRun(meta) }, - async onTaskAnnotate(testId, annotation) { - return vitest._testRun.annotate(testId, annotation) + async onTaskArtifactRecord(testId, artifact) { + return vitest._testRun.recordArtifact(testId, artifact) }, async onTaskUpdate(packs, events) { if (methodsOptions.collect) { diff --git a/packages/vitest/src/node/reporters/junit.ts b/packages/vitest/src/node/reporters/junit.ts index ae97c8666..7b504775b 100644 --- a/packages/vitest/src/node/reporters/junit.ts +++ b/packages/vitest/src/node/reporters/junit.ts @@ -336,6 +336,7 @@ export class JUnitReporter implements Reporter { suite: null as any, file: null as any, annotations: [], + artifacts: [], } satisfies Task) } diff --git a/packages/vitest/src/node/reporters/reported-tasks.ts b/packages/vitest/src/node/reporters/reported-tasks.ts index f1df17ff4..374aefee5 100644 --- a/packages/vitest/src/node/reporters/reported-tasks.ts +++ b/packages/vitest/src/node/reporters/reported-tasks.ts @@ -6,6 +6,7 @@ import type { Suite as RunnerTestSuite, TaskMeta, TestAnnotation, + TestArtifact, } from '@vitest/runner' import type { SerializedError, TestError } from '@vitest/utils' import type { TestProject } from '../project' @@ -186,6 +187,15 @@ export class TestCase extends ReportedTaskImplementation { return [...this.task.annotations] } + /** + * @experimental + * + * Test artifacts recorded via the `recordArtifact` API during the test execution. + */ + public artifacts(): ReadonlyArray { + return [...this.task.artifacts] + } + /** * Useful information about the test like duration, memory usage, etc. * Diagnostic is only available after the test has finished. diff --git a/packages/vitest/src/node/test-run.ts b/packages/vitest/src/node/test-run.ts index 109e7c901..e9a334540 100644 --- a/packages/vitest/src/node/test-run.ts +++ b/packages/vitest/src/node/test-run.ts @@ -3,10 +3,9 @@ import type { TaskEventPack, TaskResultPack, TaskUpdateEvent, - TestAnnotation, TestAttachment, } from '@vitest/runner' -import type { TaskEventData } from '@vitest/runner/types/tasks' +import type { TaskEventData, TestArtifact } from '@vitest/runner/types/tasks' import type { SerializedError } from '@vitest/utils' import type { UserConsoleLog } from '../types/general' import type { Vitest } from './core' @@ -55,19 +54,35 @@ export class TestRun { await this.vitest.report('onUserConsoleLog', log) } - async annotate(testId: string, annotation: TestAnnotation): Promise { + async recordArtifact(testId: string, artifact: Artifact): Promise { const task = this.vitest.state.idMap.get(testId) const entity = task && this.vitest.state.getReportedEntity(task) assert(task && entity, `Entity must be found for task ${task?.name || testId}`) - assert(entity.type === 'test', `Annotation can only be added to a test, instead got ${entity.type}`) + assert(entity.type === 'test', `Artifacts can only be recorded on a test, instead got ${entity.type}`) - await this.resolveTestAttachment(entity, annotation) + // annotations won't resolve as artifacts for backwards compatibility until next major + if (artifact.type === 'internal:annotation') { + await this.resolveTestAttachment(entity, artifact.annotation.attachment, artifact.annotation.message) - entity.task.annotations.push(annotation) + entity.task.annotations.push(artifact.annotation) - await this.vitest.report('onTestCaseAnnotate', entity, annotation) - return annotation + await this.vitest.report('onTestCaseAnnotate', entity, artifact.annotation) + + return artifact + } + + if (Array.isArray(artifact.attachments)) { + await Promise.all( + artifact.attachments.map(attachment => this.resolveTestAttachment(entity, attachment)), + ) + } + + entity.task.artifacts.push(artifact) + + await this.vitest.report('onTestCaseArtifactRecord', entity, artifact) + + return artifact } async updated(update: TaskResultPack[], events: TaskEventPack[]): Promise { @@ -232,9 +247,8 @@ export class TestRun { } } - private async resolveTestAttachment(test: TestCase, annotation: TestAnnotation): Promise { + private async resolveTestAttachment(test: TestCase, attachment: TestAttachment | undefined, filename?: string): Promise { const project = test.project - const attachment = annotation.attachment if (!attachment) { return attachment } @@ -244,7 +258,7 @@ export class TestRun { const hash = createHash('sha1').update(currentPath).digest('hex') const newPath = resolve( project.config.attachmentsDir, - `${sanitizeFilePath(annotation.message)}-${hash}${extname(currentPath)}`, + `${filename ? `${sanitizeFilePath(filename)}-` : ''}${hash}${extname(currentPath)}`, ) if (!existsSync(project.config.attachmentsDir)) { await mkdir(project.config.attachmentsDir, { recursive: true }) diff --git a/packages/vitest/src/node/types/reporter.ts b/packages/vitest/src/node/types/reporter.ts index 3f5d4b530..9785c3110 100644 --- a/packages/vitest/src/node/types/reporter.ts +++ b/packages/vitest/src/node/types/reporter.ts @@ -1,4 +1,4 @@ -import type { File, TaskEventPack, TaskResultPack, TestAnnotation } from '@vitest/runner' +import type { File, TaskEventPack, TaskResultPack, TestAnnotation, TestArtifact } from '@vitest/runner' import type { Awaitable, SerializedError } from '@vitest/utils' import type { UserConsoleLog } from '../../types/general' import type { Vitest } from '../core' @@ -71,6 +71,11 @@ export interface Reporter { */ onTestCaseAnnotate?: (testCase: TestCase, annotation: TestAnnotation) => Awaitable + /** + * Called when artifacts are recorded on tests via the `recordArtifact` utility. + */ + onTestCaseArtifactRecord?: (testCase: TestCase, artifact: TestArtifact) => Awaitable + /** * Called when test suite is ready to run. * Called before the `beforeAll` hooks for the test are run. diff --git a/packages/vitest/src/public/index.ts b/packages/vitest/src/public/index.ts index ff71f5830..d1655a64b 100644 --- a/packages/vitest/src/public/index.ts +++ b/packages/vitest/src/public/index.ts @@ -80,6 +80,7 @@ export { it, onTestFailed, onTestFinished, + recordArtifact, suite, test, } from '@vitest/runner' @@ -103,7 +104,13 @@ export type { TaskMeta, TaskState, TestAnnotation, + TestAnnotationArtifact, TestAPI, + TestArtifact, + TestArtifactBase, + TestArtifactLocation, + TestArtifactRegistry, + TestAttachment, TestContext, TestFunction, TestOptions, diff --git a/packages/vitest/src/runtime/runners/index.ts b/packages/vitest/src/runtime/runners/index.ts index 0fa87474d..056e4fc8f 100644 --- a/packages/vitest/src/runtime/runners/index.ts +++ b/packages/vitest/src/runtime/runners/index.ts @@ -73,10 +73,18 @@ export async function resolveTestRunner( // patch some methods, so custom runners don't need to call RPC const originalOnTestAnnotate = testRunner.onTestAnnotate testRunner.onTestAnnotate = async (test, annotation) => { - const p = rpc().onTaskAnnotate(test.id, annotation) + const p = rpc().onTaskArtifactRecord(test.id, { type: 'internal:annotation', location: annotation.location, annotation }) const overriddenResult = await originalOnTestAnnotate?.call(testRunner, test, annotation) const vitestResult = await p - return overriddenResult || vitestResult + return overriddenResult || vitestResult.annotation + } + + const originalOnTestArtifactRecord = testRunner.onTestArtifactRecord + testRunner.onTestArtifactRecord = async (test, artifact) => { + const p = rpc().onTaskArtifactRecord(test.id, artifact) + const overriddenResult = await originalOnTestArtifactRecord?.call(testRunner, test, artifact) + const vitestResult = await p + return overriddenResult as typeof artifact || vitestResult } const originalOnCollectStart = testRunner.onCollectStart diff --git a/packages/vitest/src/typecheck/collect.ts b/packages/vitest/src/typecheck/collect.ts index cf81e781d..392a07ef7 100644 --- a/packages/vitest/src/typecheck/collect.ts +++ b/packages/vitest/src/typecheck/collect.ts @@ -208,6 +208,7 @@ export async function collectTests( end: definition.end, start: definition.start, annotations: [], + artifacts: [], meta: { typecheck: true, }, diff --git a/packages/vitest/src/types/rpc.ts b/packages/vitest/src/types/rpc.ts index fa5d3e68e..caf16e628 100644 --- a/packages/vitest/src/types/rpc.ts +++ b/packages/vitest/src/types/rpc.ts @@ -1,4 +1,4 @@ -import type { CancelReason, File, TaskEventPack, TaskResultPack, TestAnnotation } from '@vitest/runner' +import type { CancelReason, File, TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner' import type { SnapshotResult } from '@vitest/snapshot' import type { FetchFunctionOptions, FetchResult } from 'vite/module-runner' import type { OTELCarrier } from '../utils/traces' @@ -20,7 +20,7 @@ export interface RuntimeRPC { onQueued: (file: File) => void onCollected: (files: File[]) => Promise onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void - onTaskAnnotate: (testId: string, annotation: TestAnnotation) => Promise + onTaskArtifactRecord: (testId: string, artifact: Artifact) => Promise onTaskUpdate: (pack: TaskResultPack[], events: TaskEventPack[]) => Promise onCancel: (reason: CancelReason) => void getCountOfFailedTests: () => number diff --git a/packages/vitest/src/utils/tasks.ts b/packages/vitest/src/utils/tasks.ts index 712d7c486..7e4e69daf 100644 --- a/packages/vitest/src/utils/tasks.ts +++ b/packages/vitest/src/utils/tasks.ts @@ -43,6 +43,9 @@ export function convertTasksToEvents(file: File, onTask?: (task: Task) => void): task.annotations.forEach((annotation) => { events.push([task.id, 'test-annotation', { annotation }]) }) + task.artifacts.forEach((artifact) => { + events.push([task.id, 'test-artifact', { artifact }]) + }) events.push([task.id, 'test-finished', undefined]) } } diff --git a/packages/ws-client/src/index.ts b/packages/ws-client/src/index.ts index 9c7b212a1..70919a59a 100644 --- a/packages/ws-client/src/index.ts +++ b/packages/ws-client/src/index.ts @@ -54,6 +54,9 @@ export function createClient(url: string, options: VitestClientOptions = {}): Vi onTestAnnotate(testId, annotation) { handlers.onTestAnnotate?.(testId, annotation) }, + onTestArtifactRecord(testId, artifact) { + handlers.onTestArtifactRecord?.(testId, artifact) + }, onSpecsCollected(specs, startTime) { specs?.forEach(([config, file]) => { ctx.state.clearFiles({ config }, [file]) diff --git a/test/cli/fixtures/custom-pool/pool/custom-pool.ts b/test/cli/fixtures/custom-pool/pool/custom-pool.ts index 2e241cb3b..cd2ad64bf 100644 --- a/test/cli/fixtures/custom-pool/pool/custom-pool.ts +++ b/test/cli/fixtures/custom-pool/pool/custom-pool.ts @@ -92,6 +92,7 @@ async function onMessage(message: WorkerRequest, project: TestProject, options: mode: 'run', meta: {}, annotations: [], + artifacts: [], timeout: 0, file: taskFile, result: { diff --git a/test/cli/test/annotations.test.ts b/test/cli/test/annotations.test.ts index fa956486f..05dac499e 100644 --- a/test/cli/test/annotations.test.ts +++ b/test/cli/test/annotations.test.ts @@ -1,3 +1,4 @@ +import type { TestArtifact } from '@vitest/runner' import type { TestAnnotation } from 'vitest' import { playwright } from '@vitest/browser-playwright' import { describe, expect, test } from 'vitest' @@ -50,6 +51,7 @@ describe('API', () => { ])('annotations are exposed correctly in $name', async (options) => { const events: string[] = [] const annotations: Record> = {} + const artifacts: Record> = {} const { stderr } = await runInlineTests( { @@ -66,6 +68,9 @@ describe('API', () => { const path = annotation.attachment?.path?.replace(testCase.project.config.root, '').replace(/\w+\.js$/, '.js') events.push(`[annotate] ${testCase.name} ${annotation.message} ${annotation.type} path=${path} contentType=${annotation.attachment?.contentType} body=${annotation.attachment?.body}`) }, + onTestCaseArtifactRecord() { + events.push('[artifact]') + }, onTestCaseReady(testCase) { events.push(`[ready] ${testCase.name}`) }, @@ -86,6 +91,8 @@ describe('API', () => { } return annotation }) + // artifacts should be empty until next major so no handling needed + artifacts[testCase.name] = testCase.artifacts() }, }, ], @@ -116,6 +123,12 @@ describe('API', () => { ] `) + expect(artifacts).toMatchInlineSnapshot(` + { + "second": [], + "simple": [], + } + `) expect(annotations).toMatchInlineSnapshot(` { "second": [ diff --git a/test/cli/test/artifacts.test.ts b/test/cli/test/artifacts.test.ts new file mode 100644 index 000000000..d6ccb2802 --- /dev/null +++ b/test/cli/test/artifacts.test.ts @@ -0,0 +1,420 @@ +import type { TestAnnotation, TestArtifact } from 'vitest' +import { format } from 'node:util' +import { playwright } from '@vitest/browser-playwright' +import { describe, expect, test } from 'vitest' +import { runInlineTests } from '../../test-utils' + +const test3Content = /* ts */` +export async function externalArtifactRecord(recordArtifact, task) { + await recordArtifact(task, { type: 'external' }) +} +` + +const artifactsTest = /* ts */` +import { test, describe, recordArtifact } from 'vitest' +import { externalArtifactRecord } from './test-3.js' + +test('simple', async ({ task }) => { + await recordArtifact(task, { type: 'with-no-attachments' }) + await recordArtifact(task, { type: 'with-one-attachment', attachments: [{ path: './test-3.js' }] }) + await recordArtifact(task, { type: 'with-multiple-attachments', attachments: [{ path: './test-3.js' }, { path: './test-4.js' }] }) + await externalArtifactRecord(recordArtifact, task) + await recordArtifact(task, { type: 'with-base64', attachments: [{ body: 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' }]}) + await recordArtifact(task, { type: 'with-bytes', attachments: [{ body: new Uint8Array(Array.from({ length: 256 }).map((_, i) => i)) }] }) + await recordArtifact(task, { type: 'with-contentType', attachments: [{ body: '', contentType: 'text/plain' }] }) +}) + +describe('suite', () => { + test('second', async ({ task }) => { + await recordArtifact(task, { type: 'with-external-link', attachments: [{ path: 'https://absolute-path.com' }] }) + }) +}) +` + +describe('API', () => { + test.for([ + { name: 'forks', pool: 'forks' }, + { name: 'threads', pool: 'threads' }, + { + name: 'browser', + browser: { + enabled: true, + provider: playwright(), + headless: true, + instances: [ + { browser: 'chromium' as const }, + ], + }, + }, + ])('artifacts are exposed correctly in $name', async (options) => { + const events: string[] = [] + const annotations: Record> = {} + const artifacts: Record> = {} + + const { stderr } = await runInlineTests( + { + 'basic.test.ts': artifactsTest, + 'test-3.js': test3Content, + 'test-4.js': '', + }, + { + includeTaskLocation: true, + reporters: [ + 'default', + { + onTestCaseAnnotate() { + events.push('[annotate]') + }, + onTestCaseArtifactRecord(testCase, artifact) { + const path = artifact.attachments?.map( + attachment => attachment.path?.replace(testCase.project.config.root, '').replace(/\w+\.js$/, '.js'), + ) + events.push(`[artifact] ${testCase.name} ${artifact.type} path=${format(path)} contentType=${format(artifact.attachments?.map(attachment => attachment.contentType))} body=${format(artifact.attachments?.map(attachment => attachment.body))}`) + }, + onTestCaseReady(testCase) { + events.push(`[ready] ${testCase.name}`) + }, + onTestCaseResult(testCase) { + events.push(`[result] ${testCase.name}`) + annotations[testCase.name] = testCase.annotations() + artifacts[testCase.name] = testCase.artifacts().map((artifact) => { + if (Array.isArray(artifact.attachments)) { + for (const attachment of artifact.attachments) { + if (attachment?.path) { + attachment.path = attachment.path.replace( + testCase.project.config.root, + '', + ).replace(/\w+\.js$/, '.js') + } + } + } + if (artifact.location) { + artifact.location.file = artifact.location.file.replace( + testCase.project.config.root, + '', + ) + } + return artifact + }) + }, + }, + ], + }, + {}, + { + test: options, + }, + ) + + expect(stderr).toBe('') + expect(events).toMatchInlineSnapshot(` + [ + "[ready] simple", + "[artifact] simple with-no-attachments path=undefined contentType=undefined body=undefined", + "[artifact] simple with-one-attachment path=[ '/.vitest-attachments/.js' ] contentType=[ 'text/javascript' ] body=[ undefined ]", + "[artifact] simple with-multiple-attachments path=[ + '/.vitest-attachments/.js', + '/.vitest-attachments/.js' + ] contentType=[ 'text/javascript', 'text/javascript' ] body=[ undefined, undefined ]", + "[artifact] simple external path=undefined contentType=undefined body=undefined", + "[artifact] simple with-base64 path=[ undefined ] contentType=[ undefined ] body=[ 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' ]", + "[artifact] simple with-bytes path=[ undefined ] contentType=[ undefined ] body=[ + 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==' + ]", + "[artifact] simple with-contentType path=[ undefined ] contentType=[ 'text/plain' ] body=[ '' ]", + "[result] simple", + "[ready] second", + "[artifact] second with-external-link path=[ 'https://absolute-path.com' ] contentType=[ undefined ] body=[ undefined ]", + "[result] second", + ] + `) + + expect(artifacts).toMatchInlineSnapshot(` + { + "second": [ + { + "attachments": [ + { + "path": "https://absolute-path.com", + }, + ], + "location": { + "column": 11, + "file": "/basic.test.ts", + "line": 17, + }, + "type": "with-external-link", + }, + ], + "simple": [ + { + "location": { + "column": 9, + "file": "/basic.test.ts", + "line": 6, + }, + "type": "with-no-attachments", + }, + { + "attachments": [ + { + "contentType": "text/javascript", + "path": "/.vitest-attachments/.js", + }, + ], + "location": { + "column": 9, + "file": "/basic.test.ts", + "line": 7, + }, + "type": "with-one-attachment", + }, + { + "attachments": [ + { + "contentType": "text/javascript", + "path": "/.vitest-attachments/.js", + }, + { + "contentType": "text/javascript", + "path": "/.vitest-attachments/.js", + }, + ], + "location": { + "column": 9, + "file": "/basic.test.ts", + "line": 8, + }, + "type": "with-multiple-attachments", + }, + { + "location": { + "column": 9, + "file": "/basic.test.ts", + "line": 9, + }, + "type": "external", + }, + { + "attachments": [ + { + "body": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", + }, + ], + "location": { + "column": 9, + "file": "/basic.test.ts", + "line": 10, + }, + "type": "with-base64", + }, + { + "attachments": [ + { + "body": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==", + }, + ], + "location": { + "column": 9, + "file": "/basic.test.ts", + "line": 11, + }, + "type": "with-bytes", + }, + { + "attachments": [ + { + "body": "", + "contentType": "text/plain", + }, + ], + "location": { + "column": 9, + "file": "/basic.test.ts", + "line": 12, + }, + "type": "with-contentType", + }, + ], + } + `) + expect(annotations).toMatchInlineSnapshot(` + { + "second": [], + "simple": [], + } + `) + }) + + test('cannot record artifacts when the test finished running', async () => { + const { stderr } = await runInlineTests({ + 'basic.test.ts': ` + import { recordArtifact } from 'vitest' + test('finished early', ({ task }) => { + setTimeout(() => { + recordArtifact(task, { type: 'invalid-artifact' }) + }, 50) + }) + + test('long running test ', async () => { + await new Promise(r => setTimeout(() => r(), 100)) + }) + `, + }, { globals: true }) + expect(stderr).toContain('Cannot record a test artifact outside of the test run. The test "finished early" finished running with the "pass" state already.') + }) +}) + +describe('reporters', () => { + test('tap', async () => { + const { stdout } = await runInlineTests( + { + 'basic.test.ts': artifactsTest, + 'test-3.js': test3Content, + 'test-4.js': '', + }, + { reporters: ['tap'] }, + ) + + expect(stdout.replace(/[\d.]+ms/g, '