diff --git a/docs/guide/reporters.md b/docs/guide/reporters.md index b773f066a..933602a80 100644 --- a/docs/guide/reporters.md +++ b/docs/guide/reporters.md @@ -445,6 +445,16 @@ export default defineConfig({ ``` ::: +### Github Actions Reporter 1.3.0+ + +Output [workflow commands](https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message) +to provide annotations for test failures. This reporter is automatically enabled when `process.env.GITHUB_ACTIONS === 'true'`, thus it doesn't require any configuration. + +Github Actions +Github Actions + + + ## Custom Reporters You can use third-party custom reporters installed from NPM by specifying their package name in the reporters' option: diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index a5b5aba39..363f81b80 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -422,6 +422,10 @@ export function resolveConfig( if (!resolved.reporters.length) resolved.reporters.push(['default', {}]) + // automatically enable github-actions reporter + if (process.env.GITHUB_ACTIONS === 'true' && !resolved.reporters.some(v => Array.isArray(v) && v[0] === 'github-actions')) + resolved.reporters.push(['github-actions', {}]) + if (resolved.changed) resolved.passWithNoTests ??= true diff --git a/packages/vitest/src/node/error.ts b/packages/vitest/src/node/error.ts index 4daa6ed96..d4e94fb70 100644 --- a/packages/vitest/src/node/error.ts +++ b/packages/vitest/src/node/error.ts @@ -22,7 +22,11 @@ interface PrintErrorOptions { showCodeFrame?: boolean } -export async function printError(error: unknown, project: WorkspaceProject | undefined, options: PrintErrorOptions) { +interface PrintErrorResult { + nearest?: ParsedStack +} + +export async function printError(error: unknown, project: WorkspaceProject | undefined, options: PrintErrorOptions): Promise { const { showCodeFrame = true, fullStack = false, type } = options const logger = options.logger let e = error as ErrorWithDiff @@ -43,8 +47,10 @@ export async function printError(error: unknown, project: WorkspaceProject | und } // Error may have occured even before the configuration was resolved - if (!project) - return printErrorMessage(e, logger) + if (!project) { + printErrorMessage(e, logger) + return + } const parserOptions: StackTraceParserOptions = { // only browser stack traces require remapping @@ -85,7 +91,7 @@ export async function printError(error: unknown, project: WorkspaceProject | und logger.error(c.yellow(e.frame)) } else { - printStack(project, stacks, nearest, errorProperties, (s) => { + printStack(logger, project, stacks, nearest, errorProperties, (s) => { if (showCodeFrame && s === nearest && nearest) { const sourceCode = readFileSync(nearest.file, 'utf-8') logger.error(generateCodeFrame(sourceCode.length > 100_000 ? sourceCode : logger.highlight(nearest.file, sourceCode), 4, s)) @@ -116,6 +122,8 @@ export async function printError(error: unknown, project: WorkspaceProject | und } handleImportOutsideModuleError(e.stack || e.stackStr || '', logger) + + return { nearest } } function printErrorType(type: string, ctx: Vitest) { @@ -229,14 +237,13 @@ function printErrorMessage(error: ErrorWithDiff, logger: Logger) { } function printStack( + logger: Logger, project: WorkspaceProject, stack: ParsedStack[], highlight: ParsedStack | undefined, errorProperties: Record, onStack?: ((stack: ParsedStack) => void), ) { - const logger = project.ctx.logger - for (const frame of stack) { const color = frame === highlight ? c.cyan : c.gray const path = relative(project.config.root, frame.file) diff --git a/packages/vitest/src/node/logger.ts b/packages/vitest/src/node/logger.ts index eb6586940..ce288f5f3 100644 --- a/packages/vitest/src/node/logger.ts +++ b/packages/vitest/src/node/logger.ts @@ -82,10 +82,10 @@ export class Logger { this.console.log(`${CURSOR_TO_START}${ERASE_DOWN}${log}`) } - printError(err: unknown, options: ErrorOptions = {}) { + async printError(err: unknown, options: ErrorOptions = {}) { const { fullStack = false, type } = options const project = options.project ?? this.ctx.getCoreWorkspaceProject() ?? this.ctx.projects[0] - return printError(err, project, { + await printError(err, project, { fullStack, type, showCodeFrame: true, diff --git a/packages/vitest/src/node/reporters/github-actions.ts b/packages/vitest/src/node/reporters/github-actions.ts new file mode 100644 index 000000000..0ec2f7de4 --- /dev/null +++ b/packages/vitest/src/node/reporters/github-actions.ts @@ -0,0 +1,116 @@ +import { Console } from 'node:console' +import { Writable } from 'node:stream' +import { getTasks } from '@vitest/runner/utils' +import stripAnsi from 'strip-ansi' +import type { File, Reporter, Vitest } from '../../types' +import { getFullName } from '../../utils' +import { printError } from '../error' +import { Logger } from '../logger' +import type { WorkspaceProject } from '../workspace' + +export class GithubActionsReporter implements Reporter { + ctx: Vitest = undefined! + + onInit(ctx: Vitest) { + this.ctx = ctx + } + + async onFinished(files: File[] = [], errors: unknown[] = []) { + // collect all errors and associate them with projects + const projectErrors = new Array<{ project: WorkspaceProject; title: string; error: unknown }>() + for (const error of errors) { + projectErrors.push({ + project: this.ctx.getCoreWorkspaceProject(), + title: 'Unhandled error', + error, + }) + } + for (const file of files) { + const tasks = getTasks(file) + const project = this.ctx.getProjectByTaskId(file.id) + for (const task of tasks) { + if (task.result?.state !== 'fail') + continue + + const title = getFullName(task, ' > ') + for (const error of task.result?.errors ?? []) { + projectErrors.push({ + project, + title, + error, + }) + } + } + } + + // format errors via `printError` + for (const { project, title, error } of projectErrors) { + const result = await printErrorWrapper(error, this.ctx, project) + const stack = result?.nearest + if (!stack) + continue + const formatted = formatMessage({ + command: 'error', + properties: { + file: stack.file, + title, + line: String(stack.line), + column: String(stack.column), + }, + message: stripAnsi(result.output), + }) + this.ctx.logger.log(`\n${formatted}`) + } + } +} + +// use Logger with custom Console to extract messgage from `processError` util +// TODO: maybe refactor `processError` to require single function `(message: string) => void` instead of full Logger? +async function printErrorWrapper(error: unknown, ctx: Vitest, project: WorkspaceProject) { + let output = '' + const writable = new Writable({ + write(chunk, _encoding, callback) { + output += String(chunk) + callback() + }, + }) + const result = await printError(error, project, { + showCodeFrame: false, + logger: new Logger(ctx, new Console(writable, writable)), + }) + return { nearest: result?.nearest, output } +} + +// workflow command formatting based on +// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message +// https://github.com/actions/toolkit/blob/f1d9b4b985e6f0f728b4b766db73498403fd5ca3/packages/core/src/command.ts#L80-L85 +function formatMessage({ + command, + properties, + message, +}: { + command: string + properties: Record + message: string +}): string { + let result = `::${command}` + Object.entries(properties).forEach(([k, v], i) => { + result += i === 0 ? ' ' : ',' + result += `${k}=${escapeProperty(v)}` + }) + result += `::${escapeData(message)}` + return result +} + +function escapeData(s: string): string { + return s.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A') +} + +function escapeProperty(s: string): string { + return s + .replace(/%/g, '%25') + .replace(/\r/g, '%0D') + .replace(/\n/g, '%0A') + .replace(/:/g, '%3A') + .replace(/,/g, '%2C') +} diff --git a/packages/vitest/src/node/reporters/index.ts b/packages/vitest/src/node/reporters/index.ts index d270e4e54..ffd2dde08 100644 --- a/packages/vitest/src/node/reporters/index.ts +++ b/packages/vitest/src/node/reporters/index.ts @@ -8,6 +8,7 @@ import { TapReporter } from './tap' import { type JUnitOptions, JUnitReporter } from './junit' import { TapFlatReporter } from './tap-flat' import { HangingProcessReporter } from './hanging-process' +import { GithubActionsReporter } from './github-actions' import type { BaseReporter } from './base' export { @@ -20,6 +21,7 @@ export { JUnitReporter, TapFlatReporter, HangingProcessReporter, + GithubActionsReporter, } export type { BaseReporter, Reporter } @@ -35,6 +37,7 @@ export const ReportersMap = { 'tap-flat': TapFlatReporter, 'junit': JUnitReporter, 'hanging-process': HangingProcessReporter, + 'github-actions': GithubActionsReporter, } export type BuiltinReporters = keyof typeof ReportersMap diff --git a/test/reporters/package.json b/test/reporters/package.json index 1e7e05161..945177646 100644 --- a/test/reporters/package.json +++ b/test/reporters/package.json @@ -3,7 +3,7 @@ "type": "module", "private": true, "scripts": { - "test": "NO_COLOR=1 vitest run" + "test": "NO_COLOR=1 GITHUB_ACTIONS=false vitest run" }, "devDependencies": { "flatted": "^3.2.9", diff --git a/test/reporters/tests/github-actions.test.ts b/test/reporters/tests/github-actions.test.ts new file mode 100644 index 000000000..bb80b4508 --- /dev/null +++ b/test/reporters/tests/github-actions.test.ts @@ -0,0 +1,18 @@ +import { resolve } from 'pathe' +import { expect, test } from 'vitest' +import { runVitest } from '../../test-utils' +import { GithubActionsReporter } from '../../../packages/vitest/src/node/reporters' + +test(GithubActionsReporter, async () => { + let { stdout, stderr } = await runVitest( + { reporters: new GithubActionsReporter(), root: './fixtures' }, + ['some-failing.test.ts'], + ) + stdout = stdout.replace(resolve(__dirname, '..').replace(/:/g, '%3A'), '__TEST_DIR__') + expect(stdout).toMatchInlineSnapshot(` + " + ::error file=__TEST_DIR__/fixtures/some-failing.test.ts,title=some-failing.test.ts > 3 + 3 = 7,line=8,column=17::AssertionError: expected 6 to be 7 // Object.is equality%0A%0A- Expected%0A+ Received%0A%0A- 7%0A+ 6%0A%0A ❯ some-failing.test.ts:8:17%0A%0A + " + `) + expect(stderr).toBe('') +}) diff --git a/test/ui/package.json b/test/ui/package.json index d1343c867..9b59a60c3 100644 --- a/test/ui/package.json +++ b/test/ui/package.json @@ -3,7 +3,7 @@ "type": "module", "private": true, "scripts": { - "test-e2e": "playwright test", + "test-e2e": "GITHUB_ACTIONS=false playwright test", "test-fixtures": "vitest" }, "devDependencies": {