From 8e6fabfee89d1c217e36d1003c33dc13d4a73e16 Mon Sep 17 00:00:00 2001 From: Raul Macarie Date: Thu, 25 Jun 2026 17:40:55 +0200 Subject: [PATCH] feat: custom IO operations for `toMatchScreenshot` screenshots --- docs/config/browser/expect.md | 71 ++++++++++-- .../client/tester/expect/toMatchScreenshot.ts | 13 ++- .../node/commands/screenshotMatcher/index.ts | 75 ++++++------- .../node/commands/screenshotMatcher/types.ts | 5 +- .../node/commands/screenshotMatcher/utils.ts | 37 ++++++- packages/vitest/src/node/types/browser.ts | 38 +++++++ packages/vitest/src/public/node.ts | 1 + .../browser/specs/to-match-screenshot.test.ts | 103 +++++++++++++++++- 8 files changed, 272 insertions(+), 71 deletions(-) diff --git a/docs/config/browser/expect.md b/docs/config/browser/expect.md index 42580174c..1974e2035 100644 --- a/docs/config/browser/expect.md +++ b/docs/config/browser/expect.md @@ -34,7 +34,7 @@ export default defineConfig({ allowedMismatchedPixels: 100, }, resolveScreenshotPath: ({ arg, browserName, ext, testFileName }) => - `custom-screenshots/${testFileName}/${arg}-${browserName}${ext}`, + path.resolve('custom-screenshots', testFileName, `${arg}-${browserName}${ext}`), }, }, }, @@ -42,9 +42,7 @@ export default defineConfig({ }) ``` -[All options available in the `toMatchScreenshot` assertion](/api/browser/assertions#options) -can be configured here. Additionally, two path resolution functions are -available: `resolveScreenshotPath` and `resolveDiffPath`. +[All options available in the `toMatchScreenshot` assertion](/api/browser/assertions#options) can be configured here. Additionally, [`screenshotDirectory`][screenshotDirectory], [`resolveScreenshotPath`][resolveScreenshotPath], [`resolveDiffPath`][resolveDiffPath], and [`io`](#browser-expect-toMatchScreenshot-io) let you customize where and how screenshots are stored. ## browser.expect.toMatchScreenshot.screenshotDirectory @@ -53,7 +51,7 @@ available: `resolveScreenshotPath` and `resolveDiffPath`. The directory name used for storing reference screenshots. -This value is passed as `screenshotDirectory` to [`browser.expect.toMatchScreenshot.resolveScreenshotPath`](#browserexpecttomatchscreenshotresolvescreenshotpath) and [`browser.expect.toMatchScreenshot.resolveDiffPath`](#browserexpecttomatchscreenshotresolvediffpath), and used in the default path resolution of `resolveScreenshotPath`. +This value is passed as `screenshotDirectory` to [`resolveScreenshotPath`][resolveScreenshotPath] and [`resolveDiffPath`][resolveDiffPath], and used in the default path resolution of [`resolveScreenshotPath`][resolveScreenshotPath]. ## browser.expect.toMatchScreenshot.resolveScreenshotPath @@ -101,7 +99,7 @@ receives an object with the following properties: - `screenshotDirectory: string` - The value provided to [`browser.expect.toMatchScreenshot.screenshotDirectory`](#browserexpecttomatchscreenshotscreenshotdirectory), if none is provided, its default value (`__screenshots__`). + The value provided to [`screenshotDirectory`][screenshotDirectory], if none is provided, its default value (`__screenshots__`). - `root: string` @@ -133,7 +131,7 @@ For example, to group screenshots by browser: ```ts resolveScreenshotPath: ({ arg, browserName, ext, root, testFileName }) => - `${root}/screenshots/${browserName}/${testFileName}/${arg}${ext}` + path.resolve(root, 'screenshots', browserName, testFileName, `${arg}${ext}`) ``` ## browser.expect.toMatchScreenshot.resolveDiffPath @@ -141,17 +139,64 @@ resolveScreenshotPath: ({ arg, browserName, ext, root, testFileName }) => - **Type:** `(data: PathResolveData) => string` - **Default output:** ``path.resolve(root, attachmentsDir, testFileDirectory, testFileName, `${arg}-${browserName}-${platform}${ext}`)`` -A function to customize where diff images are stored when screenshot comparisons -fail. Receives the same data object as -[`resolveScreenshotPath`](#browser-expect-tomatchscreenshot-resolvescreenshotpath). +A function to customize where diff images are stored when screenshot comparisons fail. Receives the same data object as [`resolveScreenshotPath`][resolveScreenshotPath]. For example, to store diffs in a subdirectory of attachments: ```ts resolveDiffPath: ({ arg, attachmentsDir, browserName, ext, root, testFileName }) => - `${root}/${attachmentsDir}/screenshot-diffs/${testFileName}/${arg}-${browserName}${ext}` + path.resolve(root, attachmentsDir, 'screenshot-diffs', testFileName, `${arg}-${browserName}${ext}`) ``` +## browser.expect.toMatchScreenshot.io 5.0.0 {#browser-expect-toMatchScreenshot-io} + +- **Type:** `{ read: Read; write: Write }` +- **Default:** Node's `fs` module, reading/writing at the paths resolved by [`resolveScreenshotPath`][resolveScreenshotPath] and [`resolveDiffPath`][resolveDiffPath]. + +Overrides the filesystem access used to read and write screenshots, letting you redirect reference, actual, and diff images to a different storage backend (e.g. object storage or a remote service) instead of the local filesystem. + +### io.read + +- **Type:** `(data: ReadData) => Promise` + +Reads image data from `path`. Should return `null` if no data exists at `path` (e.g. no reference screenshot has been captured yet). The function receives an object with the following properties: + +- `path: string` + + The path resolved by [`resolveScreenshotPath`][resolveScreenshotPath]. + +- `project: TestProject` + + The [`TestProject`](/api/advanced/test-project) the test belongs to. + +### io.write + +- **Type:** `(data: WriteData) => Promise` + +Writes image `data` to `path`. The function receives an object with the following properties: + +- `path: string` + + The path resolved by [`resolveScreenshotPath`][resolveScreenshotPath] or [`resolveDiffPath`][resolveDiffPath]. + +- `data: TypedArray` + + The image data to write, as a `Buffer` or `Uint8Array`. + +- `kind: 'reference' | 'actual' | 'diff'` + + Indicates which type of image is being written, so implementations can apply different handling (e.g. retention policies) for reference, actual, and diff images. + +- `project: TestProject` + + The [`TestProject`](/api/advanced/test-project) the test belongs to. + +::: tip +`path` here is whatever [`resolveScreenshotPath`][resolveScreenshotPath] or [`resolveDiffPath`][resolveDiffPath] resolved to. It doesn't have to be a filesystem path. + +If you're writing to a non-filesystem backend, you can use those functions to return a key (e.g. an S3 object key) instead of an absolute path. +::: + ## browser.expect.toMatchScreenshot.comparators - **Type:** `Record` @@ -264,3 +309,7 @@ myCustomComparator: ( } ``` ::: + +[resolveDiffPath]: #browser-expect-toMatchScreenshot-resolveDiffPath +[resolveScreenshotPath]: #browser-expect-toMatchScreenshot-resolveScreenshotPath +[screenshotDirectory]: #browser-expect-toMatchScreenshot-screenshotDirectory diff --git a/packages/browser/src/client/tester/expect/toMatchScreenshot.ts b/packages/browser/src/client/tester/expect/toMatchScreenshot.ts index c714b9e7b..bc180f2a8 100644 --- a/packages/browser/src/client/tester/expect/toMatchScreenshot.ts +++ b/packages/browser/src/client/tester/expect/toMatchScreenshot.ts @@ -91,12 +91,13 @@ export default async function toMatchScreenshot( } if (attachments.length > 0) { - await recordArtifact(this.task, { - type: 'internal:toMatchScreenshot', - kind: 'visual-regression', - message: result.message, - attachments, - }) + // @todo understand how to handled this + // await recordArtifact(this.task, { + // type: 'internal:toMatchScreenshot', + // kind: 'visual-regression', + // message: result.message, + // attachments, + // }) } } diff --git a/packages/browser/src/node/commands/screenshotMatcher/index.ts b/packages/browser/src/node/commands/screenshotMatcher/index.ts index d83c8ed7f..83149c072 100644 --- a/packages/browser/src/node/commands/screenshotMatcher/index.ts +++ b/packages/browser/src/node/commands/screenshotMatcher/index.ts @@ -1,15 +1,12 @@ import type { SerializedLocator } from '@vitest/browser' import type { SnapshotUpdateState } from 'vitest' import type { ScreenshotMatcherOptions } from 'vitest/browser' -import type { BrowserCommand, BrowserCommandContext, TestProject } from 'vitest/node' +import type { BrowserCommand, BrowserCommandContext, TypedArray } from 'vitest/node' import type { ScreenshotMatcherArguments, ScreenshotMatcherOutput } from '../../../shared/screenshotMatcher/types' import type { AnyCodec } from './codecs' import type { AnyComparator } from './comparators' -import type { TypedArray } from './types' -import type { ResolvedOptions } from './utils' -import { mkdir, readFile, writeFile } from 'node:fs/promises' -import { basename, dirname } from 'pathe' -import { assertBrowserApiWrite, assertBrowserFileAccess } from '../../utils' +import type { IO, ResolvedOptions } from './utils' +import { basename } from 'pathe' import { asyncTimeout, resolveOptions, takeDecodedScreenshot, takeScreenshotBuffer } from './utils' /** Decoded image data with dimensions metadata. */ @@ -19,12 +16,12 @@ type DecodedImage = Awaited> interface ScreenshotData { image: DecodedImage path: string - buffer?: Buffer + buffer?: TypedArray } interface CapturedScreenshot { image: DecodedImage - buffer: Buffer + buffer: TypedArray } /** @@ -88,6 +85,7 @@ export const screenshotMatcher: BrowserCommand = asy const { codec, comparator, + io, paths, resolvedOptions: { comparatorName, comparatorOptions, screenshotOptions, timeout }, } = resolveOptions({ context, name, testName, options }) @@ -101,7 +99,7 @@ export const screenshotMatcher: BrowserCommand = asy target, } satisfies Parameters[0] - const referenceFile = await readFile(paths.reference).catch(() => null) + const referenceFile = await io.read({ path: paths.reference, project: context.project }) let reference: DecodedImage | null = null let initialScreenshot: CapturedScreenshot | null = null @@ -149,7 +147,7 @@ export const screenshotMatcher: BrowserCommand = asy comparatorOptions, }) - await performSideEffects(outcome, codec, context.project) + await performSideEffects(outcome, codec, io, context) return buildOutput(outcome, timeout) } @@ -176,7 +174,7 @@ async function determineOutcome( reference: DecodedImage | null retries: number screenshot: DecodedImage | null - screenshotBuffer?: Buffer + screenshotBuffer?: TypedArray updateSnapshot: SnapshotUpdateState }, ): Promise { @@ -277,33 +275,37 @@ async function determineOutcome( async function performSideEffects( outcome: MatchOutcome, codec: AnyCodec, - project: TestProject, + io: IO, + context: BrowserCommandContext, ): Promise { switch (outcome.type) { case 'missing-reference': case 'update-reference': { - await writeScreenshot( - outcome.reference.path, - await encodeScreenshot(outcome.reference, codec), - project, - ) + await io.write({ + path: outcome.reference.path, + data: await encodeScreenshot(outcome.reference, codec), + kind: 'reference', + project: context.project + }) break } case 'mismatch': { - await writeScreenshot( - outcome.actual.path, - await encodeScreenshot(outcome.actual, codec), - project, - ) + await io.write({ + path: outcome.actual.path, + data: await encodeScreenshot(outcome.actual, codec), + kind: 'actual', + project: context.project + }) if (outcome.diff) { - await writeScreenshot( - outcome.diff.path, - await codec.encode(outcome.diff.image, {}), - project, - ) + await io.write({ + path: outcome.diff.path, + data: await encodeScreenshot(outcome.diff, codec), + kind: 'diff', + project: context.project + }) } break @@ -421,7 +423,7 @@ interface StableScreenshotOptions { * Wraps {@linkcode getStableScreenshot} with an abort controller that triggers when the timeout expires. Returns `null` if the page never stabilizes. */ async function waitForStableScreenshot(options: StableScreenshotOptions, timeout: number, -): Promise<{ actual: DecodedImage; buffer: Buffer; retries: number } | null> { +): Promise<{ actual: DecodedImage; buffer: TypedArray; retries: number } | null> { const abortController = new AbortController() const stableScreenshot = getStableScreenshot( @@ -470,7 +472,7 @@ async function getStableScreenshot({ }: StableScreenshotOptions, signal: AbortSignal): Promise<{ retries: number actual: DecodedImage - buffer: Buffer + buffer: TypedArray }> { const screenshotArgument = { codec, @@ -537,7 +539,7 @@ async function takeScreenshotData({ screenshotOptions, target, }: { - buffer?: Buffer + buffer?: TypedArray codec: AnyCodec context: BrowserCommandContext element?: SerializedLocator @@ -558,16 +560,3 @@ async function takeScreenshotData({ image: await codec.decode(screenshot, {}), } } - -/** Writes encoded images to disk, creating parent directories as needed. */ -async function writeScreenshot(path: string, image: TypedArray, project: TestProject) { - try { - assertBrowserApiWrite(project, path) - assertBrowserFileAccess(project, path) - await mkdir(dirname(path), { recursive: true }) - await writeFile(path, image) - } - catch (cause) { - throw new Error('Couldn\'t write file to fs', { cause }) - } -} diff --git a/packages/browser/src/node/commands/screenshotMatcher/types.ts b/packages/browser/src/node/commands/screenshotMatcher/types.ts index ea48818d7..afebac250 100644 --- a/packages/browser/src/node/commands/screenshotMatcher/types.ts +++ b/packages/browser/src/node/commands/screenshotMatcher/types.ts @@ -3,12 +3,9 @@ import type { ScreenshotComparatorRegistry, ScreenshotMatcherOptions, } from '@vitest/browser/context' +import type { TypedArray } from 'vitest/node' interface BaseMetadata { height: number; width: number } -export type TypedArray - = | Buffer - | Uint8Array - | Uint8ClampedArray export type Promisable = T | Promise export interface Codec< diff --git a/packages/browser/src/node/commands/screenshotMatcher/utils.ts b/packages/browser/src/node/commands/screenshotMatcher/utils.ts index 2b60575c0..3ee1dd0a6 100644 --- a/packages/browser/src/node/commands/screenshotMatcher/utils.ts +++ b/packages/browser/src/node/commands/screenshotMatcher/utils.ts @@ -1,23 +1,23 @@ import type { SerializedLocator } from '@vitest/browser' - // Note: this augments `screenshotOptions` types import type {} from '@vitest/browser-playwright' import type { BrowserCommandContext, BrowserConfigOptions } from 'vitest/node' import type { ScreenshotMatcherOptions } from '../../../../context' import type { ScreenshotMatcherArguments } from '../../../shared/screenshotMatcher/types' import type { AnyCodec } from './codecs' +import { mkdir, readFile, writeFile } from 'node:fs/promises' import { platform } from 'node:os' import { deepMerge } from '@vitest/utils/helpers' import { basename, dirname, extname, join, relative, resolve } from 'pathe' import { getCodec } from './codecs' import { getComparator } from './comparators' +import { assertBrowserApiWrite, assertBrowserFileAccess } from '../../utils' + +type ToMatchScreenshotGlobals = NonNullable['toMatchScreenshot']> type GlobalOptions = Required['toMatchScreenshot'] - & NonNullable> - >, - 'comparators' | 'screenshotDirectory' + ToMatchScreenshotGlobals & NonNullable>, + 'comparators' | 'screenshotDirectory' | 'io' >> const defaultOptions = { @@ -67,6 +67,24 @@ const defaultOptions = { ), } satisfies GlobalOptions +export type IO = NonNullable + +const globalIO = { + read: async ({ path }) => readFile(path).catch(() => null), + write: async ({ path, data, project }) => { + try { + assertBrowserApiWrite(project, path) + assertBrowserFileAccess(project, path) + + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, data) + } + catch (cause) { + throw new Error('Couldn\'t write file to fs', { cause }) + } + }, +} satisfies IO + type SupportedCodecs = Parameters[0] const supportedExtensions = ['png'] satisfies SupportedCodecs[] @@ -74,6 +92,7 @@ const supportedExtensions = ['png'] satisfies SupportedCodecs[] export interface ResolvedOptions { codec: ReturnType comparator: ReturnType + io: IO resolvedOptions: GlobalOptions paths: { reference: string @@ -147,6 +166,12 @@ export function resolveOptions( return { codec: getCodec(extension), comparator: getComparator(resolvedOptions.comparatorName, context), + io: context.project.config.browser.expect?.toMatchScreenshot?.io + ? { + ...globalIO, + ...context.project.config.browser.expect.toMatchScreenshot.io, + } + : globalIO, resolvedOptions, paths: { reference: resolvedOptions.resolveScreenshotPath(resolvePathData), diff --git a/packages/vitest/src/node/types/browser.ts b/packages/vitest/src/node/types/browser.ts index c18edee66..1512a9ca0 100644 --- a/packages/vitest/src/node/types/browser.ts +++ b/packages/vitest/src/node/types/browser.ts @@ -575,6 +575,44 @@ export interface ToMatchScreenshotOptions { * @default path.resolve(root, attachmentsDir, testFileDirectory, testFileName, `${arg}-${browserName}-${platform}${ext}`) */ resolveDiffPath?: ToMatchScreenshotResolvePath + /** + * Overrides the default filesystem access used to read and write screenshots. + * + * By default, reference, actual, and diff images are read from and written to disk at the paths resolved by {@linkcode resolveScreenshotPath|browser.expect.toMatchScreenshot.resolveScreenshotPath} and {@linkcode resolveDiffPath|browser.expect.toMatchScreenshot.resolveDiffPath}. Providing `io` lets you redirect this to a different storage backend (e.g. object storage or a remote service) instead of the local filesystem. + * + * @default Node's `fs` module, reading/writing at the resolved paths. + */ + io?: { + /** + * Reads image data from `path`. + * + * @returns The image data, or `null` if no data exists at `path` (e.g. no reference screenshot has been captured yet). + */ + read: (data: { + /** The path resolved by {@linkcode resolveScreenshotPath|browser.expect.toMatchScreenshot.resolveScreenshotPath}. */ + path: string + /** The {@linkcode https://vitest.dev/api/advanced/test-project|TestProject} the test belongs to. */ + project: TestProject + }) => Promise + /** + * Writes image `data` to `path`. + */ + write: (data: { + /** The path resolved by {@linkcode resolveScreenshotPath|browser.expect.toMatchScreenshot.resolveScreenshotPath} or {@linkcode resolveDiffPath|browser.expect.toMatchScreenshot.resolveDiffPath}. */ + path: string + /** The image data to write, as a `Buffer` or `Uint8Array`. */ + data: TypedArray + /** Indicates which type of image is being written, so implementations can apply different handling (e.g. retention policies) for reference, actual, and diff images. */ + kind: 'reference' | 'actual' | 'diff' + /** The {@linkcode https://vitest.dev/api/advanced/test-project|TestProject} the test belongs to. */ + project: TestProject + }) => Promise + } } export interface ToMatchScreenshotComparators {} + +/** + * Binary image data accepted and returned by {@linkcode ToMatchScreenshotOptions.io|io}'s `read` and `write` functions. + */ +export type TypedArray = Buffer | Uint8Array diff --git a/packages/vitest/src/public/node.ts b/packages/vitest/src/public/node.ts index e3e894e07..4d5492923 100644 --- a/packages/vitest/src/public/node.ts +++ b/packages/vitest/src/public/node.ts @@ -122,6 +122,7 @@ export type { ResolvedBrowserOptions, ToMatchScreenshotComparators, ToMatchScreenshotOptions, + TypedArray, } from '../node/types/browser' export const createViteServer: typeof vite.createServer = vite.createServer export type { diff --git a/test/browser/specs/to-match-screenshot.test.ts b/test/browser/specs/to-match-screenshot.test.ts index 8843f1e87..07b01d44c 100644 --- a/test/browser/specs/to-match-screenshot.test.ts +++ b/test/browser/specs/to-match-screenshot.test.ts @@ -2,7 +2,7 @@ import type { Stats } from 'node:fs' import type { ViteUserConfig } from 'vitest/config' import type { TestFsStructure } from '../../test-utils' import { platform } from 'node:os' -import { describe, expect, test } from 'vitest' +import { describe, expect, test, vi } from 'vitest' import { runInlineTests } from '../../test-utils' import { extractToMatchScreenshotPaths } from '../fixtures/expect-dom/utils' import utilsContent from '../fixtures/expect-dom/utils?raw' @@ -12,6 +12,10 @@ const testFilename = 'basic.test.ts' const testName = 'screenshot-snapshot' const bgColor = '#fff' +const comparators = { + failing: () => ({ pass: false, diff: null, message: null }) +} + const testContent = /* ts */` import { page, server } from 'vitest/browser' import { describe, test } from 'vitest' @@ -155,6 +159,7 @@ describe('--watch', () => { expect: { toMatchScreenshot: { screenshotDirectory: customDir, + comparators, }, }, }, @@ -172,6 +177,102 @@ describe('--watch', () => { }, ) + test( + 'uses custom `io.read` and `io.write` instead of the filesystem', + async () => { + const store = new Map< + string, + { + data: Buffer | Uint8Array | null + path: string + kind: 'reference' | 'actual' | 'diff' + } + >() + const read = vi.fn(async ({ path }) => store.get(path)?.data ?? null) + const write = vi.fn(async ({ path, data, kind }) => { + store.set(path, { data: kind === 'reference' ? data : null, kind, path }) + }) + + const { fs, stderr, vitest } = await runBrowserTests( + { + [testFilename]: testContent, + 'utils.ts': utilsContent, + }, + { + browser: { + enabled: true, + screenshotFailures: false, + provider, + headless: true, + instances, + viewport: { + width: 400, + height: 200, + }, + expect: { + toMatchScreenshot: { + io: { + read, + write, + }, + comparators, + }, + }, + }, + update: 'new', + }, + ) + + const references = extractToMatchScreenshotPaths(stderr, testName) + + // reference saved in memory via `io.write` + expect(references).toHaveLength(instances.length) + expect(read).toHaveBeenCalledTimes(instances.length) + expect(write).toHaveBeenCalledTimes(instances.length) + + for (const referencePath of references) { + expect(store.get(referencePath)?.kind).toBe('reference') + expect(() => fs.statFile(referencePath)).toThrow() + } + + read.mockClear() + write.mockClear() + + fs.editFile(testFilename, content => content.replace(bgColor, '#0ff')) + + vitest.resetOutput() + await vitest.waitForStdout(`Test Files ${instances.length} failed`) + + for (const instance of instances) { + expect(vitest.stdout).toContain(`× |${instance.browser}| basic.test.ts > screenshot-snapshot`) + } + + expect(vitest.stdout).toContain('Screenshot does not match the stored reference.') + + expect(read).toHaveBeenCalledTimes(instances.length) + expect(write).toHaveBeenCalledTimes(2 * instances.length) + + // artifacts saved in memory via `io.write` + const files = Array.from(store.values()) + + expect(files).toHaveLength(instances.length * 3) + + const actuals = files.filter(({ kind }) => kind === 'actual') + + expect(actuals).toHaveLength(instances.length) + for (const file of actuals) { + expect(() => fs.statFile(file.path)).toThrow() + } + + const diffs = files.filter(({ kind }) => kind === 'diff') + + expect(diffs).toHaveLength(instances.length) + for (const file of diffs) { + expect(() => fs.statFile(file.path)).toThrow() + } + }, + ) + describe('--update', () => { test( 'creates snapshot and does NOT update it if reference matches', -- 2.51.2