diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9ce85c45..741aa592c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,6 +109,35 @@ jobs: - name: Test UI run: pnpm run ui:test + test-ui-e2e: + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + fail-fast: false + + runs-on: ${{ matrix.os }} + + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup-and-cache + with: + node-version: 20 + + - name: Install + run: pnpm i + + - name: Install Playwright Dependencies + run: pnpx playwright install chromium + + - name: Build + run: pnpm run build + + - name: Test + run: pnpm -C test/ui test-e2e + test-browser: runs-on: ubuntu-latest strategy: diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9a576b112..af5967f0d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1906,12 +1906,12 @@ importers: test/ui: devDependencies: + '@playwright/test': + specifier: ^1.39.0 + version: 1.39.0 execa: specifier: ^6.1.0 version: 6.1.0 - playwright-chromium: - specifier: ^1.39.0 - version: 1.39.0 vitest: specifier: workspace:* version: link:../../packages/vitest @@ -21404,15 +21404,6 @@ packages: mlly: 1.4.2 pathe: 1.1.1 - /playwright-chromium@1.39.0: - resolution: {integrity: sha512-0WVmvn9ppPbcyb2PQherIpzsvJlyjqziCZiAiexTEYSz8k6/+/3wljmFaMRMP1lcv2xKyHDn9yWd/lHb7IzYyA==} - engines: {node: '>=16'} - hasBin: true - requiresBuild: true - dependencies: - playwright-core: 1.39.0 - dev: true - /playwright-core@1.39.0: resolution: {integrity: sha512-+k4pdZgs1qiM+OUkSjx96YiKsXsmb59evFoqv8SKO067qBA+Z2s/dCzJij/ZhdQcs2zlTAgRKfeiiLm8PQ2qvw==} engines: {node: '>=16'} diff --git a/test/ui/.gitignore b/test/ui/.gitignore index ad7fcab32..bcb45cf2f 100644 --- a/test/ui/.gitignore +++ b/test/ui/.gitignore @@ -1 +1,2 @@ -fixtures/html +html +test-results diff --git a/test/ui/fixtures/vitest.config.ts b/test/ui/fixtures/vitest.config.ts deleted file mode 100644 index 1b3a79748..000000000 --- a/test/ui/fixtures/vitest.config.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ -}) diff --git a/test/ui/globalSetup.ts b/test/ui/globalSetup.ts deleted file mode 100644 index 1e820e588..000000000 --- a/test/ui/globalSetup.ts +++ /dev/null @@ -1,21 +0,0 @@ -import os from 'node:os' -import path from 'node:path' -import { existsSync, promises as fsp } from 'node:fs' -import type { BrowserServer } from 'playwright-chromium' -import { chromium } from 'playwright-chromium' - -const DIR = path.join(os.tmpdir(), 'vitest_playwright_global_setup') - -let browserServer: BrowserServer | undefined - -export async function setup(): Promise { - browserServer = await chromium.launchServer() - - if (!existsSync(DIR)) - await fsp.mkdir(DIR, { recursive: true }) - await fsp.writeFile(path.join(DIR, 'wsEndpoint'), browserServer.wsEndpoint()) -} - -export async function teardown(): Promise { - await browserServer?.close() -} diff --git a/test/ui/package.json b/test/ui/package.json index ee433096c..f682c941b 100644 --- a/test/ui/package.json +++ b/test/ui/package.json @@ -3,11 +3,12 @@ "type": "module", "private": true, "scripts": { - "test": "vitest run" + "test-e2e": "playwright test", + "test-fixtures": "vitest" }, "devDependencies": { + "@playwright/test": "^1.39.0", "execa": "^6.1.0", - "playwright-chromium": "^1.39.0", "vitest": "workspace:*" } } diff --git a/test/ui/playwright.config.ts b/test/ui/playwright.config.ts new file mode 100644 index 000000000..ee92ab485 --- /dev/null +++ b/test/ui/playwright.config.ts @@ -0,0 +1,11 @@ +import { defineConfig, devices } from '@playwright/test' + +export default defineConfig({ + testDir: './test', + projects: [ + { + name: 'chromium', + use: devices['Desktop Chrome'], + }, + ], +}) diff --git a/test/ui/setup.ts b/test/ui/setup.ts deleted file mode 100644 index 62d20c407..000000000 --- a/test/ui/setup.ts +++ /dev/null @@ -1,163 +0,0 @@ -/* eslint-disable import/no-mutable-exports */ -import path from 'node:path' -import os from 'node:os' -import { readFileSync } from 'node:fs' -import { chromium } from 'playwright-chromium' -import type { Browser, Page } from 'playwright-chromium' -import { expect } from 'vitest' -import type { ExecaChildProcess } from 'execa' -import { execaCommand } from 'execa' - -export let page!: Page -export let browser!: Browser -export const browserErrors: Error[] = [] -export const ports = { - ui: 9000, - report: 9001, -} - -const DIR = path.join(os.tmpdir(), 'vitest_playwright_global_setup') -export const isWindows = process.platform === 'win32' - -export function timeout(time: number) { - return new Promise((resolve) => { - setTimeout(() => { - resolve(null) - }, time) - }) -} - -export async function withRetry( - func: () => Promise, -): Promise { - const maxTries = 300 - for (let tries = 0; tries < maxTries; tries++) { - try { - await func() - return - } - catch {} - await timeout(50) - } - await func() -} - -export async function withLoadUrl(url: string): Promise { - return withRetry(async () => { - const res = await fetch(url) - if (!res.ok) - throw new Error('url not loaded') - }) -} - -export async function killProcess( - serverProcess: ExecaChildProcess, -): Promise { - if (isWindows) { - try { - const { execaCommandSync } = await import('execa') - execaCommandSync(`taskkill /pid ${serverProcess.pid} /T /F`) - } - catch (e) { - console.error('failed to taskkill:', e) - } - } - else { - serverProcess.kill('SIGTERM', { forceKillAfterTimeout: 2000 }) - } -} - -export async function startChromium() { - const wsEndpoint = readFileSync(path.join(DIR, 'wsEndpoint'), 'utf-8') - if (!wsEndpoint) - throw new Error('wsEndpoint not found') - - browser = await chromium.connect(wsEndpoint) - page = await browser.newPage() - - try { - page.on('pageerror', (error) => { - browserErrors.push(error) - }) - } - catch (e) { - await page.close() - throw e - } - - return async () => { - await page?.close() - if (browser) - await browser.close() - } -} - -export async function startServerCommand(command: string, url: string) { - let error: any - const exitChromium = await startChromium() - const subProcess = execaCommand(command, { - env: { - ...process.env, - CI: 'true', - NO_COLOR: 'true', - }, - stdio: 'pipe', - }) - - subProcess.catch((e) => { - error = e - }) - - const killSubProcess = () => killProcess(subProcess) - - subProcess.stdout?.on('data', (d) => { - // eslint-disable-next-line no-console - console.log(d.toString()) - }) - - expect(error).not.toBeTruthy() - - async function exit() { - try { - await killSubProcess() - await exitChromium() - } - catch (e) { - console.error( - `error while killing process ${command}:`, - e, - ) - } - } - - try { - await withLoadUrl(url) - await page.goto(url) - } - catch (e) { - await exit() - throw e - } - - return exit -} - -/** - * Poll a getter until the value it returns includes the expected value. - */ -export async function untilUpdated( - poll: () => string | Promise | null, - expected: string, -): Promise { - const maxTries = process.env.CI ? 200 : 50 - for (let tries = 0; tries < maxTries; tries++) { - const actual = (await poll()) ?? '' - if (actual.includes(expected) || tries === maxTries - 1) { - expect(actual).toMatch(expected) - break - } - else { - await timeout(50) - } - } -} diff --git a/test/ui/shim.d.ts b/test/ui/shim.d.ts deleted file mode 100644 index fbc5dbe18..000000000 --- a/test/ui/shim.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -declare module 'kill-port' { - const kill: (port: number) => Promise - export default kill -} diff --git a/test/ui/test/html-report.spec.ts b/test/ui/test/html-report.spec.ts index 162dd8850..d69579935 100644 --- a/test/ui/test/html-report.spec.ts +++ b/test/ui/test/html-report.spec.ts @@ -1,54 +1,52 @@ -import { resolve } from 'node:path' -import { beforeAll, describe, expect, it } from 'vitest' -import { browserErrors, isWindows, page, ports, startServerCommand, untilUpdated } from '../setup' +import { expect, test } from '@playwright/test' +import type { PreviewServer } from 'vite' +import { preview } from 'vite' +import { startVitest } from 'vitest/node' -import { runVitest } from '../../test-utils' +const port = 9001 +const pageUrl = `http://localhost:${port}/` -const root = resolve(__dirname, '../fixtures') -const port = ports.report +test.describe('html report', () => { + let previewServer: PreviewServer -// TODO: fix flakyness on windows -describe.skipIf(isWindows)('html report', () => { - beforeAll(async () => { - await runVitest({ root, reporters: 'html', outputFile: 'html/index.html' }) + test.beforeAll(async () => { + // generate vitest html report + await startVitest('test', [], { run: true, reporters: 'html' }) - const exit = await startServerCommand( - `pnpm exec vite preview --outDir fixtures/html --strict-port --port ${port}`, - `http://localhost:${port}/`, - ) - - return exit + // run vite preview server + previewServer = await preview({ build: { outDir: 'html' }, preview: { port, strictPort: true } }) }) - it('dashboard', async () => { - await untilUpdated(() => page.textContent('[aria-labelledby]'), '1 Pass 0 Fail 1 Total ') + test.afterAll(async () => { + await new Promise((resolve, reject) => { + previewServer.httpServer.close((err) => { + err ? reject(err) : resolve() + }) + }) }) - describe('file detail', async () => { - beforeAll(async () => { - await page.click('.details-panel span') - }) + test('basic', async ({ page }) => { + const pageErrors: unknown[] = [] + page.on('pageerror', error => pageErrors.push(error)) - it('report', async () => { - await page.click('[data-testid=btn-report]') - await untilUpdated(() => page.textContent('[data-testid=report]'), 'All tests passed in this file') - await untilUpdated(() => page.textContent('[data-testid=filenames]'), 'sample.test.ts') - }) + await page.goto(pageUrl) - it('graph', async () => { - await page.click('[data-testid=btn-graph]') - expect(page.url()).toMatch('graph') - await untilUpdated(() => page.textContent('[data-testid=graph] text'), 'sample.test.ts') - }) + // dashbaord + await expect(page.locator('[aria-labelledby=tests]')).toContainText('1 Pass 0 Fail 1 Total') - it('console', async () => { - await page.click('[data-testid=btn-console]') - expect(page.url()).toMatch('console') - await untilUpdated(() => page.textContent('[data-testid=console] pre'), 'log test') - }) - }) + // report + await page.getByText('sample.test.ts').click() + await page.getByText('All tests passed in this file').click() + await expect(page.getByTestId('filenames')).toContainText('sample.test.ts') + + // graph tab + await page.getByTestId('btn-graph').click() + await expect(page.locator('[data-testid=graph] text')).toContainText('sample.test.ts') + + // console tab + await page.getByTestId('btn-console').click() + await expect(page.getByTestId('console')).toContainText('log test') - it('no error happen', () => { - expect(browserErrors.length).toEqual(0) + expect(pageErrors).toEqual([]) }) }) diff --git a/test/ui/test/ui.spec.ts b/test/ui/test/ui.spec.ts index 1995877d3..7b5cb8bef 100644 --- a/test/ui/test/ui.spec.ts +++ b/test/ui/test/ui.spec.ts @@ -1,48 +1,43 @@ -import { beforeAll, describe, expect, it } from 'vitest' -import { browserErrors, isWindows, page, ports, startServerCommand, untilUpdated } from '../setup' +import { expect, test } from '@playwright/test' +import { type Vitest, startVitest } from 'vitest/node' -const port = ports.ui +const port = 9000 +const pageUrl = `http://localhost:${port}/__vitest__/` -// TODO: fix flakyness on windows -describe.skipIf(isWindows)('ui', () => { - beforeAll(async () => { - const exit = await startServerCommand( - `pnpm exec vitest --root ./fixtures --ui --open false --api.port ${port} --watch --allowOnly`, - `http://localhost:${port}/__vitest__/`, - ) +test.describe('ui', () => { + let vitest: Vitest | undefined - return exit + test.beforeAll(async () => { + vitest = await startVitest('test', [], { watch: true, ui: true, open: false, api: { port } }) + expect(vitest).toBeDefined() }) - it('dashboard', async () => { - await untilUpdated(() => page.textContent('[aria-labelledby]'), '1 Pass 0 Fail 1 Total ') + test.afterAll(async () => { + await vitest?.close() }) - describe('file detail', async () => { - beforeAll(async () => { - await page.click('.details-panel span') - }) - - it('report', async () => { - await page.click('[data-testid=btn-report]') - await untilUpdated(() => page.textContent('[data-testid=report]'), 'All tests passed in this file') - await untilUpdated(() => page.textContent('[data-testid=filenames]'), 'sample.test.ts') - }) - - it('graph', async () => { - await page.click('[data-testid=btn-graph]') - expect(page.url()).toMatch('graph') - await untilUpdated(() => page.textContent('[data-testid=graph] text'), 'sample.test.ts') - }) - - it('console', async () => { - await page.click('[data-testid=btn-console]') - expect(page.url()).toMatch('console') - await untilUpdated(() => page.textContent('[data-testid=console] pre'), 'log test') - }) - }) + test('basic', async ({ page }) => { + const pageErrors: unknown[] = [] + page.on('pageerror', error => pageErrors.push(error)) + + await page.goto(pageUrl) + + // dashbaord + await expect(page.locator('[aria-labelledby=tests]')).toContainText('1 Pass 0 Fail 1 Total') + + // report + await page.getByText('sample.test.ts').click() + await page.getByText('All tests passed in this file').click() + await expect(page.getByTestId('filenames')).toContainText('sample.test.ts') + + // graph tab + await page.getByTestId('btn-graph').click() + await expect(page.locator('[data-testid=graph] text')).toContainText('sample.test.ts') + + // console tab + await page.getByTestId('btn-console').click() + await expect(page.getByTestId('console')).toContainText('log test') - it('no error happen', () => { - expect(browserErrors.length).toEqual(0) + expect(pageErrors).toEqual([]) }) }) diff --git a/test/ui/vitest.config.ts b/test/ui/vitest.config.ts index 8059722bb..bdd49afa0 100644 --- a/test/ui/vitest.config.ts +++ b/test/ui/vitest.config.ts @@ -2,10 +2,6 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ test: { - setupFiles: ['./setup.ts'], - globalSetup: ['./globalSetup.ts'], - exclude: ['node_modules', 'fixtures', 'dist'], - hookTimeout: 60_000, - testTimeout: 60_000, + dir: './fixtures', }, })