diff --git a/.github/workflows/build-status.yml b/.github/workflows/build-status.yml index ee263f6..af76bc1 100644 --- a/.github/workflows/build-status.yml +++ b/.github/workflows/build-status.yml @@ -93,7 +93,9 @@ jobs: run: pnpm typecheck bench: name: 'Bench' - if: github.event_name == 'pull_request' && github.base_ref == 'main' + # Run on PRs targeting main (to compare against the main baseline) and on pushes to + # main (to refresh that baseline). + if: (github.event_name == 'pull_request' && github.base_ref == 'main') || (github.event_name == 'push' && github.ref == 'refs/heads/main') needs: warmup_pnpm_cache runs-on: ubuntu-latest steps: @@ -112,29 +114,18 @@ jobs: # Build the current package: benchmarks run against the build (lib/), not the sources. - name: Build current package run: pnpm build - # Restore the bundle built out of main so that we can compare current against it. - - name: Restore main bundle from cache - id: main-cache + # Restore the benchmark results computed on main so that vitest can compare the + # current run against them. No previous bundle needs to be installed anymore. + - name: Restore results from last benchmark execution on main uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - path: package.tgz + path: benchmark.json # Non-matching key so restore-keys picks the most recent main cache - key: bundle-main-will-not-match - restore-keys: bundle-main- - # Install the main version under the `pure-rand-main` alias. When no main bundle is - # cached yet (e.g. very first runs) we fall back to the latest published version. - - name: Install pure-rand from main - run: | - if [ -f package.tgz ]; then - echo "Comparing against the cached main bundle" - pnpm add "pure-rand-main@file:./package.tgz" --ignore-scripts - else - echo "::warning::No cached main bundle found, comparing against the latest published version" - pnpm add "pure-rand-main@npm:pure-rand@latest" --ignore-scripts - fi + key: bench-main-will-not-match + restore-keys: bench-main- - name: Bench tests id: bench - run: pnpm bench | tee bench-output.txt + run: pnpm bench:ci | tee bench-output.txt - name: Upload bench results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -142,6 +133,18 @@ jobs: path: bench-output.txt if-no-files-found: error retention-days: 1 + # On main, refresh the cached benchmark baseline that PRs compare against. + - name: Prune old benchmark execution from caches + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + GH_TOKEN: ${{ github.token }} + run: gh cache list --key bench-main- --json key -q '.[].key' | xargs -rn1 gh cache delete || true + - name: Cache last benchmark execution for main + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: benchmark.json + key: bench-main-${{ github.sha }} comment_bench: name: 'Comment bench results' needs: bench diff --git a/.gitignore b/.gitignore index d6a0e5d..996b74c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ test-bundle/*.mjs out.txt v8.log v8.out +benchmark.json diff --git a/package.json b/package.json index 378c33c..0be94ae 100644 --- a/package.json +++ b/package.json @@ -82,7 +82,9 @@ "test-bundle": "echo \"node: $(${NODE_BIN:-node} --version)\" && rm -rf test-bundle/*.mjs && for f in test-bundle/*.cjs; do if [ -f \"$f\" ]; then echo \"Creating ${f%.cjs}.mjs\" && mjs=\"${f%.cjs}.mjs\" && sed -E \"s/^const (\\{[^}]*\\}) = require\\(([^)]+)\\);$/import \\1 from \\2;/; s/^const ([a-zA-Z_][a-zA-Z0-9_]*) = require\\(([^)]+)\\);$/import \\1 from \\2;/; s/^'use strict';$//\" \"$f\" > \"$mjs\"; fi; done && for f in test-bundle/*.cjs test-bundle/*.mjs; do if [ -f \"$f\" ]; then echo \"Running ${f##*/}\" && ${NODE_BIN:-node} \"$f\" || exit 1; fi; done", "test-legacy-bundle": "nvs add 12.17.0 && NODE_BIN=$(nvs which 12.17.0) pnpm test-bundle", "bench": "vitest bench", - "bench:setup": "pnpm add pure-rand-main@https://pkg.pr.new/dubzzz/pure-rand@main --ignore-scripts" + "bench:ci": "vitest bench --outputJson benchmark.json --compare benchmark.json", + "bench:snap": "vitest bench --outputJson benchmark.json", + "bench:compare": "vitest bench --compare benchmark.json" }, "repository": { "type": "git", diff --git a/src/__bench__/Imports.ts b/src/__bench__/Imports.ts index 168ded5..64f5571 100644 --- a/src/__bench__/Imports.ts +++ b/src/__bench__/Imports.ts @@ -1,12 +1,8 @@ -// Helper exposing two flavours of pure-rand so that benchmarks can compare the -// performance of the change being worked on against the latest `main`: -// - `current`: the freshly built package, taken from the `lib/` directory (the -// build, NOT the sources). Build it first with `pnpm build`. -// - `main`: the version of pure-rand currently on `main`, installed under the -// `pure-rand-main` alias (run `pnpm bench:setup` locally to install it). +// Expose a freshly built version of the package so that benchmarks run build NOT sources. +// Build it first with `pnpm build`. // -// Types are always taken from the sources so that typechecking keeps working -// even when neither the build nor the `main` version are available locally. +// Types are always taken from the sources so that typechecking keeps working even +// when the build is not available locally. import type { congruential32 } from '../generator/congruential32'; import type { mersenne } from '../generator/mersenne'; @@ -35,24 +31,6 @@ import { uniformBigInt as uniformBigIntCurrent } from '../../lib/esm/distributio import { uniformFloat32 as uniformFloat32Current } from '../../lib/esm/distribution/uniformFloat32.js'; // @ts-ignore - Only available once the package has been built (`pnpm build`) import { uniformFloat64 as uniformFloat64Current } from '../../lib/esm/distribution/uniformFloat64.js'; - -// Version currently on `main`, installed under the `pure-rand-main` alias. -// @ts-ignore - Only available once `pnpm bench:setup` has been run -import { congruential32 as congruential32Main } from 'pure-rand-main/generator/congruential32'; -// @ts-ignore - Only available once `pnpm bench:setup` has been run -import { mersenne as mersenneMain } from 'pure-rand-main/generator/mersenne'; -// @ts-ignore - Only available once `pnpm bench:setup` has been run -import { xoroshiro128plus as xoroshiro128plusMain } from 'pure-rand-main/generator/xoroshiro128plus'; -// @ts-ignore - Only available once `pnpm bench:setup` has been run -import { xorshift128plus as xorshift128plusMain } from 'pure-rand-main/generator/xorshift128plus'; -// @ts-ignore - Only available once `pnpm bench:setup` has been run -import { uniformInt as uniformIntMain } from 'pure-rand-main/distribution/uniformInt'; -// @ts-ignore - Only available once `pnpm bench:setup` has been run -import { uniformBigInt as uniformBigIntMain } from 'pure-rand-main/distribution/uniformBigInt'; -// @ts-ignore - Only available once `pnpm bench:setup` has been run -import { uniformFloat32 as uniformFloat32Main } from 'pure-rand-main/distribution/uniformFloat32'; -// @ts-ignore - Only available once `pnpm bench:setup` has been run -import { uniformFloat64 as uniformFloat64Main } from 'pure-rand-main/distribution/uniformFloat64'; // oxlint-enable typescript/ban-ts-comment export type PureRand = { @@ -76,14 +54,3 @@ export const current: PureRand = { uniformFloat32: uniformFloat32Current, uniformFloat64: uniformFloat64Current, }; - -export const main: PureRand = { - congruential32: congruential32Main, - mersenne: mersenneMain, - xoroshiro128plus: xoroshiro128plusMain, - xorshift128plus: xorshift128plusMain, - uniformInt: uniformIntMain, - uniformBigInt: uniformBigIntMain, - uniformFloat32: uniformFloat32Main, - uniformFloat64: uniformFloat64Main, -}; diff --git a/src/distribution/distribution.bench.ts b/src/distribution/distribution.bench.ts index 93156e3..231ce8c 100644 --- a/src/distribution/distribution.bench.ts +++ b/src/distribution/distribution.bench.ts @@ -1,126 +1,105 @@ import { describe, bench } from 'vitest'; -import { current, main, type PureRand } from '../__bench__/Imports.js'; -import type { RandomGenerator } from '../types/RandomGenerator'; +import { current } from '../__bench__/Imports.js'; -// Run each version several times so that ordering/warmup noise gets averaged out -// when comparing current against main. -const numReplicas = 3; - -// Always compare the current build against main: each comparison gets its own -// describe block holding `numReplicas` interleaved benches per version, named -// `current-${i}` and `main-${i}`. Both versions are fed the same seeded generator -// so the inputs are identical. -function compare(name: string, run: (api: PureRand, rng: RandomGenerator) => void): void { - describe(name, () => { - for (let i = 0; i !== numReplicas; ++i) { - const rngCurrent = current.xorshift128plus(0); - const rngMain = main.xorshift128plus(0); - bench(`current-${i}`, () => { - run(current, rngCurrent); - }); - bench(`main-${i}`, () => { - run(main, rngMain); - }); - } - }); -} +const { uniformInt, uniformBigInt, uniformFloat32, uniformFloat64 } = current; +const rng = current.xorshift128plus(0); describe('distribution', () => { describe('pow2 ranges', () => { // range < 2 ** 8 const smallRangeLabel = `{{S range}} [0, 2**4 -1]`; - compare(`uniformInt @@ ${smallRangeLabel}`, (api, rng) => { - api.uniformInt(rng, 0, 15); + bench(`uniformInt @@ ${smallRangeLabel}`, () => { + uniformInt(rng, 0, 15); }); - compare(`uniformBigInt @@ ${smallRangeLabel}`, (api, rng) => { - api.uniformBigInt(rng, 0n, 15n); + bench(`uniformBigInt @@ ${smallRangeLabel}`, () => { + uniformBigInt(rng, 0n, 15n); }); // 2 ** 8 <= range < 2 ** 31 const mediumRangeLabel = `{{M range}} [0, 2**21 -1]`; - compare(`uniformInt @@ ${mediumRangeLabel}`, (api, rng) => { - api.uniformInt(rng, 0, 2097151); + bench(`uniformInt @@ ${mediumRangeLabel}`, () => { + uniformInt(rng, 0, 2097151); }); - compare(`uniformBigInt @@ ${mediumRangeLabel}`, (api, rng) => { - api.uniformBigInt(rng, 0n, 2097151n); + bench(`uniformBigInt @@ ${mediumRangeLabel}`, () => { + uniformBigInt(rng, 0n, 2097151n); }); // 2 ** 31 <= range < 2 ** 32 const largeRangeLabel = `{{L range}} [0, 2**32 -1]`; - compare(`uniformInt @@ ${largeRangeLabel}`, (api, rng) => { - api.uniformInt(rng, 0, 4294967295); + bench(`uniformInt @@ ${largeRangeLabel}`, () => { + uniformInt(rng, 0, 4294967295); }); - compare(`uniformBigInt @@ ${largeRangeLabel}`, (api, rng) => { - api.uniformBigInt(rng, 0n, 4294967295n); + bench(`uniformBigInt @@ ${largeRangeLabel}`, () => { + uniformBigInt(rng, 0n, 4294967295n); }); // 2 ** 32 <= range < Number.MAX_SAFE_INTEGER const veryLargeRangeLabel = `{{XL range}} [0, 2**40 -1]`; - compare(`uniformInt @@ ${veryLargeRangeLabel}`, (api, rng) => { - api.uniformInt(rng, 0, 1099511627775); + bench(`uniformInt @@ ${veryLargeRangeLabel}`, () => { + uniformInt(rng, 0, 1099511627775); }); - compare(`uniformBigInt @@ ${veryLargeRangeLabel}`, (api, rng) => { - api.uniformBigInt(rng, 0n, 1099511627775n); + bench(`uniformBigInt @@ ${veryLargeRangeLabel}`, () => { + uniformBigInt(rng, 0n, 1099511627775n); }); // Number.MAX_SAFE_INTEGER << range // WARNING: "number" type cannot fit for such ranges const veryVeryLargeRangeLabel = `{{XXL range}} [0, 2**80 -1]`; - compare(`uniformBigInt @@ ${veryVeryLargeRangeLabel}`, (api, rng) => { - api.uniformBigInt(rng, 0n, 1208925819614629174706175n); + bench(`uniformBigInt @@ ${veryVeryLargeRangeLabel}`, () => { + uniformBigInt(rng, 0n, 1208925819614629174706175n); }); }); describe('various ranges', () => { // no specific range - compare(`uniformFloat32 @@ {{float}} [0, 1)`, (api, rng) => { - api.uniformFloat32(rng); + bench(`uniformFloat32 @@ {{float}} [0, 1)`, () => { + uniformFloat32(rng); }); - compare(`uniformFloat64 @@ {{float}} [0, 1)`, (api, rng) => { - api.uniformFloat64(rng); + bench(`uniformFloat64 @@ {{float}} [0, 1)`, () => { + uniformFloat64(rng); }); // range < 2 ** 8 const smallRangeLabel = `{{S range}} [0, 48]`; - compare(`uniformInt @@ ${smallRangeLabel}`, (api, rng) => { - api.uniformInt(rng, 0, 48); + bench(`uniformInt @@ ${smallRangeLabel}`, () => { + uniformInt(rng, 0, 48); }); - compare(`uniformBigInt @@ ${smallRangeLabel}`, (api, rng) => { - api.uniformBigInt(rng, 0n, 48n); + bench(`uniformBigInt @@ ${smallRangeLabel}`, () => { + uniformBigInt(rng, 0n, 48n); }); // 2 ** 8 <= range < 2 ** 31 const mediumRangeLabel = `{{M range}} [0, 1_000_000_000]`; - compare(`uniformInt @@ ${mediumRangeLabel}`, (api, rng) => { - api.uniformInt(rng, 0, 1_000_000_000); + bench(`uniformInt @@ ${mediumRangeLabel}`, () => { + uniformInt(rng, 0, 1_000_000_000); }); - compare(`uniformBigInt @@ ${mediumRangeLabel}`, (api, rng) => { - api.uniformBigInt(rng, 0n, 1_000_000_000n); + bench(`uniformBigInt @@ ${mediumRangeLabel}`, () => { + uniformBigInt(rng, 0n, 1_000_000_000n); }); // 2 ** 31 <= range < 2 ** 32 const largeRangeLabel = `{{L range}} [0, 4_000_000_000]`; - compare(`uniformInt @@ ${largeRangeLabel}`, (api, rng) => { - api.uniformInt(rng, 0, 4_000_000_000); + bench(`uniformInt @@ ${largeRangeLabel}`, () => { + uniformInt(rng, 0, 4_000_000_000); }); - compare(`uniformBigInt @@ ${largeRangeLabel}`, (api, rng) => { - api.uniformBigInt(rng, 0n, 4_000_000_000n); + bench(`uniformBigInt @@ ${largeRangeLabel}`, () => { + uniformBigInt(rng, 0n, 4_000_000_000n); }); // 2 ** 32 <= range < Number.MAX_SAFE_INTEGER const veryLargeRangeLabel = `{{XL range}} [0, 8_000_000_000]`; - compare(`uniformInt @@ ${veryLargeRangeLabel}`, (api, rng) => { - api.uniformInt(rng, 0, 8_000_000_000); + bench(`uniformInt @@ ${veryLargeRangeLabel}`, () => { + uniformInt(rng, 0, 8_000_000_000); }); - compare(`uniformBigInt @@ ${veryLargeRangeLabel}`, (api, rng) => { - api.uniformBigInt(rng, 0n, 8_000_000_000n); + bench(`uniformBigInt @@ ${veryLargeRangeLabel}`, () => { + uniformBigInt(rng, 0n, 8_000_000_000n); }); // Number.MAX_SAFE_INTEGER << range // WARNING: "number" type cannot fit for such ranges const veryVeryLargeRangeLabel = `{{XXL range}} [0, 100_000_000_000_000_000]`; - compare(`uniformBigInt @@ ${veryVeryLargeRangeLabel}`, (api, rng) => { - api.uniformBigInt(rng, 0n, 100_000_000_000_000_000n); + bench(`uniformBigInt @@ ${veryVeryLargeRangeLabel}`, () => { + uniformBigInt(rng, 0n, 100_000_000_000_000_000n); }); }); }); diff --git a/src/generator/generator.bench.ts b/src/generator/generator.bench.ts index 3313d4f..1d43544 100644 --- a/src/generator/generator.bench.ts +++ b/src/generator/generator.bench.ts @@ -1,74 +1,50 @@ import { describe, bench } from 'vitest'; -import { current, main, type PureRand } from '../__bench__/Imports.js'; +import { current } from '../__bench__/Imports.js'; const numInts = 5_000; -// Run each version several times so that ordering/warmup noise gets averaged out -// when comparing current against main. -const numReplicas = 3; - type GeneratorName = 'congruential32' | 'mersenne' | 'xoroshiro128plus' | 'xorshift128plus'; const generatorNames: GeneratorName[] = ['congruential32', 'mersenne', 'xoroshiro128plus', 'xorshift128plus']; -// Always compare the current build against main: each comparison gets its own -// describe block holding `numReplicas` interleaved benches per version, named -// `current-${i}` and `main-${i}`. -function compare(name: string, make: (api: PureRand) => () => void): void { - describe(name, () => { - for (let i = 0; i !== numReplicas; ++i) { - bench(`current-${i}`, make(current)); - bench(`main-${i}`, make(main)); - } - }); -} - describe('generator', () => { describe(`init and ${numInts} next`, () => { for (const name of generatorNames) { - compare(name, (api) => { - let seed = 0; - return () => { - const rng = api[name](seed++); - for (let i = 0; i !== numInts; ++i) { - rng.next(); - } - }; + const generator = current[name]; + let seed = 0; + bench(name, () => { + const rng = generator(seed++); + for (let i = 0; i !== numInts; ++i) { + rng.next(); + } }); } }); describe(`${numInts} next`, () => { for (const name of generatorNames) { - compare(name, (api) => { - const rng = api[name](0); - return () => { - for (let i = 0; i !== numInts; ++i) { - rng.next(); - } - }; + const rng = current[name](0); + bench(name, () => { + for (let i = 0; i !== numInts; ++i) { + rng.next(); + } }); } }); for (const name of generatorNames) { describe(name, () => { - compare('init', (api) => { - let seed = 0; - return () => { - api[name]((seed = (seed + 1) | 0)); - }; + const generator = current[name]; + let seed = 0; + bench('init', () => { + generator((seed = (seed + 1) | 0)); }); - compare('next', (api) => { - const rng = api[name](0); - return () => { - rng.next(); - }; + + const rng = generator(0); + bench('next', () => { + rng.next(); }); - compare('jump', (api) => { - const rng = api[name](0); - return () => { - rng.jump(); - }; + bench('jump', () => { + rng.jump(); }); }); }