From 98f0b8ba7447e9a07e80522e2638e6791429ba75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Wed, 18 Dec 2024 13:00:23 +0100 Subject: [PATCH 1/4] chore(deps): update tinybench to 3.x.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérôme Benoit --- package.json | 1 + packages/vitest/package.json | 2 +- packages/vitest/src/node/reporters/base.ts | 2 +- .../reporters/benchmark/table/tableRender.ts | 31 +++---- packages/vitest/src/public/index.ts | 6 +- packages/vitest/src/runtime/benchmark.ts | 8 +- .../vitest/src/runtime/runners/benchmark.ts | 86 ++++++++----------- .../vitest/src/runtime/types/benchmark.ts | 18 ++-- pnpm-lock.yaml | 11 +-- test/benchmark/test/reporter.test.ts | 2 +- 10 files changed, 77 insertions(+), 90 deletions(-) diff --git a/package.json b/package.json index 4f4ed7150ee6..cebabb0328a9 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "release": "tsx scripts/release.ts", "test": "pnpm --filter test-core test:threads", "test:ci": "CI=true pnpm -r --reporter-hide-prefix --stream --sequential --filter '@vitest/test-*' --filter !test-browser run test", + "test:benchmark": "CI=true pnpm -r --reporter-hide-prefix --stream --sequential --filter '@vitest/test-benchmark' --filter !test-browser run test", "test:examples": "CI=true pnpm -r --reporter-hide-prefix --stream --filter '@vitest/example-*' run test", "test:ecosystem-ci": "ECOSYSTEM_CI=true pnpm test:ci", "typecheck": "tsc -p tsconfig.check.json --noEmit", diff --git a/packages/vitest/package.json b/packages/vitest/package.json index f11892391c1d..541271346297 100644 --- a/packages/vitest/package.json +++ b/packages/vitest/package.json @@ -163,7 +163,7 @@ "magic-string": "^0.30.14", "pathe": "^1.1.2", "std-env": "^3.8.0", - "tinybench": "^2.9.0", + "tinybench": "^3.0.7", "tinyexec": "^0.3.1", "tinypool": "^1.0.2", "tinyrainbow": "^1.2.0", diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 0b93b649b357..5d6e26efdf08 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -455,7 +455,7 @@ export abstract class BaseReporter implements Reporter { .sort((a, b) => a.result!.benchmark!.rank - b.result!.benchmark!.rank) for (const sibling of siblings) { - const number = (sibling.result!.benchmark!.mean / bench.result!.benchmark!.mean).toFixed(2) + const number = (sibling.result!.benchmark!.latency.mean / bench.result!.benchmark!.latency.mean).toFixed(2) this.log(c.green(` ${number}x `) + c.gray('faster than ') + sibling.name) } diff --git a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts index f3cd66bc2094..434cfd4c81f6 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts @@ -45,10 +45,10 @@ function formatNumber(number: number) { const tableHead = [ 'name', - 'hz', + 'mean', 'min', 'max', - 'mean', + 'p50/median', 'p75', 'p99', 'p995', @@ -58,18 +58,19 @@ const tableHead = [ ] function renderBenchmarkItems(result: BenchmarkResult) { + console.log('table', result) return [ result.name, - formatNumber(result.hz || 0), - formatNumber(result.min || 0), - formatNumber(result.max || 0), - formatNumber(result.mean || 0), - formatNumber(result.p75 || 0), - formatNumber(result.p99 || 0), - formatNumber(result.p995 || 0), - formatNumber(result.p999 || 0), - `±${(result.rme || 0).toFixed(2)}%`, - (result.sampleCount || 0).toString(), + formatNumber(result.latency.mean || 0), + formatNumber(result.latency.min || 0), + formatNumber(result.latency.max || 0), + formatNumber(result.latency.p50 || 0), + formatNumber(result.latency.p75 || 0), + formatNumber(result.latency.p99 || 0), + formatNumber(result.latency.p995 || 0), + formatNumber(result.latency.p999 || 0), + `±${(result.latency.rme || 0).toFixed(2)}%`, + (result.numberOfSamples || 0).toString(), ] } @@ -93,16 +94,16 @@ function renderBenchmark(result: BenchmarkResult, widths: number[]) { const padded = padRow(renderBenchmarkItems(result), widths) return [ padded[0], // name - c.blue(padded[1]), // hz + c.blue(padded[1]), // mean c.cyan(padded[2]), // min c.cyan(padded[3]), // max - c.cyan(padded[4]), // mean + c.cyan(padded[4]), // p50/median c.cyan(padded[5]), // p75 c.cyan(padded[6]), // p99 c.cyan(padded[7]), // p995 c.cyan(padded[8]), // p999 c.dim(padded[9]), // rem - c.dim(padded[10]), // sample + c.dim(padded[10]), // samples ].join(' ') } diff --git a/packages/vitest/src/public/index.ts b/packages/vitest/src/public/index.ts index f5b6ec05947d..83c326dd0c14 100644 --- a/packages/vitest/src/public/index.ts +++ b/packages/vitest/src/public/index.ts @@ -130,14 +130,14 @@ export type { } from '../runtime/config' export type { - BenchFactory, + Bench as BenchFactory, BenchFunction, Benchmark, BenchmarkAPI, BenchmarkResult, BenchOptions, - BenchTask, - BenchTaskResult, + TinybenchTask as BenchTask, + TinybenchTaskResult as BenchTaskResult, } from '../runtime/types/benchmark' export { assertType } from '../typecheck/assertType' diff --git a/packages/vitest/src/runtime/benchmark.ts b/packages/vitest/src/runtime/benchmark.ts index 6772720a31a1..d9e200ef6903 100644 --- a/packages/vitest/src/runtime/benchmark.ts +++ b/packages/vitest/src/runtime/benchmark.ts @@ -6,10 +6,10 @@ import { noop } from '@vitest/utils' import { getWorkerState } from './utils' const benchFns = new WeakMap() -const benchOptsMap = new WeakMap() +const benchOptsMap = new WeakMap() export function getBenchOptions(key: Test): BenchOptions { - return benchOptsMap.get(key) + return benchOptsMap.get(key)! } export function getBenchFn(key: Test): BenchFunction { @@ -17,7 +17,7 @@ export function getBenchFn(key: Test): BenchFunction { } export const bench = createBenchmark(function ( - name, + name: string | Function, fn: BenchFunction = noop, options: BenchOptions = {}, ) { @@ -32,7 +32,7 @@ export const bench = createBenchmark(function ( }, }) benchFns.set(task, fn) - benchOptsMap.set(task, options) + benchOptsMap.set(task, { ...options, name: formatName(name) }) }) function createBenchmark( diff --git a/packages/vitest/src/runtime/runners/benchmark.ts b/packages/vitest/src/runtime/runners/benchmark.ts index 59aa7ac7dc64..c60d3f0cf7eb 100644 --- a/packages/vitest/src/runtime/runners/benchmark.ts +++ b/packages/vitest/src/runtime/runners/benchmark.ts @@ -9,7 +9,7 @@ import type { VitestExecutor } from '../execute' import type { Benchmark, BenchmarkResult, - BenchTask, + TinybenchTask, } from '../types/benchmark' import { updateTask as updateRunnerTask } from '@vitest/runner' import { createDefer, getSafeTimers } from '@vitest/utils' @@ -20,20 +20,15 @@ function createBenchmarkResult(name: string): BenchmarkResult { return { name, rank: 0, - rme: 0, - samples: [] as number[], + numberOfSamples: 0, } as BenchmarkResult } -const benchmarkTasks = new WeakMap() - async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) { - const { Task, Bench } = await runner.importTinybench() - const start = performance.now() const benchmarkGroup: Benchmark[] = [] - const benchmarkSuiteGroup = [] + const benchmarkSuiteGroup: Suite[] = [] for (const task of suite.tasks) { if (task.mode !== 'run' && task.mode !== 'queued') { continue @@ -47,12 +42,15 @@ async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) { } } - // run sub suites sequentially - for (const subSuite of benchmarkSuiteGroup) { - await runBenchmarkSuite(subSuite, runner) + if (benchmarkSuiteGroup.length) { + for (const subSuite of benchmarkSuiteGroup) { + await runBenchmarkSuite(subSuite, runner) + } } if (benchmarkGroup.length) { + const { Bench } = await runner.importTinybench() + const defer = createDefer() suite.result = { state: 'run', @@ -62,24 +60,25 @@ async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) { updateTask(suite) const addBenchTaskListener = ( - task: InstanceType, + task: InstanceType, benchmark: Benchmark, ) => { + task.addEventListener('start', () => { + benchmark.result!.state = 'run' + updateTask(benchmark) + }) task.addEventListener( 'complete', (e) => { const task = e.task - const taskRes = task.result! + benchmark.result!.state = 'pass' const result = benchmark.result!.benchmark! - Object.assign(result, taskRes) - // compute extra stats and free raw samples as early as possible - const samples = result.samples - result.sampleCount = samples.length - result.median = samples.length % 2 - ? samples[Math.floor(samples.length / 2)] - : (samples[samples.length / 2] + samples[samples.length / 2 - 1]) / 2 + Object.assign(result, task!.result!) + result.numberOfSamples = result.latency.samples.length if (!runner.config.benchmark?.includeSamples) { result.samples.length = 0 + result.latency.samples.length = 0 + result.throughput.samples.length = 0 } updateTask(benchmark) }, @@ -90,8 +89,9 @@ async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) { task.addEventListener( 'error', (e) => { - const task = e.task - defer.reject(benchmark ? task.result!.error : e) + benchmark.result!.state = 'fail' + updateTask(benchmark) + defer.reject(e.error) }, { once: true, @@ -99,49 +99,35 @@ async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) { ) } - benchmarkGroup.forEach((benchmark) => { - const options = getBenchOptions(benchmark) - const benchmarkInstance = new Bench(options) - - const benchmarkFn = getBenchFn(benchmark) - + const { setTimeout } = getSafeTimers() + const tasks: [TinybenchTask, Benchmark][] = [] + for (const benchmark of benchmarkGroup) { + const bench = new Bench(getBenchOptions(benchmark)) + bench.add(benchmark.name, getBenchFn(benchmark)) benchmark.result = { state: 'run', startTime: start, benchmark: createBenchmarkResult(benchmark.name), } - - const task = new Task(benchmarkInstance, benchmark.name, benchmarkFn) - benchmarkTasks.set(benchmark, task) - addBenchTaskListener(task, benchmark) + addBenchTaskListener(bench.getTask(benchmark.name)!, benchmark) updateTask(benchmark) - }) - - const { setTimeout } = getSafeTimers() - const tasks: [BenchTask, Benchmark][] = [] - for (const benchmark of benchmarkGroup) { - const task = benchmarkTasks.get(benchmark)! - await task.warmup() - tasks.push([ - await new Promise(resolve => - setTimeout(async () => { - resolve(await task.run()) - }), - ), - benchmark, - ]) + tasks.push([await new Promise(resolve => + setTimeout(async () => { + resolve((await bench.run())[0]) + }), + ), benchmark]) } suite.result!.duration = performance.now() - start - suite.result!.state = 'pass' + suite.result!.state = benchmarkGroup.every(benchmark => benchmark.result!.state === 'pass') ? 'pass' : 'fail' tasks - .sort(([taskA], [taskB]) => taskA.result!.mean - taskB.result!.mean) + .sort(([taskA], [taskB]) => taskA.result!.latency.mean - taskB.result!.latency.mean) .forEach(([, benchmark], idx) => { benchmark.result!.state = 'pass' if (benchmark) { const result = benchmark.result!.benchmark! - result.rank = Number(idx) + 1 + result.rank = idx + 1 updateTask(benchmark) } }) diff --git a/packages/vitest/src/runtime/types/benchmark.ts b/packages/vitest/src/runtime/types/benchmark.ts index 9ffd62c4f83b..647a4b1145aa 100644 --- a/packages/vitest/src/runtime/types/benchmark.ts +++ b/packages/vitest/src/runtime/types/benchmark.ts @@ -1,28 +1,26 @@ import type { Test } from '@vitest/runner' import type { ChainableFunction } from '@vitest/runner/utils' import type { - Bench as BenchFactory, + Bench, + Fn as BenchFunction, Options as BenchOptions, - Task as BenchTask, - TaskResult as BenchTaskResult, - TaskResult as TinybenchResult, + Task as TinybenchTask, + TaskResult as TinybenchTaskResult, } from 'tinybench' export interface Benchmark extends Test { meta: { benchmark: true - result?: BenchTaskResult + result?: TinybenchTaskResult } } -export interface BenchmarkResult extends TinybenchResult { +export interface BenchmarkResult extends TinybenchTaskResult { name: string rank: number - sampleCount: number - median: number + numberOfSamples: number } -export type BenchFunction = (this: BenchFactory) => Promise | void type ChainableBenchmarkAPI = ChainableFunction< 'skip' | 'only' | 'todo', (name: string | Function, fn?: BenchFunction, options?: BenchOptions) => void @@ -32,4 +30,4 @@ export type BenchmarkAPI = ChainableBenchmarkAPI & { runIf: (condition: any) => ChainableBenchmarkAPI } -export { BenchFactory, BenchOptions, BenchTask, BenchTaskResult } +export { Bench, BenchFunction, BenchOptions, TinybenchTask, TinybenchTaskResult } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e3ee66946cff..f322d62bd48a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -893,8 +893,8 @@ importers: specifier: ^3.8.0 version: 3.8.0 tinybench: - specifier: ^2.9.0 - version: 2.9.0 + specifier: ^3.0.7 + version: 3.0.7 tinyexec: specifier: ^0.3.1 version: 0.3.1 @@ -8764,8 +8764,9 @@ packages: tiny-glob@0.2.9: resolution: {integrity: sha512-g/55ssRPUjShh+xkfx9UPDXqhckHEsHr4Vd9zX55oSdGZc/MD0m3sferOkwWtp98bv+kcVfEHtRJgBVJzelrzg==} - tinybench@2.9.0: - resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + tinybench@3.0.7: + resolution: {integrity: sha512-soxV7Dp8eDKvPDv3c4qPJbUjLm1cZxFlsTaIH+FqalsazJzFrLG59dpiIN8OfgVcl11Hfj2b7apD73inCB67Mw==} + engines: {node: '>=18.0.0'} tinyexec@0.3.0: resolution: {integrity: sha512-tVGE0mVJPGb0chKhqmsoosjsS+qUnJVGJpZgsHYQcGoPlG3B51R3PouqTgEGH2Dc9jjFyOqOpix6ZHNMXp1FZg==} @@ -18307,7 +18308,7 @@ snapshots: globalyzer: 0.1.0 globrex: 0.1.2 - tinybench@2.9.0: {} + tinybench@3.0.7: {} tinyexec@0.3.0: {} diff --git a/test/benchmark/test/reporter.test.ts b/test/benchmark/test/reporter.test.ts index d118732ff71d..2b2b2deaf4d7 100644 --- a/test/benchmark/test/reporter.test.ts +++ b/test/benchmark/test/reporter.test.ts @@ -41,7 +41,7 @@ it.for([true, false])('includeSamples %s', async (includeSamples) => { assert(result.ctx) const allSamples = [...result.ctx.state.idMap.values()] .filter(t => t.meta.benchmark) - .map(t => t.result?.benchmark?.samples) + .map(t => t.result?.benchmark?.latency.samples) if (includeSamples) { expect(allSamples[0]).not.toEqual([]) } From cd23ff154846e7bb0b85938a3ca905a334b79320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Wed, 18 Dec 2024 13:36:57 +0100 Subject: [PATCH 2/4] fix: import the proprer FnOptions tinybench type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérôme Benoit --- packages/vitest/src/runtime/types/benchmark.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vitest/src/runtime/types/benchmark.ts b/packages/vitest/src/runtime/types/benchmark.ts index 647a4b1145aa..508a6f25e439 100644 --- a/packages/vitest/src/runtime/types/benchmark.ts +++ b/packages/vitest/src/runtime/types/benchmark.ts @@ -3,7 +3,7 @@ import type { ChainableFunction } from '@vitest/runner/utils' import type { Bench, Fn as BenchFunction, - Options as BenchOptions, + FnOptions as BenchOptions, Task as TinybenchTask, TaskResult as TinybenchTaskResult, } from 'tinybench' From 7a6a69f31bbe0ff37c8dc0b9c59e8dfe32539bb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Wed, 18 Dec 2024 13:50:05 +0100 Subject: [PATCH 3/4] fix: import the right BenchOptions type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérôme Benoit --- packages/vitest/src/runtime/types/benchmark.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vitest/src/runtime/types/benchmark.ts b/packages/vitest/src/runtime/types/benchmark.ts index 508a6f25e439..f573adb14439 100644 --- a/packages/vitest/src/runtime/types/benchmark.ts +++ b/packages/vitest/src/runtime/types/benchmark.ts @@ -3,7 +3,7 @@ import type { ChainableFunction } from '@vitest/runner/utils' import type { Bench, Fn as BenchFunction, - FnOptions as BenchOptions, + BenchOptions, Task as TinybenchTask, TaskResult as TinybenchTaskResult, } from 'tinybench' From 3a4d73d50a077be31db12e973972a6b1970f09ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Wed, 18 Dec 2024 14:06:40 +0100 Subject: [PATCH 4/4] refactor: remove console.log debugging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérôme Benoit --- .../vitest/src/node/reporters/benchmark/table/tableRender.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts index 434cfd4c81f6..0500e347790b 100644 --- a/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts +++ b/packages/vitest/src/node/reporters/benchmark/table/tableRender.ts @@ -58,7 +58,6 @@ const tableHead = [ ] function renderBenchmarkItems(result: BenchmarkResult) { - console.log('table', result) return [ result.name, formatNumber(result.latency.mean || 0),