diff --git a/packages/core/src/introspection.ts b/packages/core/src/introspection.ts index 8906c1d9..eeb26010 100644 --- a/packages/core/src/introspection.ts +++ b/packages/core/src/introspection.ts @@ -33,6 +33,10 @@ export const getV8Flags = () => { "--predictable-gc-schedule", "--expose-gc", "--no-concurrent-sweeping", + // --predictable keeps posting incremental marking steps as tasks on + // the event loop, so async work before the benchmark (e.g. module + // loading) shifts the GC count and heap layout it measures in. + "--no-incremental-marking-task", "--max-old-space-size=4096", ], ); diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index e40dd902..6ba97c9b 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -1,9 +1,10 @@ /** - * Process isolation for walltime runs: every benchmark runs in fresh copies - * of the benchmark process (forks), so the allocator state, JIT state, and - * handles that other benchmarks or setup code leave behind can't change its - * measurement. Rounds within one process are correlated (JIT tier-up, heap - * sizing), so the rounds of several forks are pooled into one result. + * Process isolation: every benchmark runs in fresh copies of the benchmark + * process (forks), so the allocator state, JIT state, and handles that other + * benchmarks or setup code leave behind can't change its measurement. Walltime + * rounds within one process are correlated (JIT tier-up, heap sizing), so the + * rounds of several forks are pooled into one result. Analysis runs measure + * each benchmark once, in one fork. */ import { spawn, spawnSync } from "child_process"; import { randomUUID } from "crypto"; @@ -12,6 +13,7 @@ import { tmpdir } from "os"; import { join } from "path"; import { deserialize, serialize } from "v8"; import { constants, gunzipSync, gzipSync } from "zlib"; +import { getInstrumentMode } from "./runnerMode"; /** Accepts `0`, `false`, `1`, or `true`; defaults to enabled. */ export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; @@ -44,10 +46,11 @@ export type IsolationMode = | { kind: "Fork"; uri: string; samplesFile: string }; /** - * The isolation of a walltime run, from the environment. `forks` is the - * integration's option, which {@link WALLTIME_FORKS_ENV} overrides. + * The isolation of this run, from the environment. A walltime benchmark runs + * in `forks` forks, the integration's option, which {@link WALLTIME_FORKS_ENV} + * overrides; an analysis benchmark is measured once, so it runs in one. */ -export function walltimeIsolation(forks?: number): IsolationMode { +export function processIsolation(forks?: number): IsolationMode { const uri = process.env[ISOLATED_BENCHMARK_ENV]; if (uri !== undefined) { const samplesFile = process.env[FORK_SAMPLES_FILE_ENV]; @@ -67,7 +70,8 @@ export function walltimeIsolation(forks?: number): IsolationMode { `[CodSpeed] ${PROCESS_ISOLATION_ENV} must be 0, false, 1, or true, got \`${isolation}\``, ); } - return { kind: "Forks", forks: resolveWalltimeForks(forks) }; + const analysis = getInstrumentMode() === "analysis"; + return { kind: "Forks", forks: analysis ? 1 : resolveWalltimeForks(forks) }; } /** @@ -221,7 +225,11 @@ function forkSamples( } } -/** Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. */ +/** + * Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. + * Analysis forks hand none: the instruments hold their measurement, and the + * file only tells the parent that they measured their benchmark. + */ export function writeForkSamples( samplesFile: string, samples: ArrayLike, diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs index bd2d6d2e..29048972 100644 --- a/packages/core/tests/fixtures/isolation.cjs +++ b/packages/core/tests/fixtures/isolation.cjs @@ -1,12 +1,12 @@ // Run by `isolation.integ.test.ts`. Measures benchmark "a" in forks. const { + processIsolation, runForks, runForksSync, - walltimeIsolation, writeForkSamples, } = require("../../dist/index.cjs.js"); -const mode = walltimeIsolation(); +const mode = processIsolation(); function measure() { console.log(`ran ${process.pid} ${JSON.stringify(process.execArgv)}`); diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts index c61d3f45..aaeccb3e 100644 --- a/packages/core/tests/isolation.integ.test.ts +++ b/packages/core/tests/isolation.integ.test.ts @@ -4,9 +4,9 @@ import { FORK_SAMPLES_FILE_ENV, ISOLATED_BENCHMARK_ENV, PROCESS_ISOLATION_ENV, + processIsolation, resolveWalltimeForks, WALLTIME_FORKS_ENV, - walltimeIsolation, } from "../src/isolation"; const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); @@ -21,6 +21,8 @@ const NODE_FLAG = "--no-warnings"; const SAMPLES_PER_FORK = 2; const ENV_KEYS = [ + "CODSPEED_ENV", + "CODSPEED_RUNNER_MODE", PROCESS_ISOLATION_ENV, ISOLATED_BENCHMARK_ENV, FORK_SAMPLES_FILE_ENV, @@ -75,7 +77,7 @@ describe("resolveWalltimeForks", () => { }); }); -describe("walltimeIsolation", () => { +describe("processIsolation", () => { it.each([ [undefined, "Forks"], ["1", "Forks"], @@ -84,12 +86,12 @@ describe("walltimeIsolation", () => { ["false", "SharedProcess"], ])("selects %s as %s", (value, kind) => { setEnv({ [PROCESS_ISOLATION_ENV]: value }); - expect(walltimeIsolation().kind).toBe(kind); + expect(processIsolation().kind).toBe(kind); }); it("runs everything in this process when isolation is off, whatever the fork count", () => { setEnv({ [PROCESS_ISOLATION_ENV]: "false", [WALLTIME_FORKS_ENV]: "4" }); - expect(walltimeIsolation(2).kind).toBe("SharedProcess"); + expect(processIsolation(2).kind).toBe("SharedProcess"); }); it("restricts a fork to its benchmark, whatever the isolation setting", () => { @@ -98,12 +100,21 @@ describe("walltimeIsolation", () => { [ISOLATED_BENCHMARK_ENV]: "file::bench", [FORK_SAMPLES_FILE_ENV]: "samples.json", }); - expect(walltimeIsolation()).toEqual({ + expect(processIsolation()).toEqual({ kind: "Fork", uri: "file::bench", samplesFile: "samples.json", }); }); + + it("runs an analysis benchmark in one fork, whatever the walltime fork count", () => { + setEnv({ + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + [WALLTIME_FORKS_ENV]: "4", + }); + expect(processIsolation(2)).toEqual({ kind: "Forks", forks: 1 }); + }); }); interface FixtureOptions { diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 65bd6183..f03d3faf 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -6,6 +6,8 @@ import { optimizeFunctionSync, wrapWithRootFrame, wrapWithRootFrameSync, + writeForkSamples, + type IsolationMode, } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { BaseBenchRunner } from "./shared"; @@ -13,8 +15,9 @@ import { BaseBenchRunner } from "./shared"; export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, + isolationMode: IsolationMode, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile); + const runner = new AnalysisBenchRunner(bench, rootCallingFile, isolationMode); runner.setupBenchMethods(); } @@ -28,6 +31,16 @@ class AnalysisBenchRunner extends BaseBenchRunner { return InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; } + private completeTask(uri: string): void { + this.logTaskCompletion(uri, this.taskCompletionMessage()); + if (this.isolationMode.kind === "Fork") { + writeForkSamples(this.isolationMode.samplesFile, []); + } + } + + /** The instruments hold the measurement of the forks. */ + protected collectForks(): void {} + protected async runTaskAsync(task: Task, uri: string): Promise { const { fnOpts, fn } = this.getTaskData(task); @@ -54,7 +67,7 @@ class AnalysisBenchRunner extends BaseBenchRunner { await fnOpts?.afterEach?.call(task, "run"); await fnOpts?.afterAll?.call(task, "run"); - this.logTaskCompletion(uri, this.taskCompletionMessage()); + this.completeTask(uri); } protected runTaskSync(task: Task, uri: string): void { @@ -62,9 +75,9 @@ class AnalysisBenchRunner extends BaseBenchRunner { fnOpts?.beforeAll?.call(task, "run"); - optimizeFunctionSync(async () => { + optimizeFunctionSync(() => { fnOpts?.beforeEach?.call(task, "run"); - await fn(); + fn(); fnOpts?.afterEach?.call(task, "run"); }); @@ -83,7 +96,7 @@ class AnalysisBenchRunner extends BaseBenchRunner { fnOpts?.afterEach?.call(task, "run"); fnOpts?.afterAll?.call(task, "run"); - this.logTaskCompletion(uri, this.taskCompletionMessage()); + this.completeTask(uri); } protected finalizeAsyncRun(): Task[] { diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index bb903ae0..cbd9a8f1 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -4,17 +4,18 @@ import { getInstrumentMode, InstrumentHooks, mongoMeasurement, + processIsolation, SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, wrapWithRootFrameSync, type WalltimeOptions, } from "@codspeed/core"; -import { Bench } from "tinybench"; +import { Bench, type FnOptions } from "tinybench"; import { setupCodspeedAnalysisBench } from "./analysis"; import { getOrCreateTaskDataMap } from "./taskData"; import { getOrCreateUriMap } from "./uri"; -import { setupCodspeedWalltimeBench } from "./walltime"; +import { runsTaskFn, setupCodspeedWalltimeBench } from "./walltime"; tryIntrospect(); @@ -29,6 +30,7 @@ export function withCodSpeed( const rootCallingFile = getCallingFile(1); const instrumentMode = getInstrumentMode(); + const isolationMode = processIsolation(options.forks); // Compute and register URI for bench const uriMap = getOrCreateUriMap(bench); @@ -43,25 +45,40 @@ export function withCodSpeed( uri += `::${name}`; uriMap.set(name, uri); taskDataMap.set(name, { fn, fnOpts: opts }); - // In walltime mode the task is driven by tinybench's own measured loop, so - // the root frame must be baked into the function tinybench stores rather - // than injected later (the function is an inaccessible private field on the - // task from tinybench v6 onwards). The sync wrapper is transparent to the - // function's return value, so it preserves tinybench's sync/async handling. - const registeredFn = - instrumentMode === "walltime" ? wrapWithRootFrameSync(fn) : fn; - return rawAdd.bind(bench)(name, registeredFn, opts); + const add = rawAdd.bind(bench); + // tinybench calls a function that isn't `async` once when it's added, to + // tell whether it returns a promise. A process that doesn't run the task + // registers a stub instead, so the user's function isn't called there. + if (!runsTaskFn(isolationMode, uri)) { + return add(name, notRunHere, opts); + } + // The analysis runner calls the captured function itself. + if (instrumentMode === "analysis") { + return add(name, fn, opts); + } + // The task is driven by tinybench's own measured loop, so the root frame + // must be baked into the function tinybench stores rather than injected + // later (the function is an inaccessible private field on the task from + // tinybench v6 onwards). An async wrapper would add an await to every + // iteration, so the wrapper stays sync and tinybench is told that an async + // function is async instead (honored from tinybench v6 onwards). + const fnOpts: (FnOptions & { async?: boolean }) | undefined = + fn.constructor.name === "AsyncFunction" ? { ...opts, async: true } : opts; + return add(name, wrapWithRootFrameSync(fn), fnOpts); }; if (instrumentMode === "analysis") { - setupCodspeedAnalysisBench(bench, rootCallingFile); - } else if (instrumentMode === "walltime") { - setupCodspeedWalltimeBench(bench, rootCallingFile, options); + setupCodspeedAnalysisBench(bench, rootCallingFile, isolationMode); + } else { + setupCodspeedWalltimeBench(bench, rootCallingFile, isolationMode); } return bench; } +/** Registered for tasks this process never runs; `async` so tinybench doesn't call it. */ +async function notRunHere(): Promise {} + /** * Dynamically setup the CodSpeed instruments. */ diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 2d86eced..f97b2569 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,10 @@ -import { setupCore, teardownCore, type IsolationMode } from "@codspeed/core"; +import { + runForks, + runForksSync, + setupCore, + teardownCore, + type IsolationMode, +} from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -12,8 +18,7 @@ export abstract class BaseBenchRunner { constructor( bench: Bench, rootCallingFile: string, - /** Only walltime runs are isolated. */ - protected readonly isolationMode: IsolationMode = { kind: "SharedProcess" }, + protected readonly isolationMode: IsolationMode, ) { this.bench = bench; this.rootCallingFile = rootCallingFile; @@ -72,6 +77,7 @@ export abstract class BaseBenchRunner { protected abstract runTaskSync(task: Task, uri: string): void; protected abstract finalizeAsyncRun(): Task[]; protected abstract finalizeSyncRun(): Task[]; + protected abstract collectForks(task: Task, samples: Float64Array): void; public setupBenchMethods(): void { this.bench.run = async () => { @@ -85,7 +91,12 @@ export abstract class BaseBenchRunner { for (const task of tasks) { const uri = this.getTaskUri(task); - await this.runTaskAsync(task, uri); + if (this.isolationMode.kind === "Forks") { + const samples = await runForks(uri, this.isolationMode.forks); + this.collectForks(task, samples); + } else { + await this.runTaskAsync(task, uri); + } } return this.finalizeAsyncRun(); @@ -100,7 +111,12 @@ export abstract class BaseBenchRunner { for (const task of tasks) { const uri = this.getTaskUri(task); - this.runTaskSync(task, uri); + if (this.isolationMode.kind === "Forks") { + const samples = runForksSync(uri, this.isolationMode.forks); + this.collectForks(task, samples); + } else { + this.runTaskSync(task, uri); + } } return this.finalizeSyncRun(); diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 7296ef09..6c3bf483 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -6,17 +6,14 @@ import { mongoMeasurement, msToNs, msToS, - runForks, - runForksSync, samplesToResult, sortedSamplesNs, - walltimeIsolation, writeForkSamples, writeWalltimeResults, type BenchmarkStats, type Benchmark as CodspeedBenchmark, + type IsolationMode, type RoundsResult, - type WalltimeOptions, } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { getBenchOptions } from "./benchOptions"; @@ -25,14 +22,24 @@ import { BaseBenchRunner } from "./shared"; export function setupCodspeedWalltimeBench( bench: Bench, rootCallingFile: string, - options: WalltimeOptions, + isolationMode: IsolationMode, ): void { - const isolationMode = walltimeIsolation(options.forks); const runner = new WalltimeBenchRunner(bench, rootCallingFile, isolationMode); runner.installInstrumentHooks(); runner.setupBenchMethods(); } +/** + * Whether this process runs the task's function: a fork only runs its own + * task, and the parent of the forks only collects their samples. + */ +export function runsTaskFn(isolationMode: IsolationMode, uri: string): boolean { + return ( + isolationMode.kind === "SharedProcess" || + (isolationMode.kind === "Fork" && isolationMode.uri === uri) + ); +} + class WalltimeBenchRunner extends BaseBenchRunner { private codspeedBenchmarks: CodspeedBenchmark[] = []; @@ -101,12 +108,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { } protected async runTaskAsync(task: Task, uri: string): Promise { - const { isolationMode } = this; - if (isolationMode.kind === "Forks") { - const samples = await runForks(uri, isolationMode.forks); - return this.registerForkedBenchmark(task, samples); - } - // run the warmup of the task right before its actual run if (getBenchOptions(this.bench).warmup) { await task.warmup(); @@ -119,13 +120,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { this.registerCodspeedBenchmarkFromTask(task); } - protected runTaskSync(task: Task, uri: string): void { - const { isolationMode } = this; - if (isolationMode.kind === "Forks") { - const samples = runForksSync(uri, isolationMode.forks); - return this.registerForkedBenchmark(task, samples); - } - + protected runTaskSync(task: Task): void { if (getBenchOptions(this.bench).warmup) { task.warmupSync(); } @@ -159,7 +154,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { this.registerCodspeedBenchmark(task, task.result); } - private registerForkedBenchmark(task: Task, samples: Float64Array): void { + protected collectForks(task: Task, samples: Float64Array): void { if (samples.length === 0) { return this.warnNoResult(task); } diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs index 118c5b07..a4cc9755 100644 --- a/packages/tinybench-plugin/tests/fixtures/isolation.mjs +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -1,5 +1,5 @@ -// Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a -// benchmark. +// Run by `isolation.integ.test.ts`. Prints the PID of every process that calls +// a benchmark's function, and of every process that runs the benchmark. import { Bench } from "tinybench"; import { withCodSpeed } from "../../dist/index.es5.js"; @@ -10,10 +10,15 @@ const bench = withCodSpeed( ); for (const name of ["first", "second"]) { - // tinybench calls the function once when it is added, in every process. let calls = 0; bench.add(name, () => { - if (++calls === 2) { + calls++; + if (calls === 1) { + console.log(`called ${process.pid} ${name}`); + } + // tinybench calls the function once when it is added, to tell whether it + // returns a promise. + if (calls === 2) { console.log(`ran ${process.pid} ${name}`); } }); diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index 5e71eeb2..2b85897f 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -24,6 +24,9 @@ const mockCore = vi.hoisted(() => { .mockImplementation(async (fn: () => Promise) => { await fn(); }), + optimizeFunctionSync: vi.fn().mockImplementation((fn: () => void) => { + fn(); + }), setupCore: vi.fn(), teardownCore: vi.fn(), writeWalltimeResults: vi.fn(), @@ -41,6 +44,8 @@ vi.mock("@codspeed/core", async (importOriginal) => { beforeEach(() => { process.env.CODSPEED_ENV = "true"; process.env.CODSPEED_RUNNER_MODE = "instrumentation"; + // The mocked core only exists in this process, forks would not see it. + process.env.CODSPEED_PROCESS_ISOLATION = "false"; vi.clearAllMocks(); }); @@ -208,6 +213,42 @@ describe("Benchmark.Suite", () => { expect(afterAll).toHaveBeenCalledTimes(2); }); + it("should run each runSync call between its own hooks", () => { + mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); + mockCore.optimizeFunctionSync.mockImplementation((fn: () => void) => { + fn(); + }); + const calls: string[] = []; + const bench = withCodSpeed(new Bench()).add( + "RegExp", + () => { + calls.push("fn"); + }, + { + beforeEach: () => { + calls.push("beforeEach"); + }, + afterEach: () => { + calls.push("afterEach"); + }, + }, + ); + // tinybench calls the function once when it's added + calls.length = 0; + + bench.runSync(); + + // the optimization run, then the measured run + expect(calls).toEqual([ + "beforeEach", + "fn", + "afterEach", + "beforeEach", + "fn", + "afterEach", + ]); + }); + it("should call setupCore and teardownCore only once", async () => { mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); const bench = withCodSpeed(new Bench()) diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index f7ec64cd..c080a7a4 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -1,7 +1,7 @@ import { getV8Flags } from "@codspeed/core"; import { spawn } from "child_process"; import { once } from "events"; -import { mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; import { fileURLToPath } from "url"; @@ -18,12 +18,14 @@ const DEFAULT_FORKS = 3; const MIN_ROUNDS_PER_FORK = 4; /** Fails a run whose benchmark process got stuck instead of blocking the suite. */ const RUN_TIMEOUT_MS = 60_000; -const RAN_LINE = /^ran (\d+) (.+)$/; +const PID_LINE = /^(ran|called) (\d+) (.+)$/; interface FixtureRun { status: number | null; /** PIDs that ran each benchmark. */ pidsByName: Map>; + /** PIDs that called each benchmark's function, to run it or not. */ + callerPidsByName: Map>; /** PID of the process the test started. */ parentPid: number; /** Rounds of each benchmark in the results file. */ @@ -38,6 +40,7 @@ interface FixtureOptions { /** `CODSPEED_WALLTIME_FORKS`. */ forksEnv?: number; sync?: boolean; + mode?: "walltime" | "simulation"; } async function runFixture({ @@ -45,13 +48,14 @@ async function runFixture({ forksOption, forksEnv, sync, + mode = "walltime", }: FixtureOptions = {}): Promise { const profileFolder = mkdtempSync(join(tmpdir(), "codspeed-tinybench-")); try { const env: NodeJS.ProcessEnv = { ...process.env, CODSPEED_ENV: "true", - CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_RUNNER_MODE: mode, CODSPEED_PROFILE_FOLDER: profileFolder, }; delete env[PROCESS_ISOLATION_ENV]; @@ -80,17 +84,22 @@ async function runFixture({ const [status] = (await once(child, "close")) as [number | null]; const pidsByName = new Map>(); + const callerPidsByName = new Map>(); for (const line of stdout.split("\n")) { - const ran = RAN_LINE.exec(line); - if (!ran) continue; - const pids = pidsByName.get(ran[2]) ?? new Set(); - pids.add(Number(ran[1])); - pidsByName.set(ran[2], pids); + const match = PID_LINE.exec(line); + if (!match) continue; + const [, event, pid, name] = match; + const byName = event === "ran" ? pidsByName : callerPidsByName; + const pids = byName.get(name) ?? new Set(); + pids.add(Number(pid)); + byName.set(name, pids); } const roundsByName = new Map(); const resultsDir = join(profileFolder, "results"); - for (const file of readdirSync(resultsDir)) { + // Only walltime runs write results: the instruments hold analysis ones. + const resultFiles = existsSync(resultsDir) ? readdirSync(resultsDir) : []; + for (const file of resultFiles) { const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); for (const benchmark of data.benchmarks) { roundsByName.set(benchmark.name, benchmark.stats.rounds); @@ -99,6 +108,7 @@ async function runFixture({ return { status, pidsByName, + callerPidsByName, parentPid: child.pid!, roundsByName, }; @@ -153,6 +163,19 @@ describe("walltime forks", () => { RUN_TIMEOUT_MS, ); + it( + "calls a benchmark's function only in the forks that measure it", + async () => { + const run = await runFixture(); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(run.callerPidsByName.get(name)).toEqual(forkPids(run, name)); + } + }, + RUN_TIMEOUT_MS, + ); + it( "runs every benchmark in the parent process when isolation is disabled", async () => { @@ -166,3 +189,25 @@ describe("walltime forks", () => { RUN_TIMEOUT_MS, ); }); + +describe("simulation isolation", () => { + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "runs every benchmark in its own process ($name run)", + async ({ sync }) => { + const run = await runFixture({ sync, mode: "simulation" }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(1); + } + const all = new Set( + BENCHMARKS.flatMap((name) => [...forkPids(run, name)]), + ); + expect(all.size).toBe(BENCHMARKS.length); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/__tests__/analysis.test.ts b/packages/vitest-plugin/src/__tests__/analysis.test.ts new file mode 100644 index 00000000..f31981bc --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/analysis.test.ts @@ -0,0 +1,62 @@ +import { spawn } from "child_process"; +import { once } from "events"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { createRequire } from "module"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const FIXTURE_DIR = fileURLToPath( + new URL("../../tests/fixtures/analysis", import.meta.url), +); +const VITEST_BIN = join( + dirname(createRequire(import.meta.url).resolve("vitest/package.json")), + "vitest.mjs", +); +const RUN_TIMEOUT_MS = 180_000; + +let tmpDir: string; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "codspeed-analysis-")); +}); + +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe("simulation", () => { + it( + "awaits a function that returns a promise before its afterEach hook", + async () => { + const logFile = join(tmpDir, "events.log"); + writeFileSync(logFile, ""); + // Not spawnSync: blocking this worker times out its calls to Vitest. + const child = spawn(process.execPath, [VITEST_BIN, "bench", "--run"], { + cwd: FIXTURE_DIR, + env: { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "simulation", + CODSPEED_PROFILE_FOLDER: join(tmpDir, "profile"), + ANALYSIS_LOG_FILE: logFile, + }, + timeout: RUN_TIMEOUT_MS, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr?.on("data", (chunk) => (stderr += chunk)); + const [status] = (await once(child, "close")) as [number | null]; + + expect(status, stderr).toBe(0); + // Every call, the measured one included, finishes its work in between + // its own hooks. + const events = readFileSync(logFile, "utf8").trim().split("\n"); + expect(events.join(" ")).toMatch( + /^beforeEach done afterEach( beforeEach done afterEach)*$/, + ); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts index dae589b5..43be6d84 100644 --- a/packages/vitest-plugin/src/__tests__/isolation.test.ts +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -1,6 +1,7 @@ import { spawn } from "child_process"; import { once } from "events"; import { + existsSync, mkdtempSync, readdirSync, readFileSync, @@ -44,6 +45,7 @@ interface FixtureOptions { pluginForks?: number; firstForks?: number; failing?: boolean; + mode?: "walltime" | "simulation"; } async function runFixture(options: FixtureOptions = {}) { @@ -53,7 +55,7 @@ async function runFixture(options: FixtureOptions = {}) { const env: NodeJS.ProcessEnv = { ...process.env, CODSPEED_ENV: "true", - CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_RUNNER_MODE: options.mode ?? "walltime", CODSPEED_PROFILE_FOLDER: profileFolder, ISOLATION_LOG_FILE: logFile, [PROCESS_ISOLATION_ENV]: options.isolation, @@ -85,7 +87,9 @@ async function runFixture(options: FixtureOptions = {}) { const roundsByName = new Map(); const resultsDir = join(profileFolder, "results"); - for (const file of status === 0 ? readdirSync(resultsDir) : []) { + // Only walltime runs write results: the instruments hold analysis ones. + const hasResults = status === 0 && existsSync(resultsDir); + for (const file of hasResults ? readdirSync(resultsDir) : []) { const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); for (const benchmark of data.benchmarks) { roundsByName.set(benchmark.name, benchmark.stats.rounds); @@ -147,3 +151,20 @@ describe("walltime forks", () => { RUN_TIMEOUT_MS, ); }); + +describe("simulation isolation", () => { + it( + "runs every benchmark in its own process", + async () => { + const run = await runFixture({ mode: "simulation", forksEnv: 2 }); + + expect(run.status, run.stderr).toBe(0); + const pids = ["first", "second", "third"].map((name) => { + expect(run.pidsByName.get(name)?.size).toBe(1); + return [...run.pidsByName.get(name)!][0]; + }); + expect(new Set(pids).size).toBe(pids.length); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/v5/forkReporter.ts b/packages/vitest-plugin/src/v5/forkReporter.ts index 7055eb7b..3009d154 100644 --- a/packages/vitest-plugin/src/v5/forkReporter.ts +++ b/packages/vitest-plugin/src/v5/forkReporter.ts @@ -1,4 +1,5 @@ import { + getInstrumentMode, resolveWalltimeForks, runForks, samplesToResult, @@ -15,7 +16,8 @@ import { withForkContext } from "./forkContext"; /** * Measures the benchmarks the workers queued (see `provider.ts`) in fresh - * processes and reports their pooled rounds. Runs in the main process once the + * processes and reports their pooled walltime rounds; analysis forks report to + * the instruments themselves. Runs in the main process once the * run is over: every file has torn down, so resources its hooks held (ports, * databases, ...) are free for the forks. The global setup is still up (it * tears down on close), so the forks skip it and inherit what it provided. @@ -40,6 +42,11 @@ async function measureInForks( for (const module of modules) { for (const test of module.children.allTests()) { for (const queued of forkedBenchmarks(test.meta())) { + if (getInstrumentMode() !== "walltime") { + // Analysis measures a benchmark once, so it needs only one fork. + await runForks(queued.uri, 1); + continue; + } const forks = resolveWalltimeForks(queued.forks ?? options.forks); const samples = await runForks(queued.uri, forks); const benchmark = tinybenchTaskToBenchmark( diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index 9ce135ad..ee4a6d76 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -4,13 +4,14 @@ import { InstrumentHooks, optimizeFunction, optimizeFunctionSync, + processIsolation, setupCore, teardownCore, - walltimeIsolation, wrapWithRootFrame, wrapWithRootFrameSync, writeForkSamples, type Benchmark, + type IsolationMode, type WalltimeOptions, } from "@codspeed/core"; import { createRequire } from "module"; @@ -149,24 +150,35 @@ const provider: BenchmarkProvider = { const groupUri = buildGroupUri(test); const getUri = (name: string) => `${groupUri}::${name}`; + const mode = processIsolation(); + if (mode.kind === "Forks") { + await queueForks(registrations, options, getUri, test); + return registrations.map(({ name }) => analysisResult(name)); + } if (isWalltime) { - return runWalltime(registrations, options, getUri, test); + return runWalltime(registrations, options, getUri, test, mode); } - return runAnalysis(registrations, getUri); + return runAnalysis(registrations, getUri, mode); }, }; async function runAnalysis( registrations: BenchRegistrationInput[], getUri: (name: string) => string, + mode: IsolationMode, ): Promise { const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; const results: BenchResult[] = []; for (const { name, fn, fnOpts } of registrations) { const uri = getUri(name); + results.push(analysisResult(name)); + // A fork measures only its benchmark; Vitest still gets every result. + if (mode.kind === "Fork" && mode.uri !== uri) continue; await runAnalysisTask(fn, fnOpts, uri); console.log(`[CodSpeed] ${label} ${uri}`); - results.push(analysisResult(name)); + if (mode.kind === "Fork") { + writeForkSamples(mode.samplesFile, []); + } } return results; } @@ -180,8 +192,11 @@ async function runAnalysisTask( fnOpts: TinybenchFnOptions | undefined, uri: string, ): Promise { - if (fn.constructor?.name === "AsyncFunction") { - await fnOpts?.beforeAll?.("run"); + await fnOpts?.beforeAll?.("run"); + if ( + fn.constructor?.name === "AsyncFunction" || + (await returnsPromise(fn, fnOpts)) + ) { await optimizeFunction(async () => { await fnOpts?.beforeEach?.("run"); await fn(); @@ -201,7 +216,6 @@ async function runAnalysisTask( return; } - fnOpts?.beforeAll?.("run"); optimizeFunctionSync(() => { fnOpts?.beforeEach?.("run"); fn(); @@ -220,17 +234,30 @@ async function runAnalysisTask( fnOpts?.afterAll?.("run"); } +/** + * Calls `fn` once between its hooks, like tinybench does for functions that + * aren't declared `async`, to tell whether it must be awaited. + */ +async function returnsPromise( + fn: BenchFn, + fnOpts: TinybenchFnOptions | undefined, +): Promise { + await fnOpts?.beforeEach?.("run"); + const result = fn(); + await result; + await fnOpts?.afterEach?.("run"); + return ( + typeof (result as PromiseLike | undefined)?.then === "function" + ); +} + async function runWalltime( registrations: BenchRegistrationInput[], options: BenchRunOptions | undefined, getUri: (name: string) => string, test: BenchmarkGroup["test"], + mode: IsolationMode, ): Promise { - const mode = walltimeIsolation(); - if (mode.kind === "Forks") { - await queueForks(registrations, options, getUri, test); - return registrations.map(({ name }) => analysisResult(name)); - } // A fork measures only its benchmark; Vitest still gets every result. const measured = registrations.filter( ({ name }) => mode.kind !== "Fork" || mode.uri === getUri(name), @@ -277,7 +304,7 @@ async function runWalltime( /** * Queues every benchmark for measurement in fresh processes once the run is - * over (see `forkReporter.ts`). Vitest gets zeroed results, like in analysis mode. + * over (see `forkReporter.ts`). Vitest gets zeroed results. */ async function queueForks( registrations: BenchRegistrationInput[], diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index d1867bfa..cab84f2f 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -1,7 +1,7 @@ import { getGitDir, getInstrumentMode, - walltimeIsolation, + processIsolation, type WalltimeOptions, } from "@codspeed/core"; import { join } from "path"; @@ -58,9 +58,7 @@ class V5Backend implements VitestBackend { } configureIsolation(userConfig: ViteUserConfig): void { - if (getInstrumentMode() !== "walltime") return; - - const isolationMode = walltimeIsolation(); + const isolationMode = processIsolation(); if (isolationMode.kind !== "Fork") return; restrictToBenchmarkFile(userConfig, isolationMode.uri); // The parent's global setup is still up while its forks run: forks skip @@ -69,16 +67,15 @@ class V5Backend implements VitestBackend { } /** - * The parent of a forked walltime run measures nothing in its workers: they - * queue the benchmarks, and the main process forks them once the run is over. + * The parent of a forked run measures nothing in its workers: they queue the + * benchmarks, and the main process forks them once the run is over. */ configureVitest( project: TestProject, vitest: Vitest, options: WalltimeOptions, ): void { - if (getInstrumentMode() !== "walltime") return; - const isolationMode = walltimeIsolation(); + const isolationMode = processIsolation(); if (isolationMode.kind === "Fork") { provideParentContext(project); return; diff --git a/packages/vitest-plugin/tests/fixtures/analysis/promise.bench.ts b/packages/vitest-plugin/tests/fixtures/analysis/promise.bench.ts new file mode 100644 index 00000000..5cfbce81 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/analysis/promise.bench.ts @@ -0,0 +1,24 @@ +import { appendFileSync } from "fs"; +import { test } from "vitest"; + +// Vitest swallows what workers print, so the order of the benchmark's events +// goes to a file. +function log(event: string) { + appendFileSync(process.env.ANALYSIS_LOG_FILE!, `${event}\n`); +} + +test("benchmarks", async ({ bench }) => { + // Returns a promise without being declared `async`. + await bench( + "returns a promise", + { + beforeEach: () => log("beforeEach"), + afterEach: () => log("afterEach"), + }, + () => { + const { promise, resolve } = Promise.withResolvers(); + setTimeout(resolve, 1); + return promise.then(() => log("done")); + }, + ).run(); +}); diff --git a/packages/vitest-plugin/tests/fixtures/analysis/vitest.config.ts b/packages/vitest-plugin/tests/fixtures/analysis/vitest.config.ts new file mode 100644 index 00000000..df313ac4 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/analysis/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; +import codspeedPlugin from "../../../dist/index.mjs"; + +export default defineConfig({ + plugins: [codspeedPlugin()], + define: { + __VERSION__: JSON.stringify("1.0.0"), + }, + test: { + include: ["**/*.bench.ts"], + }, +});