feat: run each analysis benchmark in its own process

packages/core/src/introspection.ts

@@ -33,6 +33,10 @@ export const getV8Flags = () => {
           "--predictable-gc-schedule",
           "--expose-gc",
           "--no-concurrent-sweeping",
+          // --predictable keeps posting incremental marking steps as tasks on
+          // the event loop, so async work before the benchmark (e.g. module
+          // loading) shifts the GC count and heap layout it measures in.
+          "--no-incremental-marking-task",
           "--max-old-space-size=4096",
         ],
       );

packages/core/src/isolation.ts

@@ -1,9 +1,10 @@
 /**
- * Process isolation for walltime runs: every benchmark runs in fresh copies
- * of the benchmark process (forks), so the allocator state, JIT state, and
- * handles that other benchmarks or setup code leave behind can't change its
- * measurement. Rounds within one process are correlated (JIT tier-up, heap
- * sizing), so the rounds of several forks are pooled into one result.
+ * Process isolation: every benchmark runs in fresh copies of the benchmark
+ * process (forks), so the allocator state, JIT state, and handles that other
+ * benchmarks or setup code leave behind can't change its measurement. Walltime
+ * rounds within one process are correlated (JIT tier-up, heap sizing), so the
+ * rounds of several forks are pooled into one result. Analysis runs measure
+ * each benchmark once, in one fork.
  */
 import { spawn, spawnSync } from "child_process";
 import { randomUUID } from "crypto";
@@ -12,6 +13,7 @@ import { tmpdir } from "os";
 import { join } from "path";
 import { deserialize, serialize } from "v8";
 import { constants, gunzipSync, gzipSync } from "zlib";
+import { getInstrumentMode } from "./runnerMode";
 
 /** Accepts `0`, `false`, `1`, or `true`; defaults to enabled. */
 export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION";
@@ -44,10 +46,11 @@ export type IsolationMode =
   | { kind: "Fork"; uri: string; samplesFile: string };
 
 /**
- * The isolation of a walltime run, from the environment. `forks` is the
- * integration's option, which {@link WALLTIME_FORKS_ENV} overrides.
+ * The isolation of this run, from the environment. A walltime benchmark runs
+ * in `forks` forks, the integration's option, which {@link WALLTIME_FORKS_ENV}
+ * overrides; an analysis benchmark is measured once, so it runs in one.
  */
-export function walltimeIsolation(forks?: number): IsolationMode {
+export function processIsolation(forks?: number): IsolationMode {
   const uri = process.env[ISOLATED_BENCHMARK_ENV];
   if (uri !== undefined) {
     const samplesFile = process.env[FORK_SAMPLES_FILE_ENV];
@@ -67,7 +70,8 @@ export function walltimeIsolation(forks?: number): IsolationMode {
       `[CodSpeed] ${PROCESS_ISOLATION_ENV} must be 0, false, 1, or true, got \`${isolation}\``,
     );
   }
-  return { kind: "Forks", forks: resolveWalltimeForks(forks) };
+  const analysis = getInstrumentMode() === "analysis";
+  return { kind: "Forks", forks: analysis ? 1 : resolveWalltimeForks(forks) };
 }
 
 /**
@@ -221,7 +225,11 @@ function forkSamples(
   }
 }
 
-/** Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. */
+/**
+ * Hands round samples (milliseconds) to the parent as gzip-compressed V8 data.
+ * Analysis forks hand none: the instruments hold their measurement, and the
+ * file only tells the parent that they measured their benchmark.
+ */
 export function writeForkSamples(
   samplesFile: string,
   samples: ArrayLike<number>,

packages/core/tests/fixtures/isolation.cjs

@@ -1,12 +1,12 @@
 // Run by `isolation.integ.test.ts`. Measures benchmark "a" in forks.
 const {
+  processIsolation,
   runForks,
   runForksSync,
-  walltimeIsolation,
   writeForkSamples,
 } = require("../../dist/index.cjs.js");
 
-const mode = walltimeIsolation();
+const mode = processIsolation();
 
 function measure() {
   console.log(`ran ${process.pid} ${JSON.stringify(process.execArgv)}`);

packages/core/tests/isolation.integ.test.ts

@@ -4,9 +4,9 @@ import {
   FORK_SAMPLES_FILE_ENV,
   ISOLATED_BENCHMARK_ENV,
   PROCESS_ISOLATION_ENV,
+  processIsolation,
   resolveWalltimeForks,
   WALLTIME_FORKS_ENV,
-  walltimeIsolation,
 } from "../src/isolation";
 
 const FIXTURE = join(__dirname, "fixtures", "isolation.cjs");
@@ -21,6 +21,8 @@ const NODE_FLAG = "--no-warnings";
 const SAMPLES_PER_FORK = 2;
 
 const ENV_KEYS = [
+  "CODSPEED_ENV",
+  "CODSPEED_RUNNER_MODE",
   PROCESS_ISOLATION_ENV,
   ISOLATED_BENCHMARK_ENV,
   FORK_SAMPLES_FILE_ENV,
@@ -75,7 +77,7 @@ describe("resolveWalltimeForks", () => {
   });
 });
 
-describe("walltimeIsolation", () => {
+describe("processIsolation", () => {
   it.each([
     [undefined, "Forks"],
     ["1", "Forks"],
@@ -84,12 +86,12 @@ describe("walltimeIsolation", () => {
     ["false", "SharedProcess"],
   ])("selects %s as %s", (value, kind) => {
     setEnv({ [PROCESS_ISOLATION_ENV]: value });
-    expect(walltimeIsolation().kind).toBe(kind);
+    expect(processIsolation().kind).toBe(kind);
   });
 
   it("runs everything in this process when isolation is off, whatever the fork count", () => {
     setEnv({ [PROCESS_ISOLATION_ENV]: "false", [WALLTIME_FORKS_ENV]: "4" });
-    expect(walltimeIsolation(2).kind).toBe("SharedProcess");
+    expect(processIsolation(2).kind).toBe("SharedProcess");
   });
 
   it("restricts a fork to its benchmark, whatever the isolation setting", () => {
@@ -98,12 +100,21 @@ describe("walltimeIsolation", () => {
       [ISOLATED_BENCHMARK_ENV]: "file::bench",
       [FORK_SAMPLES_FILE_ENV]: "samples.json",
     });
-    expect(walltimeIsolation()).toEqual({
+    expect(processIsolation()).toEqual({
       kind: "Fork",
       uri: "file::bench",
       samplesFile: "samples.json",
     });
   });
+
+  it("runs an analysis benchmark in one fork, whatever the walltime fork count", () => {
+    setEnv({
+      CODSPEED_ENV: "true",
+      CODSPEED_RUNNER_MODE: "simulation",
+      [WALLTIME_FORKS_ENV]: "4",
+    });
+    expect(processIsolation(2)).toEqual({ kind: "Forks", forks: 1 });
+  });
 });
 
 interface FixtureOptions {

packages/tinybench-plugin/src/analysis.ts

@@ -6,15 +6,18 @@ import {
   optimizeFunctionSync,
   wrapWithRootFrame,
   wrapWithRootFrameSync,
+  writeForkSamples,
+  type IsolationMode,
 } from "@codspeed/core";
 import { Bench, Task } from "tinybench";
 import { BaseBenchRunner } from "./shared";
 
 export function setupCodspeedAnalysisBench(
   bench: Bench,
   rootCallingFile: string,
+  isolationMode: IsolationMode,
 ): void {
-  const runner = new AnalysisBenchRunner(bench, rootCallingFile);
+  const runner = new AnalysisBenchRunner(bench, rootCallingFile, isolationMode);
   runner.setupBenchMethods();
 }
 
@@ -28,6 +31,16 @@ class AnalysisBenchRunner extends BaseBenchRunner {
     return InstrumentHooks.isInstrumented() ? "Measured" : "Checked";
   }
 
+  private completeTask(uri: string): void {
+    this.logTaskCompletion(uri, this.taskCompletionMessage());
+    if (this.isolationMode.kind === "Fork") {
+      writeForkSamples(this.isolationMode.samplesFile, []);
+    }
+  }
+
+  /** The instruments hold the measurement of the forks. */
+  protected collectForks(): void {}
+
   protected async runTaskAsync(task: Task, uri: string): Promise<void> {
     const { fnOpts, fn } = this.getTaskData(task);
 
@@ -54,17 +67,17 @@ class AnalysisBenchRunner extends BaseBenchRunner {
     await fnOpts?.afterEach?.call(task, "run");
     await fnOpts?.afterAll?.call(task, "run");
 
-    this.logTaskCompletion(uri, this.taskCompletionMessage());
+    this.completeTask(uri);
   }
 
   protected runTaskSync(task: Task, uri: string): void {
     const { fnOpts, fn } = this.getTaskData(task);
 
     fnOpts?.beforeAll?.call(task, "run");
 
-    optimizeFunctionSync(async () => {
+    optimizeFunctionSync(() => {
       fnOpts?.beforeEach?.call(task, "run");
-      await fn();
+      fn();
       fnOpts?.afterEach?.call(task, "run");
     });
 
@@ -83,7 +96,7 @@ class AnalysisBenchRunner extends BaseBenchRunner {
     fnOpts?.afterEach?.call(task, "run");
     fnOpts?.afterAll?.call(task, "run");
 
-    this.logTaskCompletion(uri, this.taskCompletionMessage());
+    this.completeTask(uri);
   }
 
   protected finalizeAsyncRun(): Task[] {

packages/tinybench-plugin/src/index.ts

@@ -4,17 +4,18 @@ import {
   getInstrumentMode,
   InstrumentHooks,
   mongoMeasurement,
+  processIsolation,
   SetupInstrumentsRequestBody,
   SetupInstrumentsResponse,
   tryIntrospect,
   wrapWithRootFrameSync,
   type WalltimeOptions,
 } from "@codspeed/core";
-import { Bench } from "tinybench";
+import { Bench, type FnOptions } from "tinybench";
 import { setupCodspeedAnalysisBench } from "./analysis";
 import { getOrCreateTaskDataMap } from "./taskData";
 import { getOrCreateUriMap } from "./uri";
-import { setupCodspeedWalltimeBench } from "./walltime";
+import { runsTaskFn, setupCodspeedWalltimeBench } from "./walltime";
 
 tryIntrospect();
 
@@ -29,6 +30,7 @@ export function withCodSpeed(
 
   const rootCallingFile = getCallingFile(1);
   const instrumentMode = getInstrumentMode();
+  const isolationMode = processIsolation(options.forks);
 
   // Compute and register URI for bench
   const uriMap = getOrCreateUriMap(bench);
@@ -43,25 +45,40 @@ export function withCodSpeed(
     uri += `::${name}`;
     uriMap.set(name, uri);
     taskDataMap.set(name, { fn, fnOpts: opts });
-    // In walltime mode the task is driven by tinybench's own measured loop, so
-    // the root frame must be baked into the function tinybench stores rather
-    // than injected later (the function is an inaccessible private field on the
-    // task from tinybench v6 onwards). The sync wrapper is transparent to the
-    // function's return value, so it preserves tinybench's sync/async handling.
-    const registeredFn =
-      instrumentMode === "walltime" ? wrapWithRootFrameSync(fn) : fn;
-    return rawAdd.bind(bench)(name, registeredFn, opts);
+    const add = rawAdd.bind(bench);
+    // tinybench calls a function that isn't `async` once when it's added, to
+    // tell whether it returns a promise. A process that doesn't run the task
+    // registers a stub instead, so the user's function isn't called there.
+    if (!runsTaskFn(isolationMode, uri)) {
+      return add(name, notRunHere, opts);
+    }
+    // The analysis runner calls the captured function itself.
+    if (instrumentMode === "analysis") {
+      return add(name, fn, opts);
+    }
+    // The task is driven by tinybench's own measured loop, so the root frame
+    // must be baked into the function tinybench stores rather than injected
+    // later (the function is an inaccessible private field on the task from
+    // tinybench v6 onwards). An async wrapper would add an await to every
+    // iteration, so the wrapper stays sync and tinybench is told that an async
+    // function is async instead (honored from tinybench v6 onwards).
+    const fnOpts: (FnOptions & { async?: boolean }) | undefined =
+      fn.constructor.name === "AsyncFunction" ? { ...opts, async: true } : opts;
+    return add(name, wrapWithRootFrameSync(fn), fnOpts);
   };
 
   if (instrumentMode === "analysis") {
-    setupCodspeedAnalysisBench(bench, rootCallingFile);
-  } else if (instrumentMode === "walltime") {
-    setupCodspeedWalltimeBench(bench, rootCallingFile, options);
+    setupCodspeedAnalysisBench(bench, rootCallingFile, isolationMode);
+  } else {
+    setupCodspeedWalltimeBench(bench, rootCallingFile, isolationMode);
   }
 
   return bench;
 }
 
+/** Registered for tasks this process never runs; `async` so tinybench doesn't call it. */
+async function notRunHere(): Promise<void> {}
+
 /**
  * Dynamically setup the CodSpeed instruments.
  */

packages/tinybench-plugin/src/shared.ts

@@ -1,4 +1,10 @@
-import { setupCore, teardownCore, type IsolationMode } from "@codspeed/core";
+import {
+  runForks,
+  runForksSync,
+  setupCore,
+  teardownCore,
+  type IsolationMode,
+} from "@codspeed/core";
 import { Bench, Task } from "tinybench";
 import { CapturedTaskData, getTaskData } from "./taskData";
 import { getTaskUri } from "./uri";
@@ -12,8 +18,7 @@ export abstract class BaseBenchRunner {
   constructor(
     bench: Bench,
     rootCallingFile: string,
-    /** Only walltime runs are isolated. */
-    protected readonly isolationMode: IsolationMode = { kind: "SharedProcess" },
+    protected readonly isolationMode: IsolationMode,
   ) {
     this.bench = bench;
     this.rootCallingFile = rootCallingFile;
@@ -72,6 +77,7 @@ export abstract class BaseBenchRunner {
   protected abstract runTaskSync(task: Task, uri: string): void;
   protected abstract finalizeAsyncRun(): Task[];
   protected abstract finalizeSyncRun(): Task[];
+  protected abstract collectForks(task: Task, samples: Float64Array): void;
 
   public setupBenchMethods(): void {
     this.bench.run = async () => {
@@ -85,7 +91,12 @@ export abstract class BaseBenchRunner {
 
       for (const task of tasks) {
         const uri = this.getTaskUri(task);
-        await this.runTaskAsync(task, uri);
+        if (this.isolationMode.kind === "Forks") {
+          const samples = await runForks(uri, this.isolationMode.forks);
+          this.collectForks(task, samples);
+        } else {
+          await this.runTaskAsync(task, uri);
+        }
       }
 
       return this.finalizeAsyncRun();
@@ -100,7 +111,12 @@ export abstract class BaseBenchRunner {
 
       for (const task of tasks) {
         const uri = this.getTaskUri(task);
-        this.runTaskSync(task, uri);
+        if (this.isolationMode.kind === "Forks") {
+          const samples = runForksSync(uri, this.isolationMode.forks);
+          this.collectForks(task, samples);
+        } else {
+          this.runTaskSync(task, uri);
+        }
       }
 
       return this.finalizeSyncRun();