ci: use staging environment

.github/workflows/ci.yml

@@ -8,7 +8,10 @@ on:
 
 jobs:
   check:
-    runs-on: "ubuntu-latest"
+    strategy:
+      matrix:
+        os: ["ubuntu-latest", "codspeedhq-arm64-ubuntu-22.04"]
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: "actions/checkout@v4"
         with:
@@ -35,7 +38,7 @@ jobs:
       # list the directories in ./examples and output them to a github action workflow variables as a JSON array
       - run: |
           examples=$(find ./examples -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | jq -R -s -c 'split("\n") | map(select(length > 0))')
-          echo "::set-output name=examples::$examples"
+          echo "examples=$examples" >> $GITHUB_OUTPUT
         id: list-examples
 
   node-versions:
@@ -64,6 +67,7 @@ jobs:
         # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
         uses: CodSpeedHQ/action@main
         with:
+          mode: instrumentation
           run: pnpm --filter ${{ matrix.example }} bench-tinybench
         env:
           CODSPEED_SKIP_UPLOAD: true
@@ -72,6 +76,7 @@ jobs:
         # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
         uses: CodSpeedHQ/action@main
         with:
+          mode: instrumentation
           run: pnpm --filter ${{ matrix.example }} bench-benchmark-js
         env:
           CODSPEED_SKIP_UPLOAD: true

.github/workflows/codspeed.yml

@@ -27,8 +27,12 @@ jobs:
         # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
         uses: CodSpeedHQ/action@main
         with:
+          mode: instrumentation
+          upload-url: ${{ secrets.STAGING_CODSPEED_UPLOAD_URL }}
           run: |
-            pnpm moon run --concurrency 1 :bench
+            pnpm moon run tinybench-plugin:bench
+            pnpm moon run vitest-plugin:bench
+            pnpm moon run benchmark.js-plugin:bench
             pnpm --workspace-concurrency 1 -r bench-tinybench
             pnpm --workspace-concurrency 1 -r bench-benchmark-js
             pnpm --workspace-concurrency 1 -r bench-vitest
@@ -53,8 +57,11 @@ jobs:
         # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
         uses: CodSpeedHQ/action@main
         with:
-          # Only tinybench supports walltime for now
+          mode: walltime
+          upload-url: ${{ secrets.STAGING_CODSPEED_UPLOAD_URL }}
           run: |
-            pnpm moon run --concurrency 1 :bench
+            pnpm moon run tinybench-plugin:bench
+            pnpm moon run vitest-plugin:bench
+            pnpm moon run benchmark.js-plugin:bench
             pnpm --workspace-concurrency 1 -r bench-tinybench
             pnpm --workspace-concurrency 1 -r bench-vitest

.github/workflows/release.yml

@@ -10,8 +10,31 @@ permissions:
   contents: write
 
 jobs:
+  build-native-arm:
+    runs-on: codspeedhq-arm64-ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: pnpm/action-setup@v2
+      - uses: actions/setup-node@v3
+        with:
+          cache: pnpm
+          node-version-file: .nvmrc
+      - run: pnpm install --frozen-lockfile --prefer-offline
+      - name: Build native code on ARM
+        run: pnpm moon core:build-native-addon
+      - name: Upload ARM prebuilds
+        uses: actions/upload-artifact@v4
+        with:
+          name: arm-prebuilds
+          path: packages/core/prebuilds
+
   build:
     runs-on: ubuntu-latest
+    needs: build-native-arm
 
     steps:
       - uses: actions/checkout@v4
@@ -28,8 +51,19 @@ jobs:
       - name: Build the libraries
         run: pnpm moon run :build
 
+      - name: Download ARM prebuilds
+        uses: actions/download-artifact@v4
+        with:
+          name: arm-prebuilds
+          path: packages/core/prebuilds
+
       - name: Publish the libraries
-        run: pnpm publish -r --access=public --no-git-checks
+        run: |
+          if [[ "${{ github.ref }}" == *"-alpha"* ]]; then
+            pnpm publish -r --access=public --no-git-checks --tag=alpha
+          else
+            pnpm publish -r --access=public --no-git-checks
+          fi
         env:
           NPM_CONFIG_PROVENANCE: true
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.gitignore

@@ -7,6 +7,9 @@ yarn-error.log*
 lerna-debug.log*
 .pnpm-debug.log*
 
+# JIT dumps
+jit-*.dump
+
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 

lerna.json

@@ -1,7 +1,9 @@
 {
   "npmClient": "pnpm",
   "useWorkspaces": true,
-  "packages": ["packages/*"],
+  "packages": [
+    "packages/*"
+  ],
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "4.0.1"
+  "version": "5.0.0"
 }

packages/benchmark.js-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codspeed/benchmark.js-plugin",
-  "version": "4.0.1",
+  "version": "5.0.0",
   "description": "Benchmark.js compatibility layer for CodSpeed",
   "keywords": [
     "codspeed",
@@ -27,7 +27,7 @@
     "jest-mock-extended": "^3.0.4"
   },
   "dependencies": {
-    "@codspeed/core": "workspace:^4.0.1",
+    "@codspeed/core": "workspace:^5.0.0",
     "lodash": "^4.17.10",
     "stack-trace": "1.0.0-pre2"
   },

packages/core/binding.gyp

@@ -9,6 +9,7 @@
         "-fno-exceptions"
       ],
       "cflags": [
+        "-g",
         "-Wno-maybe-uninitialized",
         "-Wno-unused-variable",
         "-Wno-unused-parameter",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codspeed/core",
-  "version": "4.0.1",
+  "version": "5.0.0",
   "description": "The core Node library used to integrate with Codspeed runners",
   "keywords": [
     "codspeed",

packages/core/src/index.ts

@@ -33,6 +33,12 @@ export function getCodspeedRunnerMode(): CodSpeedRunnerMode {
 }
 
 export const setupCore = () => {
+  if (!native_core.isBound) {
+    throw new Error(
+      "Native core module is not bound, CodSpeed integration will not work properly"
+    );
+  }
+
   native_core.InstrumentHooks.setIntegration("codspeed-node", __VERSION__);
   linuxPerf.start();
   checkV8Flags();

packages/core/src/introspection.ts

@@ -1,28 +1,35 @@
 import { writeFileSync } from "fs";
+import { getCodspeedRunnerMode } from ".";
 
 const CUSTOM_INTROSPECTION_EXIT_CODE = 0;
 
 export const getV8Flags = () => {
   const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]);
+  const codspeedRunnerMode = getCodspeedRunnerMode();
 
-  const flags = [
-    "--hash-seed=1",
-    "--random-seed=1",
-    "--no-opt",
-    "--predictable",
-    "--predictable-gc-schedule",
-    "--interpreted-frames-native-stack",
-    "--allow-natives-syntax",
-    "--expose-gc",
-    "--no-concurrent-sweeping",
-    "--max-old-space-size=4096",
-  ];
-  if (nodeVersionMajor < 18) {
-    flags.push("--no-randomize-hashes");
-  }
-  if (nodeVersionMajor < 20) {
-    flags.push("--no-scavenge-task");
+  const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"];
+
+  if (codspeedRunnerMode === "instrumented") {
+    flags.push(
+      ...[
+        "--hash-seed=1",
+        "--random-seed=1",
+        "--no-opt",
+        "--predictable",
+        "--predictable-gc-schedule",
+        "--expose-gc",
+        "--no-concurrent-sweeping",
+        "--max-old-space-size=4096",
+      ]
+    );
+    if (nodeVersionMajor < 18) {
+      flags.push("--no-randomize-hashes");
+    }
+    if (nodeVersionMajor < 20) {
+      flags.push("--no-scavenge-task");
+    }
   }
+
   return flags;
 };
 

packages/core/src/native_core/index.ts

@@ -1,4 +1,5 @@
 import path from "path";
+import { logDebug } from "../utils";
 import { InstrumentHooks } from "./instruments/hooks";
 import { LinuxPerf } from "./linux_perf/linux_perf";
 interface NativeCore {
@@ -21,6 +22,8 @@ try {
     isBound: true,
   };
 } catch (e) {
+  logDebug("Failed to bind native core, instruments will not work.");
+  logDebug(e);
   native_core = {
     LinuxPerf: class LinuxPerf {
       start() {
@@ -40,12 +43,17 @@ try {
       stopBenchmark: () => {
         return 0;
       },
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
       setExecutedBenchmark: (_pid: number, _uri: string) => {
         return 0;
       },
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
       setIntegration: (_name: string, _version: string) => {
         return 0;
       },
+      __codspeed_root_frame__: <T>(callback: () => T): T => {
+        return callback();
+      },
     },
     isBound: false,
   };

packages/core/src/native_core/instruments/hooks.ts

@@ -31,4 +31,10 @@ export interface InstrumentHooks {
    * @returns 0 on success, non-zero on error
    */
   setIntegration(name: string, version: string): number;
+
+  /**
+   * Execute a callback function with __codspeed_root_frame__ in its stack trace
+   * @param callback Function to execute
+   */
+  __codspeed_root_frame__<T>(callback: () => T): T;
 }

packages/core/src/native_core/instruments/hooks_wrapper.cc

@@ -81,6 +81,27 @@ Napi::Number SetIntegration(const Napi::CallbackInfo &info) {
   return Napi::Number::New(env, result);
 }
 
+Napi::Value __attribute__ ((noinline)) __codspeed_root_frame__(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+
+  if (info.Length() != 1) {
+    Napi::TypeError::New(env, "Expected 1 argument: callback function")
+        .ThrowAsJavaScriptException();
+    return env.Undefined();
+  }
+
+  if (!info[0].IsFunction()) {
+    Napi::TypeError::New(env, "Expected function argument")
+        .ThrowAsJavaScriptException();
+    return env.Undefined();
+  }
+
+  Napi::Function callback = info[0].As<Napi::Function>();
+  Napi::Value result = callback.Call(env.Global(), {});
+
+  return result;
+}
+
 Napi::Object Initialize(Napi::Env env, Napi::Object exports) {
   Napi::Object instrumentHooksObj = Napi::Object::New(env);
 
@@ -96,6 +117,8 @@ Napi::Object Initialize(Napi::Env env, Napi::Object exports) {
                          Napi::Function::New(env, SetExecutedBenchmark));
   instrumentHooksObj.Set(Napi::String::New(env, "setIntegration"),
                          Napi::Function::New(env, SetIntegration));
+  instrumentHooksObj.Set(Napi::String::New(env, "__codspeed_root_frame__"),
+                         Napi::Function::New(env, __codspeed_root_frame__));
 
   exports.Set(Napi::String::New(env, "InstrumentHooks"), instrumentHooksObj);
 

packages/core/src/native_core/linux_perf/linux_perf.cc

@@ -39,4 +39,4 @@ Napi::Value LinuxPerf::Stop(const Napi::CallbackInfo &info) {
   return Napi::Boolean::New(info.Env(), false);
 }
 
-} // namespace codspeed_native
+} // namespace codspeed_native

packages/core/src/walltime/index.ts

@@ -8,19 +8,34 @@ export function getProfileFolder(): string | null {
   return process.env.CODSPEED_PROFILE_FOLDER || null;
 }
 
-export function writeWalltimeResults(benchmarks: Benchmark[]) {
+export function writeWalltimeResults(
+  benchmarks: Benchmark[],
+  asyncWarning = false
+): void {
   const profileFolder = getProfileFolder();
-  let resultPath: string;
-
-  if (profileFolder) {
-    const resultsDir = path.join(profileFolder, "results");
-    fs.mkdirSync(resultsDir, { recursive: true });
-    resultPath = path.join(resultsDir, `${process.pid}.json`);
-  } else {
-    // Fallback: write to .codspeed in current working directory
-    const codspeedDir = path.join(process.cwd(), ".codspeed");
-    fs.mkdirSync(codspeedDir, { recursive: true });
-    resultPath = path.join(codspeedDir, `results_${Date.now()}.json`);
+
+  const resultDir = (() => {
+    if (profileFolder) {
+      return path.join(profileFolder, "results");
+    } else {
+      // Fallback: write to .codspeed in current working directory
+      return path.join(process.cwd(), ".codspeed");
+    }
+  })();
+  fs.mkdirSync(resultDir, { recursive: true });
+  const resultPath = path.join(resultDir, `${process.pid}.json`);
+
+  // Check if file already exists and merge benchmarks
+  let existingBenchmarks: Benchmark[] = [];
+  if (fs.existsSync(resultPath)) {
+    try {
+      const existingData = JSON.parse(
+        fs.readFileSync(resultPath, "utf-8")
+      ) as ResultData;
+      existingBenchmarks = existingData.benchmarks || [];
+    } catch (error) {
+      console.warn(`[CodSpeed] Failed to read existing results file: ${error}`);
+    }
   }
 
   const data: ResultData = {
@@ -30,11 +45,18 @@ export function writeWalltimeResults(benchmarks: Benchmark[]) {
       pid: process.pid,
     },
     instrument: { type: "walltime" },
-    benchmarks: benchmarks,
+    benchmarks: [...existingBenchmarks, ...benchmarks],
+    metadata: asyncWarning
+      ? {
+          async_warning: "Profiling is inaccurate due to async operations",
+        }
+      : undefined,
   };
 
   fs.writeFileSync(resultPath, JSON.stringify(data, null, 2));
-  console.log(`[CodSpeed] Results written to ${resultPath}`);
+  console.log(
+    `[CodSpeed] Results written to ${resultPath} (${data.benchmarks.length} total benchmarks)`
+  );
 }
 
 export * from "./interfaces";