feat(binaries): precompile binaries

.github/workflows/build.yml

@@ -11,30 +11,94 @@ on:
         description: If the commit you want to test isn't the head of a branch, provide its SHA here
         required: false
 jobs:
-  artifacts:
-    name: Upload Artifacts
-    runs-on: ubuntu-latest
+  precompile-bindings:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-2019, ubuntu-latest]
+        node-version: [12.x, 14.x, 16.x, 18.x]
+        target: [x64]
+        host: [x64]
+        # arm64 is not supported by gh runners yet
+        # include: 
+        #   - os: macos-latest
+        #     node-version: 18.x
+        #     target: arm64
+        #     host: arm64
+
+    runs-on: ${{ matrix.os }}
     # Build artifacts are only needed for releasing workflow.
     if: startsWith(github.ref, 'refs/heads/release/')
+    name: ${{ matrix.os }} (node=${{ matrix.node-version }}, host=${{ matrix.host }}, target=${{ matrix.target }})
     steps:
       - name: Check out current commit
         uses: actions/checkout@v3
         with:
           ref: ${{ env.HEAD_COMMIT }}
+
       - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          architecture: ${{ matrix.host }}
+
+      - name: Setup windows build tools
+        uses: microsoft/setup-msbuild@v1.1
+        if: ${{ runner.os == 'Windows' }}
+        with:
+          msbuild-architecture: ${{ matrix.target }}
+
       - name: Install dependencies
         run: npm install
-      - name: Lint,
-        run: npm run lint
+
+      - name: Configure gyp
+        run: npm run build:configure --arch=${{ matrix.target }}
+
+      - name: Build bindings
+        run: npm run build:bindings --arch=${{ matrix.target }}
+
       - name: Test
         run: npm run test
-      - name: build
-        run: npm run build:lib
-      - name: pack
-        run: npm pack
+
+      - name: Copy Binary
+        run: node scripts/copy-target.js
+
+      - name: Log binary
+        run: du -sh binaries/*
+
+      - name: Archive binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: binaries-${{ github.sha }}
+          path: |
+            ${{ github.workspace }}/binaries/
+
+  build-and-pack:
+    runs-on: ubuntu-latest
+    needs: [precompile-bindings]
+    # Build artifacts are only needed for releasing workflow.
+    if: startsWith(github.ref, 'refs/heads/release/')
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 18
+
+      - run: npm ci
+
+      - run: npm run build:lib
+
+      - uses: actions/download-artifact@v3
+        with: 
+          name: binaries-${{ github.sha }}
+          path: binaries
+      - run: ls -l binaries
+      - run: npm pack
+
       - name: Archive artifacts
         uses: actions/upload-artifact@v3
         with:
           name: ${{ github.sha }}
           path: |
             ${{ github.workspace }}/*.tgz
+

.github/workflows/ci.yml

@@ -28,6 +28,12 @@ jobs:
           node-version: ${{ matrix.node-version }}
       - run: npm i -g npm
       - run: npm run clean
+
+      - name: Setup windows build tools
+        uses: microsoft/setup-msbuild@v1.1
+        if: ${{ runner.os == 'Windows' }}
+
       - run: npm install
+      - run: npm run build:configure
       - run: npm run build
       - run: npm run test

.gitignore

@@ -6,6 +6,7 @@
 
 coverage/
 lib/
+binaries/
 
 node_modules/
 memory_db

benchmarks/format/benchmark.format.js

@@ -6,7 +6,7 @@ const gzip = require('zlib');
 const { ZSTDCompress } = require('simple-zstd');
 
 const { mean, stdev, variancepct, variance, quantile } = require('./../cpu/utils');
-const cpu_profiler = require('../../build/Release/cpu_profiler.node');
+const cpu_profiler = require('../../build/Release/sentry_cpu_profiler.node');
 const { threadId } = require('worker_threads');
 
 const relativeChange = (final, initial) => {

benchmarks/memory/benchmark.memory.js

@@ -1,6 +1,6 @@
 // Run with
 // node --expose-gc inspect benchmarks/memory/benchmark.memory.js to attach a debugger and take heap snapshots
-const { CpuProfilerBindings } = require('../../lib/cpu_profiler.js');
+const { CpuProfilerBindings } = require('../../lib/sentry_cpu_profiler.js');
 
 function iterateOverLargeHashTable() {
   const table = {};

binding.gyp

@@ -1,7 +1,7 @@
 {
   "targets": [
     {
-      "target_name": "cpu_profiler",
+      "target_name": "sentry_cpu_profiler",
       "sources": [ "bindings/cpu_profiler.cc" ],
       "defines": ["PROFILER_FORMAT=FORMAT_SAMPLED"],
       'include_dirs': [

package.json

@@ -18,14 +18,21 @@
     "LICENSE",
     "README.md",
     "package.json",
-    "package-lock.json"
+    "package-lock.json",
+    "binaries",
+    "scripts/binaries.js",
+    "scripts/check-build.js",
+    "scripts/copy-target.js",
+    "scripts/empty.js"
   ],
   "scripts": {
-    "postinstall": "npm run build:bindings",
+    "install": "node scripts/empty.js",
+    "postinstall": "node scripts/check-build.js",
     "clean": "rm -rf ./lib && rm -rf build",
     "lint": "eslint ./src --ext .ts",
     "build": "npm run build:bindings && npm run build:lib",
     "build:lib": "tsc",
+    "build:configure": "node-gyp configure",
     "build:bindings": "node-gyp build",
     "build:benchmark:format": "node-gyp -DFORMAT_BENCHMARK=1 build",
     "benchmark": "npm run benchmark:methods && npm run benchmark:profiler && npm run benchmark:server && npm run benchmark:format",
@@ -54,12 +61,14 @@
     "@sentry/types": "^7.16.0",
     "@sentry/utils": "^7.16.0",
     "nan": "^2.17.0",
+    "node-abi": "^3.28.0",
     "node-gyp": "^9.3.0"
   },
   "devDependencies": {
     "@types/express": "^4.17.14",
     "@types/jest": "^29.0.0",
     "@types/node": "^18.0.2",
+    "@types/node-abi": "^3.0.0",
     "@typescript-eslint/eslint-plugin": "^5.36.2",
     "@typescript-eslint/parser": "^5.36.2",
     "autocannon": "^7.9.0",

scripts/binaries.js

@@ -0,0 +1,14 @@
+/* eslint-env node */
+const os = require('os');
+const path = require('path');
+const abi = require('node-abi');
+
+function getModuleName() {
+  return `sentry_cpu_profiler-v${abi.getAbi(process.versions.node, 'node')}-${os.platform()}-${os.arch()}.node`;
+}
+
+const source = path.join(__dirname, '..', 'build', 'Release', 'sentry_cpu_profiler.node');
+const target = path.join(__dirname, '..', 'binaries', getModuleName());
+
+module.exports.target = target;
+module.exports.source = source;

scripts/check-build.js

@@ -0,0 +1,44 @@
+/* eslint-env node */
+const cp = require('child_process');
+const os = require('os');
+
+const { target } = require('./binaries');
+
+function recompileFromSource() {
+  try {
+    console.log('@sentry/profiling-node: Precompiled binary not found, compiling from source...');
+    cp.execSync(`npm run build:configure --arch=${os.arch()}`);
+    cp.execSync(`npm run build:bindings`);
+    cp.execSync('node scripts/copy-target.js');
+    return true;
+  } catch (e) {
+    console.error(
+      '@sentry/profiling-node: Failed to build from source, please report this a bug at https://github.com/getsentry/profiling-node/issues/new?assignees=&labels=Type%3A+Bug&template=bug.yml'
+    );
+    return false;
+  }
+}
+
+try {
+  require(target);
+  console.log('@sentry/profiling-node: Precompiled binary found, skipping build from source.');
+} catch (e) {
+  // Check for node version missmatch
+  if (/was compiled against a different Node.js/.test(e.message)) {
+    const success = recompileFromSource();
+    if (success) {
+      process.exit(0);
+    }
+  }
+  // Not sure if this could even happen, but just in case it somehow does,
+  //  we can provide a better experience than just crashing with cannot find module message.
+  if (/Cannot find module/.test(e.message)) {
+    const success = recompileFromSource();
+    if (success) {
+      process.exit(0);
+    }
+  }
+
+  // re-throw so we dont end up swallowing errors
+  throw e;
+}

scripts/copy-target.js

@@ -0,0 +1,17 @@
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const abi = require('node-abi');
+
+const binaries = path.resolve(__dirname, '..', 'binaries');
+if (!fs.existsSync(binaries)) {
+  fs.mkdirSync(binaries);
+}
+
+const moduleName = `sentry_cpu_profiler-v${abi.getAbi()}-${os.platform()}-${os.arch()}.node`;
+
+const source = path.join(__dirname, '..', 'build', 'Release', 'sentry_cpu_profiler.node');
+const target = path.join(__dirname, '..', 'binaries', moduleName);
+
+fs.renameSync(source, target);

scripts/empty.js

@@ -0,0 +1,4 @@
+/* eslint-env node */
+// This does nothing, but it's a valid script and we npm install
+// to point at something so that it doesnt run gyp build by default
+process.exit(0);

src/cpu_profiler.test.ts

@@ -102,10 +102,10 @@ describe('Profiler bindings', () => {
 
     if (!profile) fail('Profile is null');
 
-    // Exception for macos - we seem to get way less samples there, but I'm not sure if that's due to poor
+    // Exception for macos and windows - we seem to get way less samples there, but I'm not sure if that's due to poor
     // performance of the actions runner, machine or something else. This needs more investigation to determine
     // the cause of low sample count. https://github.com/actions/runner-images/issues/1336 seems relevant.
-    if (process.platform === 'darwin') {
+    if (process.platform === 'darwin' || process.platform === 'win32') {
       if (profile.samples.length < 2) {
         fail('Only ' + profile.samples.length + ' samples obtained on ' + process.platform + ', expected at least 2');
       }

src/cpu_profiler.ts

@@ -1,7 +1,13 @@
-// @ts-expect-error this screams because it cannot resolve the module?
-import profiler from './../build/Release/cpu_profiler.node';
+import os from 'os';
+import path from 'path';
+import abi from 'node-abi';
 import { threadId } from 'worker_threads';
 
+export function importCppBindingsModule(): PrivateV8CpuProfilerBindings {
+  const name = `sentry_cpu_profiler-v${abi.getAbi(process.versions.node, 'node')}-${os.platform()}-${os.arch()}.node`;
+  return require(path.join(__dirname, '..', 'binaries', name));
+}
+
 interface Sample {
   stack_id: number;
   thread_id: string;
@@ -44,7 +50,7 @@ interface V8CpuProfilerBindings {
   stopProfiling(name: string): RawThreadCpuProfile | null;
 }
 
-const privateBindings: PrivateV8CpuProfilerBindings = profiler;
+const privateBindings: PrivateV8CpuProfilerBindings = importCppBindingsModule();
 const CpuProfilerBindings: V8CpuProfilerBindings = {
   startProfiling(name: string) {
     return privateBindings.startProfiling(name);