tools: cache V8 build on test-shared workflow

.github/workflows/test-shared.yml

@@ -182,25 +182,20 @@ jobs:
             core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
             core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
             core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
-
-      - name: Load shell.nix
-        if: github.event_name != 'workflow_dispatch'
-        run: |
-          mv "$TAR_DIR"/*.nix .
-          mkdir tools
-          mv "$TAR_DIR"/tools/nix tools/.
+            core.exportVariable('NIX_SCCACHE', '(import <nixpkgs> {}).sccache');
 
       - name: Build Node.js and run tests
         run: |
           nix-shell \
-            -I nixpkgs=./tools/nix/pkgs.nix \
+            -I "nixpkgs=$TAR_DIR/tools/nix/pkgs.nix" \
             --pure --keep TAR_DIR --keep FLAKY_TESTS \
             --keep SCCACHE_GHA_ENABLED --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
             --arg loadJSBuiltinsDynamically false \
-            --arg ccache '(import <nixpkgs> {}).sccache' \
+            --arg useSeparateDerivationForV8 true \
+            --arg ccache "${NIX_SCCACHE:-null}" \
             --arg devTools '[]' \
             --arg benchmarkTools '[]' \
             ${{ endsWith(matrix.system, '-darwin') && '--arg withAmaro false --arg withLief false --arg withSQLite false --arg extraConfigFlags ''["--without-inspector" "--without-node-options"]'' \' || '\' }}
             --run '
                 make -C "$TAR_DIR" run-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS"
-            '
+            ' "$TAR_DIR/shell.nix"

configure.py

@@ -2064,10 +2064,16 @@ def configure_v8(o, configs):
   o['variables']['node_enable_v8windbg'] = b(options.enable_v8windbg)
   if options.enable_d8:
     o['variables']['test_isolation_mode'] = 'noop'  # Needed by d8.gyp.
-  if options.without_bundled_v8 and options.enable_d8:
-    raise Exception('--enable-d8 is incompatible with --without-bundled-v8.')
-  if options.without_bundled_v8 and options.enable_v8windbg:
-    raise Exception('--enable-v8windbg is incompatible with --without-bundled-v8.')
+  if options.without_bundled_v8:
+    if options.enable_d8:
+      raise Exception('--enable-d8 is incompatible with --without-bundled-v8.')
+    if options.enable_v8windbg:
+      raise Exception('--enable-v8windbg is incompatible with --without-bundled-v8.')
+    (pkg_libs, pkg_cflags, pkg_libpath, _) = pkg_config("v8")
+    if pkg_libs and pkg_libpath:
+      output['libraries'] += [pkg_libpath] + pkg_libs.split()
+    if pkg_cflags:
+      output['include_dirs'] += [flag for flag in [flag.strip() for flag in pkg_cflags.split('-I')] if flag]
   if options.static_zoslib_gyp:
     o['variables']['static_zoslib_gyp'] = options.static_zoslib_gyp
   if flavor != 'linux' and options.v8_enable_hugepage:

node.gyp

@@ -641,7 +641,6 @@
           'xcode_settings': {
             'OTHER_LDFLAGS': [
               '-Wl,-force_load,<(PRODUCT_DIR)/<(STATIC_LIB_PREFIX)<(node_core_target_name)<(STATIC_LIB_SUFFIX)',
-              '-Wl,-force_load,<(PRODUCT_DIR)/<(STATIC_LIB_PREFIX)v8_base_without_compiler<(STATIC_LIB_SUFFIX)',
             ],
           },
           'msvs_settings': {
@@ -653,14 +652,28 @@
             },
           },
           'conditions': [
-            ['OS != "aix" and OS != "os400" and OS != "mac" and OS != "ios"', {
+            ['node_use_bundled_v8=="true"', {
+              'xcode_settings': {
+                'OTHER_LDFLAGS': [
+                  '-Wl,-force_load,<(PRODUCT_DIR)/<(STATIC_LIB_PREFIX)v8_base_without_compiler<(STATIC_LIB_SUFFIX)',
+                ],
+              },
+            }],
+            ['node_use_bundled_v8=="true" and OS != "aix" and OS != "os400" and OS != "mac" and OS != "ios"', {
               'ldflags': [
                 '-Wl,--whole-archive',
                 '<(obj_dir)/<(STATIC_LIB_PREFIX)<(node_core_target_name)<(STATIC_LIB_SUFFIX)',
                 '<(obj_dir)/tools/v8_gypfiles/<(STATIC_LIB_PREFIX)v8_base_without_compiler<(STATIC_LIB_SUFFIX)',
                 '-Wl,--no-whole-archive',
               ],
             }],
+            ['node_use_bundled_v8!="true" and OS != "aix" and OS != "os400" and OS != "mac" and OS != "ios"', {
+              'ldflags': [
+                '-Wl,--whole-archive',
+                '<(obj_dir)/<(STATIC_LIB_PREFIX)<(node_core_target_name)<(STATIC_LIB_SUFFIX)',
+                '-Wl,--no-whole-archive',
+              ],
+            }],
             [ 'OS=="win"', {
               'sources': [ 'src/res/node.rc' ],
             }],
@@ -1530,6 +1543,9 @@
         'src/builtin_info.cc',
       ],
       'conditions': [
+        [ 'OS=="mac"', {
+          'libraries': [ '-framework CoreFoundation -framework Security' ],
+        }],
         [ 'node_shared_simdutf=="false"', {
           'dependencies': [ 'tools/v8_gypfiles/v8.gyp:simdutf#host' ],
         }],

node.gypi

@@ -317,8 +317,8 @@
         'NODE_PLATFORM="sunos"',
       ],
     }],
-    [ '(OS=="freebsd" or OS=="linux" or OS=="openharmony") and node_shared=="false"'
-        ' and force_load=="true"', {
+    [ 'node_use_bundled_v8=="true" and (OS=="freebsd" or OS=="linux" or OS=="openharmony") '
+        'and node_shared=="false" and force_load=="true"', {
       'ldflags': [
         '-Wl,-z,noexecstack',
         '-Wl,--whole-archive <(v8_base)',

shell.nix

@@ -8,6 +8,7 @@
   extraConfigFlags ? [
     "--debug-node"
   ],
+  useSeparateDerivationForV8 ? false, # to help CI better managed its binary cache, not recommended outside of CI usage.
 
   # Build options
   icu ? pkgs.icu,
@@ -17,16 +18,21 @@
   withSQLite ? true,
   withSSL ? true,
   withTemporal ? false,
-  sharedLibDeps ? import ./tools/nix/sharedLibDeps.nix {
-    inherit
-      pkgs
-      withLief
-      withQuic
-      withSQLite
-      withSSL
-      withTemporal
-      ;
-  },
+  sharedLibDeps ?
+    let
+      d = import ./tools/nix/sharedLibDeps.nix {
+        inherit
+          pkgs
+          withLief
+          withQuic
+          withSQLite
+          withSSL
+          withTemporal
+          ;
+      };
+    in
+    # To avoid conflicts with V8's bundled simdutf lib, it's easier to remove it when using a precompiled V8.
+    if (useSeparateDerivationForV8 != false) then builtins.removeAttrs d [ "simdutf" ] else d,
 
   # dev tools (not needed to build Node.js, useful to maintain it)
   ncu-path ? null, # Provide this if you want to use a local version of NCU
@@ -40,11 +46,49 @@ let
   useSharedOpenSSL = builtins.hasAttr "openssl" sharedLibDeps;
 
   needsRustCompiler = withTemporal && !builtins.hasAttr "temporal_capi" sharedLibDeps;
+
+  buildInputs = builtins.attrValues sharedLibDeps ++ pkgs.lib.optional useSharedICU icu;
+  configureFlags = [
+    (
+      if icu == null then
+        "--without-intl"
+      else
+        "--with-intl=${if useSharedICU then "system" else icu}-icu"
+    )
+  ]
+  ++ extraConfigFlags
+  ++ pkgs.lib.optional (!withAmaro) "--without-amaro"
+  ++ pkgs.lib.optional (!withLief) "--without-lief"
+  ++ pkgs.lib.optional withQuic "--experimental-quic"
+  ++ pkgs.lib.optional (!withSQLite) "--without-sqlite"
+  ++ pkgs.lib.optional (!withSSL) "--without-ssl"
+  ++ pkgs.lib.optional withTemporal "--v8-enable-temporal-support"
+  ++ pkgs.lib.optional (ninja != null) "--ninja"
+  ++ pkgs.lib.optional loadJSBuiltinsDynamically "--node-builtin-modules-path=${builtins.toString ../..}"
+  ++ pkgs.lib.concatMap (name: [
+    "--shared-${name}"
+    "--shared-${name}-libpath=${pkgs.lib.getLib sharedLibDeps.${name}}/lib"
+    "--shared-${name}-include=${pkgs.lib.getInclude sharedLibDeps.${name}}/include"
+  ]) (builtins.attrNames sharedLibDeps);
 in
 pkgs.mkShell {
   inherit (pkgs.nodejs_latest) nativeBuildInputs;
 
-  buildInputs = builtins.attrValues sharedLibDeps ++ pkgs.lib.optional useSharedICU icu;
+  buildInputs =
+    buildInputs
+    ++ pkgs.lib.optional (useSeparateDerivationForV8 != false) (
+      if useSeparateDerivationForV8 == true then
+        import ./tools/nix/v8.nix {
+          inherit
+            pkgs
+            configureFlags
+            buildInputs
+            needsRustCompiler
+            ;
+        }
+      else
+        useSeparateDerivationForV8
+    );
 
   packages =
     pkgs.lib.optional (ccache != null) ccache
@@ -74,28 +118,7 @@ pkgs.mkShell {
     ]
   );
   CONFIG_FLAGS = builtins.toString (
-    [
-      (
-        if icu == null then
-          "--without-intl"
-        else
-          "--with-intl=${if useSharedICU then "system" else icu}-icu"
-      )
-    ]
-    ++ extraConfigFlags
-    ++ pkgs.lib.optional (!withAmaro) "--without-amaro"
-    ++ pkgs.lib.optional (!withLief) "--without-lief"
-    ++ pkgs.lib.optional withQuic "--experimental-quic"
-    ++ pkgs.lib.optional (!withSQLite) "--without-sqlite"
-    ++ pkgs.lib.optional (!withSSL) "--without-ssl"
-    ++ pkgs.lib.optional withTemporal "--v8-enable-temporal-support"
-    ++ pkgs.lib.optional (ninja != null) "--ninja"
-    ++ pkgs.lib.optional loadJSBuiltinsDynamically "--node-builtin-modules-path=${builtins.toString ./.}"
-    ++ pkgs.lib.concatMap (name: [
-      "--shared-${name}"
-      "--shared-${name}-libpath=${pkgs.lib.getLib sharedLibDeps.${name}}/lib"
-      "--shared-${name}-include=${pkgs.lib.getInclude sharedLibDeps.${name}}/include"
-    ]) (builtins.attrNames sharedLibDeps)
+    configureFlags ++ pkgs.lib.optional (useSeparateDerivationForV8 != false) "--without-bundled-v8"
   );
   NOSQLITE = pkgs.lib.optionalString (!withSQLite) "1";
 }

tools/nix/v8.nix

@@ -0,0 +1,130 @@
+# Derivation for Node.js CI (not officially supported for regular applications)
+{
+  pkgs ? import ./pkgs.nix { },
+  buildInputs ? [ ],
+  configureFlags ? [ ],
+  needsRustCompiler ? false,
+}:
+
+let
+  nodejs = pkgs.nodejs-slim_latest;
+
+  version =
+    let
+      v8Version = builtins.match (
+        ".*#define V8_MAJOR_VERSION ([0-9]+).*"
+        + "#define V8_MINOR_VERSION ([0-9]+).*"
+        + "#define V8_BUILD_NUMBER ([0-9]+).*"
+        + "#define V8_PATCH_LEVEL ([0-9]+).*"
+      ) (builtins.readFile ../../deps/v8/include/v8-version.h);
+      v8_embedder_string = builtins.match ".*'v8_embedder_string': '-(node.[0-9]+)'.*" (
+        builtins.readFile ../../common.gypi
+      );
+    in
+    if v8Version == null || v8_embedder_string == null then
+      throw "V8 version not found"
+    else
+      "${builtins.elemAt v8Version 0}.${builtins.elemAt v8Version 1}.${builtins.elemAt v8Version 2}.${builtins.elemAt v8Version 3}-${builtins.elemAt v8_embedder_string 0}";
+in
+pkgs.stdenv.mkDerivation (finalAttrs: {
+  pname = "v8";
+  inherit version;
+  src =
+    let
+      inherit (pkgs.lib) fileset;
+    in
+    fileset.toSource {
+      root = ../../.;
+      fileset = fileset.unions [
+        ../../common.gypi
+        ../../configure
+        ../../configure.py
+        ../../deps/inspector_protocol/inspector_protocol.gyp
+        ../../deps/ncrypto/ncrypto.gyp
+        ../../deps/v8
+        ../../node.gyp
+        ../../node.gypi
+        ../../src/inspector/node_inspector.gypi
+        ../../src/node_version.h
+        ../../tools/configure.d
+        ../../tools/getmoduleversion.py
+        ../../tools/getnapibuildversion.py
+        ../../tools/gyp
+        ../../tools/gyp_node.py
+        ../../tools/icu/icu_versions.json
+        ../../tools/icu/icu-system.gyp
+        ../../tools/utils.py
+        ../../tools/v8_gypfiles
+      ];
+    };
+
+  # We need to patch tools/gyp/ to work from within Nix sandbox
+  prePatch = ''
+    patches=()
+    for patch in ${pkgs.lib.concatStringsSep " " nodejs.patches}; do
+      filtered=$(mktemp)
+      filterdiff -p1 -i 'tools/gyp/*' "$patch" > "$filtered"
+      if [ -s "$filtered" ]; then
+        patches+=("$filtered")
+      fi
+    done
+  '';
+
+  inherit (nodejs) configureScript;
+  inherit configureFlags buildInputs;
+
+  nativeBuildInputs =
+    nodejs.nativeBuildInputs
+    ++ [
+      pkgs.patchutils
+      pkgs.validatePkgConfig
+    ]
+    ++ pkgs.lib.optionals needsRustCompiler [
+      pkgs.cargo
+      pkgs.rustc
+    ];
+
+  buildPhase = ''
+    ninja -v -C out/Release v8_snapshot v8_libplatform
+  '';
+  installPhase = ''
+    ${
+      if pkgs.stdenv.buildPlatform.isDarwin then
+        # Darwin is excluded from creating thin archive in tools/gyp/pylib/gyp/generator/ninja.py:2488
+        "install -Dm644 out/Release/lib* -t $out/lib"
+      else
+        # On other platforms, we need to create non-thin archive.
+        ''
+          mkdir -p $out/lib
+          for a in out/Release/obj/tools/v8_gypfiles/lib*; do
+            base=$(basename "$a")
+            dir=$(dirname "$a")
+
+            (
+              cd "$dir"
+              "$AR" rc "$out/lib/$base" $("$AR" t "$base")
+            )
+
+            "$RANLIB" "$out/lib/$base"
+          done
+        ''
+    }
+    libs=$(for f in $out/lib/lib*.a; do
+      b=$(basename "$f" .a)
+      printf " -l%s" "''${b#lib}"
+    done)
+
+    # copy v8 headers
+    cp -r deps/v8/include $out/
+
+    # create a pkgconfig file for v8
+    mkdir -p $out/lib/pkgconfig
+    cat -> $out/lib/pkgconfig/v8.pc << EOF
+    Name: v8
+    Description: V8 JavaScript Engine build for Node.js CI
+    Version: ${version}
+    Libs: -L$out/lib $libs
+    Cflags: -I$out/include
+    EOF
+  '';
+})