diff --git a/.github/workflows/test-shared.yml b/.github/workflows/test-shared.yml index 452cb1d6db8fa3..101001ea64e16e 100644 --- a/.github/workflows/test-shared.yml +++ b/.github/workflows/test-shared.yml @@ -182,25 +182,20 @@ jobs: core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on'); core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - - - name: Load shell.nix - if: github.event_name != 'workflow_dispatch' - run: | - mv "$TAR_DIR"/*.nix . - mkdir tools - mv "$TAR_DIR"/tools/nix tools/. + core.exportVariable('NIX_SCCACHE', '(import {}).sccache'); - name: Build Node.js and run tests run: | nix-shell \ - -I nixpkgs=./tools/nix/pkgs.nix \ + -I "nixpkgs=$TAR_DIR/tools/nix/pkgs.nix" \ --pure --keep TAR_DIR --keep FLAKY_TESTS \ --keep SCCACHE_GHA_ENABLED --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \ --arg loadJSBuiltinsDynamically false \ - --arg ccache '(import {}).sccache' \ + --arg useSeparateDerivationForV8 true \ + --arg ccache "${NIX_SCCACHE:-null}" \ --arg devTools '[]' \ --arg benchmarkTools '[]' \ ${{ endsWith(matrix.system, '-darwin') && '--arg withAmaro false --arg withLief false --arg withSQLite false --arg extraConfigFlags ''["--without-inspector" "--without-node-options"]'' \' || '\' }} --run ' make -C "$TAR_DIR" run-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS" - ' + ' "$TAR_DIR/shell.nix" diff --git a/configure.py b/configure.py index 59a263d4e77246..fa47e9c48547f2 100755 --- a/configure.py +++ b/configure.py @@ -2064,10 +2064,16 @@ def configure_v8(o, configs): o['variables']['node_enable_v8windbg'] = b(options.enable_v8windbg) if options.enable_d8: o['variables']['test_isolation_mode'] = 'noop' # Needed by d8.gyp. - if options.without_bundled_v8 and options.enable_d8: - raise Exception('--enable-d8 is incompatible with --without-bundled-v8.') - if options.without_bundled_v8 and options.enable_v8windbg: - raise Exception('--enable-v8windbg is incompatible with --without-bundled-v8.') + if options.without_bundled_v8: + if options.enable_d8: + raise Exception('--enable-d8 is incompatible with --without-bundled-v8.') + if options.enable_v8windbg: + raise Exception('--enable-v8windbg is incompatible with --without-bundled-v8.') + (pkg_libs, pkg_cflags, pkg_libpath, _) = pkg_config("v8") + if pkg_libs and pkg_libpath: + output['libraries'] += [pkg_libpath] + pkg_libs.split() + if pkg_cflags: + output['include_dirs'] += [flag for flag in [flag.strip() for flag in pkg_cflags.split('-I')] if flag] if options.static_zoslib_gyp: o['variables']['static_zoslib_gyp'] = options.static_zoslib_gyp if flavor != 'linux' and options.v8_enable_hugepage: diff --git a/node.gyp b/node.gyp index 566377625bc712..47ec88c43d0bcd 100644 --- a/node.gyp +++ b/node.gyp @@ -641,7 +641,6 @@ 'xcode_settings': { 'OTHER_LDFLAGS': [ '-Wl,-force_load,<(PRODUCT_DIR)/<(STATIC_LIB_PREFIX)<(node_core_target_name)<(STATIC_LIB_SUFFIX)', - '-Wl,-force_load,<(PRODUCT_DIR)/<(STATIC_LIB_PREFIX)v8_base_without_compiler<(STATIC_LIB_SUFFIX)', ], }, 'msvs_settings': { @@ -653,7 +652,14 @@ }, }, 'conditions': [ - ['OS != "aix" and OS != "os400" and OS != "mac" and OS != "ios"', { + ['node_use_bundled_v8=="true"', { + 'xcode_settings': { + 'OTHER_LDFLAGS': [ + '-Wl,-force_load,<(PRODUCT_DIR)/<(STATIC_LIB_PREFIX)v8_base_without_compiler<(STATIC_LIB_SUFFIX)', + ], + }, + }], + ['node_use_bundled_v8=="true" and OS != "aix" and OS != "os400" and OS != "mac" and OS != "ios"', { 'ldflags': [ '-Wl,--whole-archive', '<(obj_dir)/<(STATIC_LIB_PREFIX)<(node_core_target_name)<(STATIC_LIB_SUFFIX)', @@ -661,6 +667,13 @@ '-Wl,--no-whole-archive', ], }], + ['node_use_bundled_v8!="true" and OS != "aix" and OS != "os400" and OS != "mac" and OS != "ios"', { + 'ldflags': [ + '-Wl,--whole-archive', + '<(obj_dir)/<(STATIC_LIB_PREFIX)<(node_core_target_name)<(STATIC_LIB_SUFFIX)', + '-Wl,--no-whole-archive', + ], + }], [ 'OS=="win"', { 'sources': [ 'src/res/node.rc' ], }], @@ -1530,6 +1543,9 @@ 'src/builtin_info.cc', ], 'conditions': [ + [ 'OS=="mac"', { + 'libraries': [ '-framework CoreFoundation -framework Security' ], + }], [ 'node_shared_simdutf=="false"', { 'dependencies': [ 'tools/v8_gypfiles/v8.gyp:simdutf#host' ], }], diff --git a/node.gypi b/node.gypi index 6587270c9e52f3..dd31ac4abcb2c6 100644 --- a/node.gypi +++ b/node.gypi @@ -317,8 +317,8 @@ 'NODE_PLATFORM="sunos"', ], }], - [ '(OS=="freebsd" or OS=="linux" or OS=="openharmony") and node_shared=="false"' - ' and force_load=="true"', { + [ 'node_use_bundled_v8=="true" and (OS=="freebsd" or OS=="linux" or OS=="openharmony") ' + 'and node_shared=="false" and force_load=="true"', { 'ldflags': [ '-Wl,-z,noexecstack', '-Wl,--whole-archive <(v8_base)', diff --git a/shell.nix b/shell.nix index 4a9f71b0372787..dabb03e88ff6fe 100644 --- a/shell.nix +++ b/shell.nix @@ -8,6 +8,7 @@ extraConfigFlags ? [ "--debug-node" ], + useSeparateDerivationForV8 ? false, # to help CI better managed its binary cache, not recommended outside of CI usage. # Build options icu ? pkgs.icu, @@ -17,16 +18,21 @@ withSQLite ? true, withSSL ? true, withTemporal ? false, - sharedLibDeps ? import ./tools/nix/sharedLibDeps.nix { - inherit - pkgs - withLief - withQuic - withSQLite - withSSL - withTemporal - ; - }, + sharedLibDeps ? + let + d = import ./tools/nix/sharedLibDeps.nix { + inherit + pkgs + withLief + withQuic + withSQLite + withSSL + withTemporal + ; + }; + in + # To avoid conflicts with V8's bundled simdutf lib, it's easier to remove it when using a precompiled V8. + if (useSeparateDerivationForV8 != false) then builtins.removeAttrs d [ "simdutf" ] else d, # dev tools (not needed to build Node.js, useful to maintain it) ncu-path ? null, # Provide this if you want to use a local version of NCU @@ -40,11 +46,49 @@ let useSharedOpenSSL = builtins.hasAttr "openssl" sharedLibDeps; needsRustCompiler = withTemporal && !builtins.hasAttr "temporal_capi" sharedLibDeps; + + buildInputs = builtins.attrValues sharedLibDeps ++ pkgs.lib.optional useSharedICU icu; + configureFlags = [ + ( + if icu == null then + "--without-intl" + else + "--with-intl=${if useSharedICU then "system" else icu}-icu" + ) + ] + ++ extraConfigFlags + ++ pkgs.lib.optional (!withAmaro) "--without-amaro" + ++ pkgs.lib.optional (!withLief) "--without-lief" + ++ pkgs.lib.optional withQuic "--experimental-quic" + ++ pkgs.lib.optional (!withSQLite) "--without-sqlite" + ++ pkgs.lib.optional (!withSSL) "--without-ssl" + ++ pkgs.lib.optional withTemporal "--v8-enable-temporal-support" + ++ pkgs.lib.optional (ninja != null) "--ninja" + ++ pkgs.lib.optional loadJSBuiltinsDynamically "--node-builtin-modules-path=${builtins.toString ../..}" + ++ pkgs.lib.concatMap (name: [ + "--shared-${name}" + "--shared-${name}-libpath=${pkgs.lib.getLib sharedLibDeps.${name}}/lib" + "--shared-${name}-include=${pkgs.lib.getInclude sharedLibDeps.${name}}/include" + ]) (builtins.attrNames sharedLibDeps); in pkgs.mkShell { inherit (pkgs.nodejs_latest) nativeBuildInputs; - buildInputs = builtins.attrValues sharedLibDeps ++ pkgs.lib.optional useSharedICU icu; + buildInputs = + buildInputs + ++ pkgs.lib.optional (useSeparateDerivationForV8 != false) ( + if useSeparateDerivationForV8 == true then + import ./tools/nix/v8.nix { + inherit + pkgs + configureFlags + buildInputs + needsRustCompiler + ; + } + else + useSeparateDerivationForV8 + ); packages = pkgs.lib.optional (ccache != null) ccache @@ -74,28 +118,7 @@ pkgs.mkShell { ] ); CONFIG_FLAGS = builtins.toString ( - [ - ( - if icu == null then - "--without-intl" - else - "--with-intl=${if useSharedICU then "system" else icu}-icu" - ) - ] - ++ extraConfigFlags - ++ pkgs.lib.optional (!withAmaro) "--without-amaro" - ++ pkgs.lib.optional (!withLief) "--without-lief" - ++ pkgs.lib.optional withQuic "--experimental-quic" - ++ pkgs.lib.optional (!withSQLite) "--without-sqlite" - ++ pkgs.lib.optional (!withSSL) "--without-ssl" - ++ pkgs.lib.optional withTemporal "--v8-enable-temporal-support" - ++ pkgs.lib.optional (ninja != null) "--ninja" - ++ pkgs.lib.optional loadJSBuiltinsDynamically "--node-builtin-modules-path=${builtins.toString ./.}" - ++ pkgs.lib.concatMap (name: [ - "--shared-${name}" - "--shared-${name}-libpath=${pkgs.lib.getLib sharedLibDeps.${name}}/lib" - "--shared-${name}-include=${pkgs.lib.getInclude sharedLibDeps.${name}}/include" - ]) (builtins.attrNames sharedLibDeps) + configureFlags ++ pkgs.lib.optional (useSeparateDerivationForV8 != false) "--without-bundled-v8" ); NOSQLITE = pkgs.lib.optionalString (!withSQLite) "1"; } diff --git a/tools/nix/v8.nix b/tools/nix/v8.nix new file mode 100644 index 00000000000000..f3fa6149646155 --- /dev/null +++ b/tools/nix/v8.nix @@ -0,0 +1,130 @@ +# Derivation for Node.js CI (not officially supported for regular applications) +{ + pkgs ? import ./pkgs.nix { }, + buildInputs ? [ ], + configureFlags ? [ ], + needsRustCompiler ? false, +}: + +let + nodejs = pkgs.nodejs-slim_latest; + + version = + let + v8Version = builtins.match ( + ".*#define V8_MAJOR_VERSION ([0-9]+).*" + + "#define V8_MINOR_VERSION ([0-9]+).*" + + "#define V8_BUILD_NUMBER ([0-9]+).*" + + "#define V8_PATCH_LEVEL ([0-9]+).*" + ) (builtins.readFile ../../deps/v8/include/v8-version.h); + v8_embedder_string = builtins.match ".*'v8_embedder_string': '-(node.[0-9]+)'.*" ( + builtins.readFile ../../common.gypi + ); + in + if v8Version == null || v8_embedder_string == null then + throw "V8 version not found" + else + "${builtins.elemAt v8Version 0}.${builtins.elemAt v8Version 1}.${builtins.elemAt v8Version 2}.${builtins.elemAt v8Version 3}-${builtins.elemAt v8_embedder_string 0}"; +in +pkgs.stdenv.mkDerivation (finalAttrs: { + pname = "v8"; + inherit version; + src = + let + inherit (pkgs.lib) fileset; + in + fileset.toSource { + root = ../../.; + fileset = fileset.unions [ + ../../common.gypi + ../../configure + ../../configure.py + ../../deps/inspector_protocol/inspector_protocol.gyp + ../../deps/ncrypto/ncrypto.gyp + ../../deps/v8 + ../../node.gyp + ../../node.gypi + ../../src/inspector/node_inspector.gypi + ../../src/node_version.h + ../../tools/configure.d + ../../tools/getmoduleversion.py + ../../tools/getnapibuildversion.py + ../../tools/gyp + ../../tools/gyp_node.py + ../../tools/icu/icu_versions.json + ../../tools/icu/icu-system.gyp + ../../tools/utils.py + ../../tools/v8_gypfiles + ]; + }; + + # We need to patch tools/gyp/ to work from within Nix sandbox + prePatch = '' + patches=() + for patch in ${pkgs.lib.concatStringsSep " " nodejs.patches}; do + filtered=$(mktemp) + filterdiff -p1 -i 'tools/gyp/*' "$patch" > "$filtered" + if [ -s "$filtered" ]; then + patches+=("$filtered") + fi + done + ''; + + inherit (nodejs) configureScript; + inherit configureFlags buildInputs; + + nativeBuildInputs = + nodejs.nativeBuildInputs + ++ [ + pkgs.patchutils + pkgs.validatePkgConfig + ] + ++ pkgs.lib.optionals needsRustCompiler [ + pkgs.cargo + pkgs.rustc + ]; + + buildPhase = '' + ninja -v -C out/Release v8_snapshot v8_libplatform + ''; + installPhase = '' + ${ + if pkgs.stdenv.buildPlatform.isDarwin then + # Darwin is excluded from creating thin archive in tools/gyp/pylib/gyp/generator/ninja.py:2488 + "install -Dm644 out/Release/lib* -t $out/lib" + else + # On other platforms, we need to create non-thin archive. + '' + mkdir -p $out/lib + for a in out/Release/obj/tools/v8_gypfiles/lib*; do + base=$(basename "$a") + dir=$(dirname "$a") + + ( + cd "$dir" + "$AR" rc "$out/lib/$base" $("$AR" t "$base") + ) + + "$RANLIB" "$out/lib/$base" + done + '' + } + libs=$(for f in $out/lib/lib*.a; do + b=$(basename "$f" .a) + printf " -l%s" "''${b#lib}" + done) + + # copy v8 headers + cp -r deps/v8/include $out/ + + # create a pkgconfig file for v8 + mkdir -p $out/lib/pkgconfig + cat -> $out/lib/pkgconfig/v8.pc << EOF + Name: v8 + Description: V8 JavaScript Engine build for Node.js CI + Version: ${version} + Libs: -L$out/lib $libs + Cflags: -I$out/include + EOF + ''; +})