build: remove native mksnapshot for arm/arm64 (#17561)

* build: remove native mksnapshot for arm/arm64

.circleci/config.yml

@@ -429,44 +429,29 @@ step-maybe-zip-symbols: &step-maybe-zip-symbols
      export BUILD_PATH="$PWD/out/Default"
      electron/script/zip-symbols.py -b $BUILD_PATH

step-maybe-native-mksnapshot-gn-gen: &step-maybe-native-mksnapshot-gn-gen
step-maybe-cross-arch-snapshot: &step-maybe-cross-arch-snapshot
  run:
    name: Native mksnapshot GN gen (arm/arm64)
    name: Generate cross arch snapshot (arm/arm64)
    command: |
      if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then
        cd src
        gn gen out/native_mksnapshot --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS v8_enable_embedded_builtins = false"
      else
        echo 'Skipping native mksnapshot GN gen for non arm build'
      fi

step-maybe-native-mksnapshot-build: &step-maybe-native-mksnapshot-build
  run:
    name: Native mksnapshot build (arm/arm64)
    no_output_timeout: 30m
    command: |
      if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then
        cd src
        ninja -C out/native_mksnapshot electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
      else
        echo 'Skipping native mksnapshot build for non arm build'
      fi

step-maybe-native-mksnapshot-strip: &step-maybe-native-mksnapshot-strip
  run:
    name: Native mksnapshot binary strip (arm/arm64)
    command: |
      if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then
      if [ "$TRIGGER_ARM_TEST" == "true" ] && [ -z "$CIRCLE_PR_NUMBER" ]; then
        cd src
        electron/script/strip-binaries.py --file $PWD/out/native_mksnapshot/mksnapshot --target-cpu="$TARGET_ARCH"
      else
        echo 'Skipping native mksnapshot binary strip'
        if [ "$TARGET_ARCH" == "arm" ]; then
          export MKSNAPSHOT_PATH="clang_x86_v8_arm"
        elif [ "$TARGET_ARCH" == "arm64" ]; then
          export MKSNAPSHOT_PATH="clang_x64_v8_arm64"
        fi
        cp "out/Default/$MKSNAPSHOT_PATH/mksnapshot" out/Default
        cp "out/Default/$MKSNAPSHOT_PATH/libffmpeg.so" out/Default
        cp "out/Default/$MKSNAPSHOT_PATH/v8_context_snapshot_generator" out/Default
        python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --create-snapshot-only
        mkdir cross-arch-snapshots
        cp out/Default-mksnapshot-test/*.bin cross-arch-snapshots
      fi

step-maybe-native-mksnapshot-store: &step-maybe-native-mksnapshot-store
step-maybe-cross-arch-snapshot-store: &step-maybe-cross-arch-snapshot-store
  store_artifacts:
    path: src/out/native_mksnapshot/mksnapshot.zip
    destination: native_mksnapshot.zip
    path: src/cross-arch-snapshots
    destination: cross-arch-snapshots

step-maybe-trigger-arm-test: &step-maybe-trigger-arm-test
  run:
@@ -626,12 +611,8 @@ steps-electron-build-for-tests: &steps-electron-build-for-tests
    # mksnapshot
    - *step-mksnapshot-build
    - *step-mksnapshot-store

    # native_mksnapshot
    - *step-maybe-native-mksnapshot-gn-gen
    - *step-maybe-native-mksnapshot-build
    - *step-maybe-native-mksnapshot-strip
    - *step-maybe-native-mksnapshot-store
    - *step-maybe-cross-arch-snapshot
    - *step-maybe-cross-arch-snapshot-store

    # ffmpeg
    - *step-ffmpeg-gn-gen
@@ -673,12 +654,6 @@ steps-electron-build-for-publish: &steps-electron-build-for-publish
    - *step-mksnapshot-build
    - *step-mksnapshot-store

    # native_mksnapshot
    - *step-maybe-native-mksnapshot-gn-gen
    - *step-maybe-native-mksnapshot-build
    - *step-maybe-native-mksnapshot-strip
    - *step-maybe-native-mksnapshot-store

    # chromedriver
    - *step-electron-chromedriver-build
    - *step-electron-chromedriver-store
@@ -712,18 +687,6 @@ steps-chromedriver-build: &steps-chromedriver-build

    - *step-maybe-notify-slack-failure

steps-native-mksnapshot-build: &steps-native-mksnapshot-build
  steps:
    - attach_workspace:
        at: .
    - *step-depot-tools-add-to-path
    - *step-setup-env-for-build
    - *step-maybe-native-mksnapshot-gn-gen
    - *step-maybe-native-mksnapshot-build
    - *step-maybe-native-mksnapshot-store

    - *step-maybe-notify-slack-failure

steps-native-tests: &steps-native-tests
  steps:
    - attach_workspace:
@@ -1017,16 +980,6 @@ jobs:
      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True'
    <<: *steps-electron-build-for-publish

  linux-arm-native-mksnapshot:
    <<: *machine-linux-2xlarge
    environment:
      <<: *env-linux-medium
      <<: *env-arm
      <<: *env-release-build
      <<: *env-enable-sccache
      <<: *env-send-slack-notifications
    <<: *steps-native-mksnapshot-build

  linux-arm64-debug:
    <<: *machine-linux-2xlarge
    environment:
@@ -1091,16 +1044,6 @@ jobs:
      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True'
    <<: *steps-electron-build-for-publish

  linux-arm64-native-mksnapshot:
    <<: *machine-linux-2xlarge
    environment:
      <<: *env-linux-medium
      <<: *env-arm64
      <<: *env-release-build
      <<: *env-enable-sccache
      <<: *env-send-slack-notifications
    <<: *steps-native-mksnapshot-build

  osx-testing:
    <<: *machine-mac-large
    environment:
@@ -1565,32 +1508,25 @@ workflows:
      - linux-arm-release:
          requires:
            - linux-checkout
      - linux-arm-native-mksnapshot:
          requires:
            - linux-checkout
      - linux-arm-chromedriver:
          requires:
            - linux-checkout
      - linux-arm-release-summary:
          requires:
            - linux-arm-release
            - linux-arm-chromedriver
            - linux-arm-native-mksnapshot


      - linux-arm64-release:
          requires:
            - linux-checkout
      - linux-arm64-native-mksnapshot:
          requires:
            - linux-checkout
      - linux-arm64-chromedriver:
          requires:
            - linux-checkout
      - linux-arm64-release-summary:
          requires:
            - linux-arm64-release
            - linux-arm64-chromedriver
            - linux-arm64-native-mksnapshot

  nightly-mac-release-test:
    triggers:

patches/common/v8/.patches

@@ -2,7 +2,5 @@ deps_backport_detailed_line_info_for_cpu_profiler.patch
add_realloc.patch
build_gn.patch
expose_mksnapshot.patch
build-torque-with-x64-toolchain-on-arm.patch
do_not_run_arm_arm64_mksnapshot_binaries.patch
deps_provide_more_v8_backwards_compatibility.patch
dcheck.patch

patches/common/v8/expose_mksnapshot.patch

@@ -3,7 +3,7 @@ From: Shelley Vohr <shelley.vohr@gmail.com>
Date: Mon, 22 Oct 2018 10:47:13 -0700
Subject: expose_mksnapshot.patch

Needed in order to build mksnapshot on arm.
Needed in order to target mksnapshot for mksnapshot zip.

diff --git a/BUILD.gn b/BUILD.gn
index d8a2b8e11a3e0e9820dca061a00dbf1a6859bcf4..d3dbe37d0a145921dddaea72e394c87826d6d5fe 100644

script/upload.py

@@ -96,10 +96,6 @@ def main():
  mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION)
  mksnapshot_zip = os.path.join(OUT_DIR, mksnapshot)
  if get_target_arch().startswith('arm'):
    # Upload the native mksnapshot as mksnapshot.zip
    shutil.copy2(os.path.join(SRC_DIR, 'out', 'native_mksnapshot',
                              'mksnapshot.zip'), mksnapshot_zip)
    upload_electron(release, mksnapshot_zip, args)
    # Upload the x64 binary for arm/arm64 mksnapshot
    mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION, 'x64')
    mksnapshot_zip = os.path.join(OUT_DIR, mksnapshot)

script/verify-mksnapshot.py

@@ -23,20 +23,28 @@ def main():
  returncode = 0
  try:
    with scoped_cwd(app_path):
      mkargs = [ get_binary_path('mksnapshot', app_path), \
                   SNAPSHOT_SOURCE, '--startup_blob', 'snapshot_blob.bin', \
                   '--turbo_instruction_scheduling' ]
      subprocess.check_call(mkargs)
      print 'ok mksnapshot successfully created snapshot_blob.bin.'
      context_snapshot = 'v8_context_snapshot.bin'
      context_snapshot_path = os.path.join(app_path, context_snapshot)
      gen_binary = get_binary_path('v8_context_snapshot_generator', \
                                   app_path)
      genargs = [ gen_binary, \
                 '--output_file={0}'.format(context_snapshot_path) ]
      subprocess.check_call(genargs)
      print 'ok v8_context_snapshot_generator successfully created ' \
            + context_snapshot
      if args.snapshot_files_dir is None:
        mkargs = [ get_binary_path('mksnapshot', app_path), \
                    SNAPSHOT_SOURCE, '--startup_blob', 'snapshot_blob.bin', \
                    '--turbo_instruction_scheduling' ]
        subprocess.check_call(mkargs)
        print 'ok mksnapshot successfully created snapshot_blob.bin.'
        context_snapshot = 'v8_context_snapshot.bin'
        context_snapshot_path = os.path.join(app_path, context_snapshot)
        gen_binary = get_binary_path('v8_context_snapshot_generator', \
                                    app_path)
        genargs = [ gen_binary, \
                  '--output_file={0}'.format(context_snapshot_path) ]
        subprocess.check_call(genargs)
        print 'ok v8_context_snapshot_generator successfully created ' \
              + context_snapshot
        if args.create_snapshot_only:
          return 0
      else:
        gen_bin_path = os.path.join(args.snapshot_files_dir, '*.bin')
        generated_bin_files = glob.glob(gen_bin_path)
        for bin_file in generated_bin_files:
          shutil.copy2(bin_file, app_path)

      test_path = os.path.join(SOURCE_ROOT, 'spec', 'fixtures', \
                               'snapshot-items-available')
@@ -93,6 +101,14 @@ def parse_args():
                          Relative to the --source-root.',
                      default=None,
                      required=True)
  parser.add_argument('--create-snapshot-only',
                      help='Just create snapshot files, but do not run test',
                      action='store_true')
  parser.add_argument('--snapshot-files-dir',
                      help='Directory containing snapshot files to use \
                          for testing',
                      default=None,
                      required=False)
  parser.add_argument('--source-root',
                      default=SOURCE_ROOT,
                      required=False)

vsts-arm-test-steps.yml

@@ -32,15 +32,6 @@ steps:
  env:
    CIRCLE_TOKEN: $(CIRCLECI_TOKEN)

- bash: |
    cd src/electron
    node script/download-circleci-artifacts.js --buildNum=$CIRCLE_BUILD_NUM --name=native_mksnapshot.zip --dest=$ZIP_DEST
    cd $ZIP_DEST
    unzip -o native_mksnapshot.zip
  displayName: 'Download and unzip native_mksnapshot.zip for test'
  env:
    CIRCLE_TOKEN: $(CIRCLECI_TOKEN)

- bash: |
   export NODE_HEADERS_DEST=$PWD/src/out/Default/gen
   mkdir -p $NODE_HEADERS_DEST
@@ -52,6 +43,17 @@ steps:
  env:
    CIRCLE_TOKEN: $(CIRCLECI_TOKEN)

- bash: |
   export CROSS_ARCH_SNAPSHOTS=$PWD/src/out/Default/cross-arch-snapshots
   mkdir -p $CROSS_ARCH_SNAPSHOTS
   cd src/electron
   node script/download-circleci-artifacts.js --buildNum=$CIRCLE_BUILD_NUM --name=cross-arch-snapshots/snapshot_blob.bin --dest=$CROSS_ARCH_SNAPSHOTS
   node script/download-circleci-artifacts.js --buildNum=$CIRCLE_BUILD_NUM --name=cross-arch-snapshots/v8_context_snapshot.bin --dest=$CROSS_ARCH_SNAPSHOTS
  displayName: 'Download cross arch snapshot files'
  env:
    CIRCLE_TOKEN: $(CIRCLECI_TOKEN)


- bash: |
    cd src
    export npm_config_nodedir=$PWD/out/Default/gen/node_headers
@@ -83,9 +85,9 @@ steps:

- bash: |
    cd src
    echo Verify mksnapshot temporarily disabled due to M74 upgrade
    #python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default
  displayName: Verify mksnapshot
    echo Verify cross arch snapshot
    python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --snapshot-files-dir $PWD/out/Default/cross-arch-snapshots
  displayName: Verify cross arch snapshot
  timeoutInMinutes: 5
  env:
    ELECTRON_DISABLE_SANDBOX: 1