devops: refactor chromium automation scripts (#5486)

Split preparing checkout, archiving and compiling into separate scripts similarly how we do it with other browsers.
microsoft · Feb 17, 2021 · 8c18b90 · 8c18b90
1 parent b2227c1
commit 8c18b90
Show file tree

Hide file tree

Showing 4 changed files with 249 additions and 161 deletions.
diff --git a/browser_patches/checkout_build_archive_upload.sh b/browser_patches/checkout_build_archive_upload.sh
@@ -33,15 +33,25 @@ fi
 
 BROWSER_NAME=""
 EXTRA_BUILD_ARGS=""
+EXTRA_ARCHIVE_ARGS=""
 BUILD_FLAVOR="$1"
 BUILD_BLOB_NAME=""
 EXPECTED_HOST_OS=""
 EXPECTED_HOST_OS_VERSION=""
 EXPECTED_ARCH="x86_64"
+
+# ===========================
+#    WINLDD COMPILATION
+# ===========================
 if [[ "$BUILD_FLAVOR" == "winldd-win64" ]]; then
   BROWSER_NAME="winldd"
   EXPECTED_HOST_OS="MINGW"
   BUILD_BLOB_NAME="winldd-win64.zip"
+
+
+# ===========================
+#    FFMPEG COMPILATION
+# ===========================
 elif [[ "$BUILD_FLAVOR" == "ffmpeg-mac" ]]; then
   BROWSER_NAME="ffmpeg"
   EXTRA_BUILD_ARGS="--mac"
@@ -66,58 +76,81 @@ elif [[ "$BUILD_FLAVOR" == "ffmpeg-cross-compile-win64" ]]; then
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="20.04"
   BUILD_BLOB_NAME="ffmpeg-win64.zip"
+
+# ===========================
+#    CHROMIUM COMPILATION
+# ===========================
 elif [[ "$BUILD_FLAVOR" == "chromium-win32" ]]; then
   BROWSER_NAME="chromium"
   EXTRA_BUILD_ARGS="--compile-win32"
+  EXTRA_ARCHIVE_ARGS="--compile-win32"
   EXPECTED_HOST_OS="MINGW"
   BUILD_BLOB_NAME="chromium-win32.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-win64" ]]; then
   BROWSER_NAME="chromium"
   EXTRA_BUILD_ARGS="--compile-win64"
+  EXTRA_ARCHIVE_ARGS="--compile-win64"
   EXPECTED_HOST_OS="MINGW"
   BUILD_BLOB_NAME="chromium-win64.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-mac" ]]; then
   BROWSER_NAME="chromium"
   EXTRA_BUILD_ARGS="--compile-mac"
+  EXTRA_ARCHIVE_ARGS="--compile-mac"
   EXPECTED_HOST_OS="Darwin"
   EXPECTED_HOST_OS_VERSION="10.15"
   BUILD_BLOB_NAME="chromium-mac.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-mac-arm64" ]]; then
   BROWSER_NAME="chromium"
   EXTRA_BUILD_ARGS="--compile-mac-arm64"
+  EXTRA_ARCHIVE_ARGS="--compile-mac-arm64"
   EXPECTED_HOST_OS="Darwin"
   EXPECTED_HOST_OS_VERSION="10.15"
   BUILD_BLOB_NAME="chromium-mac-arm64.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-linux" ]]; then
   BROWSER_NAME="chromium"
   EXTRA_BUILD_ARGS="--compile-linux"
+  EXTRA_ARCHIVE_ARGS="--compile-linux"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-linux.zip"
+
+
+# ===========================
+#    CHROMIUM MIRRORING
+# ===========================
 elif [[ "$BUILD_FLAVOR" == "chromium-linux-mirror-to-cdn" ]]; then
   BROWSER_NAME="chromium"
   EXTRA_BUILD_ARGS="--mirror-linux"
+  EXTRA_ARCHIVE_ARGS="--mirror-linux"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-linux.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-mac-mirror-to-cdn" ]]; then
   BROWSER_NAME="chromium"
   EXTRA_BUILD_ARGS="--mirror-mac"
+  EXTRA_ARCHIVE_ARGS="--mirror-mac"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-mac.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-win32-mirror-to-cdn" ]]; then
   BROWSER_NAME="chromium"
   EXTRA_BUILD_ARGS="--mirror-win32"
+  EXTRA_ARCHIVE_ARGS="--mirror-win32"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-win32.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-win64-mirror-to-cdn" ]]; then
   BROWSER_NAME="chromium"
   EXTRA_BUILD_ARGS="--mirror-win64"
+  EXTRA_ARCHIVE_ARGS="--mirror-win64"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-win64.zip"
+
+
+# ===========================
+#    FIREFOX COMPILATION
+# ===========================
 elif [[ "$BUILD_FLAVOR" == "firefox-ubuntu-18.04" ]]; then
   BROWSER_NAME="firefox"
   EXTRA_BUILD_ARGS="--full"
@@ -146,6 +179,11 @@ elif [[ "$BUILD_FLAVOR" == "firefox-win64" ]]; then
   EXTRA_BUILD_ARGS="--win64"
   EXPECTED_HOST_OS="MINGW"
   BUILD_BLOB_NAME="firefox-win64.zip"
+
+
+# ===========================
+#    WEBKIT COMPILATION
+# ===========================
 elif [[ "$BUILD_FLAVOR" == "webkit-ubuntu-18.04" ]]; then
   BROWSER_NAME="webkit"
   EXTRA_BUILD_ARGS="--full"
@@ -255,7 +293,7 @@ function generate_and_upload_browser_build {
   fi
 
   echo "-- archiving to $ZIP_PATH"
-  if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH; then
+  if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH "$EXTRA_ARCHIVE_ARGS"; then
     return 23
   fi
 

diff --git a/browser_patches/chromium/archive.sh b/browser_patches/chromium/archive.sh
@@ -2,33 +2,170 @@
 set -e
 set +x
 
-if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
-  echo "usage: $(basename $0) [output-absolute-path]"
-  echo
-  echo "Generate distributable .zip archive from ./output folder that was previously downloaded."
-  echo
-  exit 0
-fi
-
-ZIP_PATH=$1
-if [[ $ZIP_PATH != /* ]]; then
-  echo "ERROR: path $ZIP_PATH is not absolute"
-  exit 1
-fi
-if [[ $ZIP_PATH != *.zip ]]; then
-  echo "ERROR: path $ZIP_PATH must have .zip extension"
-  exit 1
-fi
-if [[ -f $ZIP_PATH ]]; then
-  echo "ERROR: path $ZIP_PATH exists; can't do anything."
-  exit 1
-fi
-if ! [[ -d $(dirname $ZIP_PATH) ]]; then
-  echo "ERROR: folder for path $($ZIP_PATH) does not exist."
-  exit 1
-fi
-
 trap "cd $(pwd -P)" EXIT
 cd "$(dirname $0)"
+SCRIPT_PATH=$(pwd -P)
+
+main() {
+  if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
+    echo "usage: $(basename $0) [output-absolute-path]"
+    echo
+    echo "Generate distributable .zip archive from ./output folder that was previously downloaded."
+    echo
+    exit 0
+  fi
+
+  ZIP_PATH=$1
+
+  if [[ $ZIP_PATH != /* ]]; then
+    echo "ERROR: path $ZIP_PATH is not absolute"
+    exit 1
+  fi
+  if [[ $ZIP_PATH != *.zip ]]; then
+    echo "ERROR: path $ZIP_PATH must have .zip extension"
+    exit 1
+  fi
+  if [[ -f $ZIP_PATH ]]; then
+    echo "ERROR: path $ZIP_PATH exists; can't do anything."
+    exit 1
+  fi
+  if ! [[ -d $(dirname $ZIP_PATH) ]]; then
+    echo "ERROR: folder for path $($ZIP_PATH) does not exist."
+    exit 1
+  fi
+
+  BUILD_TYPE=$2
+  if [[ "${BUILD_TYPE}" == "--compile"* ]]; then
+    archive_compiled_chromium "${BUILD_TYPE}"
+  elif [[ "${BUILD_TYPE}" == "--mirror"* ]]; then
+    archive_mirrored_chromium "${BUILD_TYPE}"
+  else
+    echo "ERROR: unknown build type - ${BUILD_TYPE}"
+    exit 1
+  fi
+
+  cd "${SCRIPT_PATH}"
+  cp output/build.zip $ZIP_PATH
+}
+
+function archive_compiled_chromium() {
+  CHROMIUM_FOLDER_NAME=""
+  CHROMIUM_FILES_TO_ARCHIVE=()
+
+  if [[ $1 == "--compile-mac"* ]]; then
+    CHROMIUM_FOLDER_NAME="chrome-mac"
+    CHROMIUM_FILES_TO_ARCHIVE=("Chromium.app")
+  elif [[ $1 == "--compile-linux" ]]; then
+    CHROMIUM_FOLDER_NAME="chrome-linux"
+    CHROMIUM_FILES_TO_ARCHIVE=(
+      "chrome"
+      "chrome_100_percent.pak"
+      "chrome_200_percent.pak"
+      "chrome_sandbox"
+      "chrome-wrapper"
+      "ClearKeyCdm"
+      "crashpad_handler"
+      "icudtl.dat"
+      "libEGL.so"
+      "libGLESv2.so"
+      "locales"
+      "MEIPreload"
+      "nacl_helper"
+      "nacl_helper_bootstrap"
+      "nacl_helper_nonsfi"
+      "nacl_irt_x86_64.nexe"
+      "product_logo_48.png"
+      "resources"
+      "resources.pak"
+      "swiftshader"
+      "v8_context_snapshot.bin"
+      "xdg-mime"
+      "xdg-settings"
+    )
+  elif [[ $1 == "--compile-win"* ]]; then
+    CHROMIUM_FOLDER_NAME="chrome-win"
+    CHROMIUM_FILES_TO_ARCHIVE=(
+      "chrome.dll"
+      "chrome.exe"
+      "chrome_100_percent.pak"
+      "chrome_200_percent.pak"
+      "chrome_elf.dll"
+      "chrome_proxy.exe"
+      "chrome_pwa_launcher.exe"
+      "D3DCompiler_47.dll"
+      "elevation_service.exe"
+      "eventlog_provider.dll"
+      "First Run"
+      "icudtl.dat"
+      "libEGL.dll"
+      "libGLESv2.dll"
+      "locales"
+      "MEIPreload"
+      "mojo_core.dll"
+      "nacl_irt_x86_64.nexe"
+      "notification_helper.exe"
+      "resources.pak"
+      "swiftshader/libEGL.dll"
+      "swiftshader/libGLESv2.dll"
+      "v8_context_snapshot.bin"
+    )
+  else
+    echo "ERROR: unknown command, use --help for details"
+    exit 1
+  fi
+
+  # Prepare resulting archive.
+  cd "$SCRIPT_PATH"
+  rm -rf output
+  mkdir -p "output/${CHROMIUM_FOLDER_NAME}"
+
+  # On Mac, use 'ditto' to copy directories instead of 'cp'.
+  COPY_COMMAND="cp -R"
+  if [[ $(uname) == "Darwin" ]]; then
+    COPY_COMMAND="ditto"
+  fi
+
+  for ((i = 0; i < ${#CHROMIUM_FILES_TO_ARCHIVE[@]}; i++)) do
+    file="${CHROMIUM_FILES_TO_ARCHIVE[$i]}"
+    mkdir -p "output/${CHROMIUM_FOLDER_NAME}/$(dirname $file)"
+    $COPY_COMMAND "${CR_CHECKOUT_PATH}/src/out/Default/${file}" "output/${CHROMIUM_FOLDER_NAME}/${file}"
+  done
+
+  if [[ $1 == "--compile-win"* ]]; then
+    $COPY_COMMAND "${CR_CHECKOUT_PATH}/src/out/Default/"*.manifest "output/${CHROMIUM_FOLDER_NAME}/"
+  fi
+
+  cd output
+  zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
+}
+
+archive_mirrored_chromium() {
+  cd "${SCRIPT_PATH}/output"
+
+  CHROMIUM_FOLDER_NAME=""
+  CHROMIUM_FILES_TO_REMOVE=()
+
+  PLATFORM="$1"
+  if [[ "${PLATFORM}" == "--mirror-win32" ]]; then
+    CHROMIUM_FOLDER_NAME="chrome-win"
+    CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
+  elif [[ "${PLATFORM}" == "--mirror-win64" ]]; then
+    CHROMIUM_FOLDER_NAME="chrome-win"
+    CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
+  elif [[ "${PLATFORM}" == "--mirror-mac" ]]; then
+    CHROMIUM_FOLDER_NAME="chrome-mac"
+  elif [[ "${PLATFORM}" == "--mirror-linux" ]]; then
+    CHROMIUM_FOLDER_NAME="chrome-linux"
+  else
+    echo "ERROR: unknown platform to build: $PLATFORM"
+    exit 1
+  fi
+
+  for file in ${CHROMIUM_FILES_TO_REMOVE[@]}; do
+    rm -f "${file}"
+  done
+
+  zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
+}
 
-cp output/build.zip $ZIP_PATH
+main "$@"