Sync recent changes to windows-new-dir-hierarchy (#8680)

* [MacOS] Add xcode 15.0.1 (#8604)

* Updating readme file for macos-13 version 20231018.14 (#8601)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* Updating readme file for macos-13-arm64 version 20231017.1 (#8586)

Co-authored-by: Image generation service account <no-reply@microsoft.com>

* [MacOS] improve errexit option handling (#8432)

* [MacOS] improve errexit option handling

* Prevent prompting for override when unzipping

* Explicitly allow failing when installing virtualbox

* Remove libtcl symlinks

* Add debug output

* Updating readme file for ubuntu20 version 20231022.1.1 (#8637)

Co-authored-by: Image generation service account <no-reply@microsoft.com>

* Updating readme file for ubuntu22 version 20231022.1.1 (#8636)

Co-authored-by: Image generation service account <no-reply@microsoft.com>

* Updating readme file for win19 version 20231023.1.1 (#8644)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* Updating readme file for win22 version 20231023.1.1 (#8643)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* [macos] simplify "brew_smart_install" helper (#8639)

* [macos] homebrew: use hardcoded condition for jq installation

we cannot use "jq" if we are asked to install "jq"

* [macos] always use "brew install"

* [macos] add retries to "get_github_package_download_url" helper

* [macos] add retries to chrome install script

* [macos] add retries to OpenJDK install script

* [macos] add retries to miniconda installer

* Update images/macos/provision/core/openjdk.sh

Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>

* fix copy-paste error

* Update images/macos/provision/core/openjdk.sh

Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>

* Update images/macos/provision/core/openjdk.sh

Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>

---------

Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>

* Updating readme file for ubuntu22 version 20231025.1.1 (#8663)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* Updating readme file for ubuntu20 version 20231025.1.1 (#8664)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* Updating readme file for win22 version 20231024.1.1 (#8668)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* Windows Server 2019 (20231025) Image Update (#8667)

* Updating readme file for win19 version 20231025.2.1

* Update Windows2019-Readme.md

---------

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Mikhail Koliada <88318005+mikhailkoliada@users.noreply.github.com>
Co-authored-by: Actions service account <no-reply@github.com>

* [Mac OS] Fix logic for retrying AppleScript execution (#8662)

* [Ubuntu] Add PGP validation for swift (#8652)

* [Windows] Remove out-of-support vs components (#8678)

* [MacOS] Unpin Virtualbox (#8653)

* [MacOS] Unpin Virtualbox

* Remove obsolete virtualbox kext pester test

---------

Co-authored-by: sergei-pyshnoi <121864472+sergei-pyshnoi@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>
Co-authored-by: ilia-shipitsin <125650415+ilia-shipitsin@users.noreply.github.com>
Co-authored-by: Mikhail Koliada <88318005+mikhailkoliada@users.noreply.github.com>

Author: 7 people

images/linux/Ubuntu2004-Readme.md

@@ -15,6 +15,19 @@ swift_tar_name="swift-$swift_version-RELEASE-ubuntu$image_label.tar.gz"
 swift_tar_url="https://swift.org/builds/swift-$swift_version-release/ubuntu${image_label//./}/swift-$swift_version-RELEASE/$swift_tar_name"
 download_with_retries $swift_tar_url "/tmp" "$swift_tar_name"
 
+# Verifing pgp signature using official swift pgp key. Referring to https://www.swift.org/install/linux/#Installation-via-Tarball
+# Download swift pgp key
+download_with_retries "https://swift.org/keys/all-keys.asc" "/tmp" "all-keys.asc"
+# Import swift pgp key
+gpg --no-default-keyring --keyring swift --import /tmp/all-keys.asc
+# Download signature file
+download_with_retries "$swift_tar_url.sig" "/tmp" "$swift_tar_name.sig"
+# Verify signature
+gpg --no-default-keyring --keyring swift --verify "/tmp/$swift_tar_name.sig" "/tmp/$swift_tar_name"
+# Remove swift pgp public key with temporary keyring
+rm ~/.gnupg/swift
+
+
 tar xzf /tmp/$swift_tar_name
 
 SWIFT_INSTALL_ROOT="/usr/share/swift"

images/linux/Ubuntu2204-Readme.md

@@ -126,7 +126,7 @@ sdkTools="android-sdk-tools.zip"
 sdkToolsVersion=$(get_toolset_value '.android."sdk-tools"')
 if [ "$sdkToolsVersion" != "null" ]; then
     download_with_retries "https://dl.google.com/android/repository/${sdkToolsVersion}" "." $sdkTools
-    unzip -qq $sdkTools -d ${ANDROID_SDK_ROOT}
+    unzip -o -qq $sdkTools -d ${ANDROID_SDK_ROOT}
     rm -f $sdkTools
 fi
 

images/linux/scripts/installers/swift.sh

@@ -14,7 +14,8 @@ echo "Google Chrome version is $FULL_CHROME_VERSION"
 # Get Google Chrome versions information
 CHROME_PLATFORM="mac-$arch"
 CHROME_VERSIONS_URL="https://googlechromelabs.github.io/chrome-for-testing/latest-patch-versions-per-build-with-downloads.json"
-CHROME_VERSIONS_JSON=$(curl -fsSL "${CHROME_VERSIONS_URL}")
+download_with_retries "$CHROME_VERSIONS_URL" "/tmp" "latest-patch-versions-per-build-with-downloads.json"
+CHROME_VERSIONS_JSON=$(cat /tmp/latest-patch-versions-per-build-with-downloads.json)
 
 # Download and unpack the latest release of Chrome Driver
 CHROMEDRIVER_VERSION=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].version')

images/macos/macos-13-Readme.md

@@ -13,18 +13,7 @@ done
 cask_packages=$(get_toolset_value '.brew.cask_packages[]')
 for package in $cask_packages; do
     echo "Installing $package..."
-    if [[ $package == "virtualbox" ]]; then
-        if ! is_Ventura || ! is_VenturaArm64; then
-            # VirtualBox 7 crashes
-            # macOS host: Dropped all kernel extensions. VirtualBox relies fully on the hypervisor and vmnet frameworks provided by Apple now.
-            vbcask_url="https://raw.githubusercontent.com/Homebrew/homebrew-cask/aa3c55951fc9d687acce43e5c0338f42c1ddff7b/Casks/virtualbox.rb"
-            download_with_retries $vbcask_url
-            brew install ./virtualbox.rb
-            rm ./virtualbox.rb
-        fi
-    else
-        brew install --cask $package
-    fi
+    brew install --cask $package
 done
 
 # Load "Parallels International GmbH"
@@ -36,18 +25,24 @@ fi
 # System Preferences -> Security & Privacy -> General -> Unlock -> Allow -> Not now
 if is_Monterey; then
     if is_Veertu; then
-        retry=5
-        while [ $retry -gt 0 ]; do
+        for retry in {4..0}; do
+            echo "Executing AppleScript to change security preferences. Retries left: $retry"
             {
+                set -e
                 osascript -e 'tell application "System Events" to get application processes where visible is true'
-            }
-            osascript $HOME/utils/confirm-identified-developers.scpt $USER_PASSWORD
+                osascript $HOME/utils/confirm-identified-developers.scpt $USER_PASSWORD
+            } && break
+
+            if [ "$retry" -eq 0 ]; then
+                echo "Executing AppleScript failed. No retries left"
+                exit 1
+            fi
 
-            retry=$((retry-1))
-            echo "retries left "$retry
+            echo "Executing AppleScript failed. Sleeping for 10 seconds and retrying"
             sleep 10
         done
     else
+        echo "Executing AppleScript to change security preferences"
         osascript $HOME/utils/confirm-identified-developers.scpt $USER_PASSWORD
     fi
 fi
@@ -82,13 +77,5 @@ bazel
 # Install Azure DevOps extension for Azure Command Line Interface
 az extension add -n azure-devops
 
-# Workaround https://github.com/actions/runner-images/issues/4931
-# by making Tcl/Tk paths the same on macOS 10.15 and macOS 11
-if is_Monterey; then
-    version=$(brew info tcl-tk --json | jq -r '.[].installed[].version')
-    ln -s /usr/local/Cellar/tcl-tk/$version/lib/libtcl8.6.dylib /usr/local/lib/libtcl8.6.dylib
-    ln -s /usr/local/Cellar/tcl-tk/$version/lib/libtk8.6.dylib /usr/local/lib/libtk8.6.dylib
-fi
-
 # Invoke tests for all basic tools
 invoke_tests "BasicTools"

images/macos/macos-13-arm64-Readme.md

@@ -1,9 +1,11 @@
 #!/bin/bash -e -o pipefail
 
-MINICONDA_INSTALLER="/tmp/miniconda.sh"
-curl -fsSL https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o $MINICONDA_INSTALLER
-chmod +x $MINICONDA_INSTALLER
-sudo $MINICONDA_INSTALLER -b -p /usr/local/miniconda
+source ~/utils/utils.sh
+
+download_with_retries "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" "/tmp" "miniconda.sh"
+
+chmod +x /tmp/miniconda.sh
+sudo /tmp/miniconda.sh -b -p /usr/local/miniconda
 
 # Chmod with full permissions recursively to avoid permissions restrictions
 sudo chmod -R 777 /usr/local/miniconda

images/macos/provision/core/android-toolsets.sh

@@ -29,12 +29,12 @@ installOpenJDK() {
     local JAVA_VERSION=$1
 
     # Get link for Java binaries and Java version
-    assetUrl=$(curl -fsSL "https://api.adoptium.net/v3/assets/latest/${JAVA_VERSION}/hotspot")
+    download_with_retries "https://api.adoptium.net/v3/assets/latest/${JAVA_VERSION}/hotspot" "/tmp" "openjdk-hotspot.json"
 
     if [[ $arch == "arm64" ]]; then
-        asset=$(echo ${assetUrl} | jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="aarch64")')
+        asset=$(jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="aarch64")' /tmp/openjdk-hotspot.json)
     else
-        asset=$(echo ${assetUrl} | jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="x64")')
+        asset=$(jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="x64")' /tmp/openjdk-hotspot.json)
     fi
 
     archivePath=$(echo ${asset} | jq -r '.binary.package.link')