Add ARM64 packer template, update scripts for multiple architectures (#2

)

* Add ARM64 packer template, update scripts for multiple architectures

* Update AMI creation instance type

* Remove debug output from software report

images/ubuntu/scripts/build/Configure-Toolset.ps1

@@ -6,6 +6,8 @@
 
 Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1"
 
+$arch = Get-Architecture
+
 function Get-TCToolVersionPath {
     param(
         [Parameter(Mandatory)]
@@ -19,6 +21,9 @@ function Get-TCToolVersionPath {
     $toolPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath $ToolName
     $toolPathVersion = Join-Path -Path $toolPath -ChildPath $ToolVersion
     $foundVersion = Get-Item $toolPathVersion | Sort-Object -Property { [version] $_.name } -Descending | Select-Object -First 1
+    if (-not $foundVersion) {
+        return $null
+    }
     $installationDir = Join-Path -Path $foundVersion -ChildPath $ToolArchitecture
 
     return $installationDir
@@ -40,10 +45,11 @@ function Add-GlobalEnvironmentVariable {
 $ErrorActionPreference = "Stop"
 
 Write-Host "Configure toolcache tools environment..."
+$toolEnvArch = ($arch -eq "arm64") ? "ARM64" : "X64"
 $toolEnvConfigs = @{
     go = @{
         command          = "ln -s {0}/bin/* /usr/bin/"
-        variableTemplate = "GOROOT_{0}_{1}_X64"
+        variableTemplate = "GOROOT_{0}_{1}_${toolEnvArch}"
     }
 }
 
@@ -56,7 +62,11 @@ foreach ($tool in $tools) {
     if (-not ([string]::IsNullOrEmpty($toolEnvConfig.variableTemplate))) {
         foreach ($toolVersion in $tool.versions) {
             Write-Host "Set $($tool.name) $toolVersion environment variable..."
-            $toolPath = Get-TCToolVersionPath -ToolName $tool.name -ToolVersion $toolVersion -ToolArchitecture $tool.arch
+            $toolPath = Get-TCToolVersionPath -ToolName $tool.name -ToolVersion $toolVersion -ToolArchitecture $arch
+            if (-not $toolPath) {
+                Write-Host "Tool $($tool.name) $toolVersion not found in toolcache for $arch"
+                continue
+            }
             $envVariableName = $toolEnvConfig.variableTemplate -f $toolVersion.split(".")
 
             Add-GlobalEnvironmentVariable -Name $envVariableName -Value $toolPath
@@ -65,7 +75,11 @@ foreach ($tool in $tools) {
 
     # Invoke command and add env variable for the default tool version
     if (-not ([string]::IsNullOrEmpty($tool.default))) {
-        $toolDefaultPath = Get-TCToolVersionPath -ToolName $tool.name -ToolVersion $tool.default -ToolArchitecture $tool.arch
+        $toolDefaultPath = Get-TCToolVersionPath -ToolName $tool.name -ToolVersion $tool.default -ToolArchitecture $arch
+        if (-not $toolDefaultPath) {
+            Write-Host "Tool $($tool.name) $($tool.default) not found in toolcache for $arch"
+            continue
+        }
 
         if (-not ([string]::IsNullOrEmpty($toolEnvConfig.defaultVariable))) {
             Write-Host "Set default $($toolEnvConfig.defaultVariable) for $($tool.name) $($tool.default) environment variable..."

images/ubuntu/scripts/build/Install-Toolset.ps1

@@ -5,6 +5,15 @@
 ################################################################################
 
 Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1"
+$arch = Get-Architecture
+
+if ($arch -eq "amd64") {
+    $assetArchs = @("x64", "amd64")
+}
+
+if ($arch -eq "arm64") {
+    $assetArchs = @("arm64", "aarch64")
+}
 
 function Install-Asset {
     param(
@@ -39,15 +48,15 @@ foreach ($tool in $tools) {
     foreach ($toolVersion in $tool.versions) {
         $asset = $assets | Where-Object version -like $toolVersion `
             | Select-Object -ExpandProperty files `
-            | Where-Object { ($_.platform -eq $tool.platform) -and ($_.arch -eq $tool.arch) -and ($_.platform_version -eq $tool.platform_version)} `
+            | Where-Object { ($_.platform -eq $tool.platform) -and ($_.platform_version -eq $tool.platform_version) -and ($assetArchs -contains $_.arch) } `
             | Select-Object -First 1
 
         if (-not $asset) {
-            Write-Host "Asset for $($tool.name) $toolVersion $($tool.arch) not found in versions manifest"
-            exit 1
+            Write-Host "Asset for $($tool.name) $toolVersion $($arch) not found in versions manifest"
+            continue
         }
 
-        Write-Host "Installing $($tool.name) $toolVersion $($tool.arch)..."
+        Write-Host "Installing $($tool.name) $toolVersion $($arch)..."
         Install-Asset -ReleaseAsset $asset
     }
 

images/ubuntu/scripts/build/install-aliyun-cli.sh

@@ -9,23 +9,25 @@
 source $HELPER_SCRIPTS/os.sh
 source $HELPER_SCRIPTS/install.sh
 
+arch=$(get_arch)
+
 # Install Alibaba Cloud CLI
 # Pin tool version on ubuntu20 due to issues with GLIBC_2.32 not available
 if is_ubuntu20; then
     toolset_version=$(get_toolset_value '.aliyunCli.version')
-    download_url="https://github.com/aliyun/aliyun-cli/releases/download/v$toolset_version/aliyun-cli-linux-$toolset_version-amd64.tgz"
+    download_url="https://github.com/aliyun/aliyun-cli/releases/download/v$toolset_version/aliyun-cli-linux-$toolset_version-$arch.tgz"
 else
-    download_url=$(resolve_github_release_asset_url "aliyun/aliyun-cli" "contains(\"aliyun-cli-linux\") and endswith(\"amd64.tgz\")" "latest")
+    download_url=$(resolve_github_release_asset_url "aliyun/aliyun-cli" "contains(\"aliyun-cli-linux\") and endswith(\"$arch.tgz\")" "latest")
     hash_url="https://github.com/aliyun/aliyun-cli/releases/latest/download/SHASUMS256.txt"
 fi
 
 archive_path=$(download_with_retry "$download_url")
 
 # Supply chain security - Alibaba Cloud CLI
 if is_ubuntu20; then
-    external_hash=$(get_toolset_value '.aliyunCli.sha256')
+    external_hash=$(get_toolset_value ".aliyunCli.sha256.$arch")
 else
-    external_hash=$(get_checksum_from_url "$hash_url" "aliyun-cli-linux.*amd64.tgz" "SHA256")
+    external_hash=$(get_checksum_from_url "$hash_url" "aliyun-cli-linux.*$arch.tgz" "SHA256")
 fi
 
 use_checksum_comparison "$archive_path" "$external_hash"

images/ubuntu/scripts/build/install-android-sdk.sh

@@ -9,6 +9,8 @@ source $HELPER_SCRIPTS/os.sh
 source $HELPER_SCRIPTS/install.sh
 source $HELPER_SCRIPTS/etc-environment.sh
 
+arch=$(get_arch)
+
 add_filtered_installation_components() {
     local minimum_version=$1
     shift
@@ -87,6 +89,7 @@ ndk_default_full_version=$(get_full_ndk_version $android_ndk_major_default)
 ndk_latest_full_version=$(get_full_ndk_version $android_ndk_major_latest)
 ANDROID_NDK=${ANDROID_SDK_ROOT}/ndk/${ndk_default_full_version}
 # ANDROID_NDK, ANDROID_NDK_HOME, and ANDROID_NDK_ROOT variables should be set as many customer builds depend on them https://github.com/actions/runner-images/issues/5879
+set_etc_environment_variable "ANDROID_ARCH" "${arch}"
 set_etc_environment_variable "ANDROID_NDK" "${ANDROID_NDK}"
 set_etc_environment_variable "ANDROID_NDK_HOME" "${ANDROID_NDK}"
 set_etc_environment_variable "ANDROID_NDK_ROOT" "${ANDROID_NDK}"
@@ -98,6 +101,11 @@ addons=$(get_toolset_value '.android.addon_list[] | "add-ons;" + .')
 additional=$(get_toolset_value '.android.additional_tools[]')
 components=("${extras[@]}" "${addons[@]}" "${additional[@]}")
 
+# ARM64 cant handle additional components
+if [[ $arch == "arm64" ]]; then
+    components=("${extras[@]}" "${addons[@]}")
+fi
+
 for ndk_major_version in "${android_ndk_major_versions[@]}"; do
     ndk_full_version=$(get_full_ndk_version $ndk_major_version)
     components+=("ndk;$ndk_full_version")

images/ubuntu/scripts/build/install-aws-tools.sh

@@ -9,15 +9,29 @@
 source $HELPER_SCRIPTS/os.sh
 source $HELPER_SCRIPTS/install.sh
 
-awscliv2_archive_path=$(download_with_retry "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip")
+arch=$(get_arch)
+
+if [[ $arch == "amd64" ]]; then
+    arch="x86_64"
+    sam_arch="x86_64"
+    plugin_arch="64bit"
+fi
+
+if [[ $arch == "arm64" ]]; then
+    arch="aarch64"
+    sam_arch="arm64"
+    plugin_arch="arm64"
+fi
+
+awscliv2_archive_path=$(download_with_retry "https://awscli.amazonaws.com/awscli-exe-linux-$arch.zip")
 unzip -qq "$awscliv2_archive_path" -d /tmp
 /tmp/aws/install -i /usr/local/aws-cli -b /usr/local/bin
 
-smplugin_deb_path=$(download_with_retry "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb")
+smplugin_deb_path=$(download_with_retry "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_$plugin_arch/session-manager-plugin.deb")
 apt install "$smplugin_deb_path"
 
 # Download the latest aws sam cli release
-aws_sam_cli_archive_name="aws-sam-cli-linux-x86_64.zip"
+aws_sam_cli_archive_name="aws-sam-cli-linux-$sam_arch.zip"
 sam_cli_download_url=$(resolve_github_release_asset_url "aws/aws-sam-cli" "endswith(\"$aws_sam_cli_archive_name\")" "latest")
 aws_sam_cli_archive_path=$(download_with_retry "$sam_cli_download_url")
 

images/ubuntu/scripts/build/install-azcopy.sh

@@ -6,9 +6,17 @@
 
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/install.sh
+source $HELPER_SCRIPTS/os.sh
+
+arch=$(get_arch)
+suffix=""
+
+if [[ $arch == "arm64" ]]; then
+    suffix="-arm64"
+fi
 
 # Install AzCopy10
-archive_path=$(download_with_retry "https://aka.ms/downloadazcopy-v10-linux")
+archive_path=$(download_with_retry "https://aka.ms/downloadazcopy-v10-linux$suffix")
 tar xzf "$archive_path" --strip-components=1 -C /tmp
 install /tmp/azcopy /usr/local/bin/azcopy
 

images/ubuntu/scripts/build/install-bicep.sh

@@ -6,9 +6,16 @@
 
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/install.sh
+source $HELPER_SCRIPTS/os.sh
+
+arch=$(get_arch)
+
+if [[ $arch == "amd64" ]]; then
+    arch="x64"
+fi
 
 # Install Bicep CLI
-download_url=$(resolve_github_release_asset_url "Azure/bicep" "endswith(\"bicep-linux-x64\")" "latest")
+download_url=$(resolve_github_release_asset_url "Azure/bicep" "endswith(\"bicep-linux-$arch\")" "latest")
 bicep_binary_path=$(download_with_retry "${download_url}")
 
 # Mark it as executable

images/ubuntu/scripts/build/install-cmake.sh

@@ -7,19 +7,30 @@
 
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/install.sh
+source $HELPER_SCRIPTS/os.sh
+
+arch=$(get_arch)
+
+if [[ $arch == "amd64" ]]; then
+    arch="x86_64"
+fi
+
+if [[ $arch == "arm64" ]]; then
+    arch="aarch64"
+fi
 
 # Test to see if the software in question is already installed, if not install it
 echo "Checking to see if the installer script has already been run"
 if command -v cmake; then
     echo "cmake is already installed"
 else
 	# Download script to install CMake
-	download_url=$(resolve_github_release_asset_url "Kitware/CMake" "endswith(\"inux-x86_64.sh\")" "latest")
+	download_url=$(resolve_github_release_asset_url "Kitware/CMake" "endswith(\"inux-$arch.sh\")" "latest")
 	curl -fsSL "${download_url}" -o cmakeinstall.sh
 
 	# Supply chain security - CMake
 	hash_url=$(resolve_github_release_asset_url "Kitware/CMake" "endswith(\"SHA-256.txt\")" "latest")
-	external_hash=$(get_checksum_from_url "$hash_url" "linux-x86_64.sh" "SHA256")
+	external_hash=$(get_checksum_from_url "$hash_url" "linux-$arch.sh" "SHA256")
 	use_checksum_comparison "cmakeinstall.sh" "$external_hash"
 
 	# Install CMake and remove the install script

images/ubuntu/scripts/build/install-codeql-bundle.sh

@@ -6,6 +6,9 @@
 
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/install.sh
+source $HELPER_SCRIPTS/os.sh
+
+arch=$(get_arch)
 
 # Retrieve the CLI version of the latest CodeQL bundle.
 base_url="$(curl -fsSL https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json)"
@@ -17,7 +20,7 @@ echo "Downloading CodeQL bundle $bundle_version..."
 # different operating systems within containers.
 codeql_archive=$(download_with_retry "https://github.com/github/codeql-action/releases/download/$bundle_tag_name/codeql-bundle.tar.gz")
 
-codeql_toolcache_path="$AGENT_TOOLSDIRECTORY/CodeQL/$bundle_version/x64"
+codeql_toolcache_path="$AGENT_TOOLSDIRECTORY/CodeQL/$bundle_version/$arch"
 mkdir -p "$codeql_toolcache_path"
 
 echo "Unpacking the downloaded CodeQL bundle archive..."

images/ubuntu/scripts/build/install-container-tools.sh

@@ -7,6 +7,8 @@
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/os.sh
 
+arch=$(get_arch)
+
 #
 # pin podman due to https://github.com/actions/runner-images/issues/7753
 #                   https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394
@@ -24,7 +26,7 @@ if is_ubuntu20; then
     REPO_PATH="/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list"
 
     curl -fsSL "${REPO_URL}/Release.key" | gpg --dearmor -o $GPG_KEY
-    echo "deb [arch=amd64 signed-by=$GPG_KEY] ${REPO_URL}/ /" > $REPO_PATH
+    echo "deb [arch=$arch signed-by=$GPG_KEY] ${REPO_URL}/ /" > $REPO_PATH
 fi
 
 # Install podman, buildah, scopeo container's tools

images/ubuntu/scripts/build/install-docker.sh

@@ -7,14 +7,17 @@
 
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/install.sh
+source $HELPER_SCRIPTS/os.sh
+
+arch=$(get_arch)
 
 REPO_URL="https://download.docker.com/linux/ubuntu"
 GPG_KEY="/usr/share/keyrings/docker.gpg"
 REPO_PATH="/etc/apt/sources.list.d/docker.list"
 os_codename=$(lsb_release -cs)
 
 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o $GPG_KEY
-echo "deb [arch=amd64 signed-by=$GPG_KEY] $REPO_URL ${os_codename} stable" > $REPO_PATH
+echo "deb [arch=$arch signed-by=$GPG_KEY] $REPO_URL ${os_codename} stable" > $REPO_PATH
 apt-get update
 
 # Install docker components which available via apt-get
@@ -37,7 +40,7 @@ done
 plugins=$(get_toolset_value '.docker.plugins[] .plugin')
 for plugin in $plugins; do
     version=$(get_toolset_value ".docker.plugins[] | select(.plugin == \"$plugin\") | .version")
-    filter=$(get_toolset_value ".docker.plugins[] | select(.plugin == \"$plugin\") | .asset")
+    filter=$(get_toolset_value ".docker.plugins[] | select(.plugin == \"$plugin\") | .asset.$arch")
     url=$(resolve_github_release_asset_url "docker/$plugin" "endswith(\"$filter\")" "$version")
     binary_path=$(download_with_retry "$url" "/tmp/docker-$plugin")
     mkdir -pv "/usr/libexec/docker/cli-plugins"
@@ -80,7 +83,7 @@ fi
 
 # Download amazon-ecr-credential-helper
 aws_latest_release_url="https://api.github.com/repos/awslabs/amazon-ecr-credential-helper/releases/latest"
-aws_helper_url=$(curl -fsSL "${aws_latest_release_url}" | jq -r '.body' | awk -F'[()]' '/linux-amd64/ {print $2}')
+aws_helper_url=$(curl -fsSL "${aws_latest_release_url}" | jq -r '.body' | awk -F'[()]' '/linux-'"$arch"'/ {print $2}')
 aws_helper_binary_path=$(download_with_retry "$aws_helper_url")
 
 # Supply chain security - amazon-ecr-credential-helper

images/ubuntu/scripts/build/install-dotnetcore-sdk.sh

@@ -9,6 +9,12 @@ source $HELPER_SCRIPTS/etc-environment.sh
 source $HELPER_SCRIPTS/install.sh
 source $HELPER_SCRIPTS/os.sh
 
+arch=$(get_arch)
+
+if [[ $arch == "amd64" ]]; then
+    arch="x64"
+fi
+
 extract_dotnet_sdk() {
     local archive_name=$1
 
@@ -74,13 +80,24 @@ done
 
 sorted_sdks=$(echo ${sdks[@]} | tr ' ' '\n' | sort -r | uniq -w 5)
 
+# Manually install SDK for arm64 in case it's not available in the apt repository
+if [[ $arch == "arm64" && ! -d "/usr/share/dotnet" ]]; then
+    curl -SL -o dotnet.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/Sdk/master/dotnet-sdk-latest-linux-arm64.tar.gz
+    mkdir -p /usr/share/dotnet
+    tar -zxf dotnet.tar.gz -C /usr/share/dotnet
+    rm -f /usr/bin/dotnet
+    ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
+fi
+
+
 # Download/install additional SDKs in parallel
 export -f download_with_retry
 export -f extract_dotnet_sdk
 
 parallel --jobs 0 --halt soon,fail=1 \
-    'url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/{}/dotnet-sdk-{}-linux-x64.tar.gz"; \
-    download_with_retry $url' ::: "${sorted_sdks[@]}"
+    'url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/{}/dotnet-sdk-{}-linux-'"$arch"'.tar.gz"; \
+    path="dotnet-sdk-{}-linux-'"$arch"'.tar.gz"; \
+    download_with_retry $url $path' ::: "${sorted_sdks[@]}"
 
 find . -name "*.tar.gz" | parallel --halt soon,fail=1 'extract_dotnet_sdk {}'
 

images/ubuntu/scripts/build/install-firefox.sh

@@ -7,6 +7,18 @@
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/install.sh
 source $HELPER_SCRIPTS/etc-environment.sh
+source $HELPER_SCRIPTS/os.sh
+
+arch=$(get_arch)
+suffix=""
+
+if [[ $arch == "amd64" ]]; then
+    suffix="64"
+fi
+
+if [[ $arch == "arm64" ]]; then
+    suffix="-aarch64"
+fi
 
 # Mozillateam PPA is added manually because sometimes
 # launchpad portal sends empty answer when trying to add it automatically
@@ -32,7 +44,7 @@ echo "mozillateam $REPO_URL" >> $HELPER_SCRIPTS/apt-sources.txt
 echo 'pref("intl.locale.requested","en_US");' >> "/usr/lib/firefox/browser/defaults/preferences/syspref.js"
 
 # Download and unpack latest release of geckodriver
-download_url=$(resolve_github_release_asset_url "mozilla/geckodriver" "test(\"linux64.tar.gz$\")" "latest")
+download_url=$(resolve_github_release_asset_url "mozilla/geckodriver" "test(\"linux$suffix.tar.gz$\")" "latest")
 driver_archive_path=$(download_with_retry "$download_url")
 
 GECKODRIVER_DIR="/usr/local/share/gecko_driver"