Sync changes to macos-new-folder-structure branch (#8765)

* New folder structure

* Update packer templates

* Update refs in scripts

* Updating readme file for macOS-12 version 20231029.1 (#8695)

Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>

* [Windows] Remove Android SDK Patch applier v4 (#8750)

* remove ndk 23 from macos images (#8747)

* [Mac OS] Remove 'LTS' suffix from OpenJDK version (#8751)

* [windows] split docker install into 3 scripts (#8688)

* [windows] split docker install into 3 scripts

the idea is to end with 1 script per 1 component for better
observability

* Update images/win/scripts/Installers/Install-Docker-Compose.ps1

Co-authored-by: Erik Bershel <110455084+erik-bershel@users.noreply.github.com>

* Update images/win/scripts/Installers/Install-Docker-WinCred.ps1

Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>

* fix test

---------

Co-authored-by: Erik Bershel <110455084+erik-bershel@users.noreply.github.com>
Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>

* remove ndk 23 from ubuntu images (#8745)

* [macos] add android SDK to SW report on Ventura ARM64 (#8715)

* Fix utils path in templates

* Sync changes

* [Mac OS] Update Anka Helpers to build Mac OS 14 ARM image (#8757)

* Update Anka Helpers to build Mac OS 14 ARM image

* Drop Powershell 5 support

* [Mac OS] Add missing macOS Sonoma selector (#8759)

* remove ndk23 from windows images (#8746)

* Updating readme file for macos-13 version 20231025.2 (#8672)

Co-authored-by: Image generation service account <no-reply@microsoft.com>

* Fix macOS-12.anka template with assets path

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Image generation service account <no-reply@microsoft.com>
Co-authored-by: Actions service account <no-reply@github.com>
Co-authored-by: Erik Bershel <110455084+erik-bershel@users.noreply.github.com>
Co-authored-by: sergei-pyshnoi <121864472+sergei-pyshnoi@users.noreply.github.com>
Co-authored-by: Vasilii Polikarpov <126792224+vpolikarpov-akvelon@users.noreply.github.com>
Co-authored-by: ilia-shipitsin <125650415+ilia-shipitsin@users.noreply.github.com>

Author: 8 people

images.CI/macos/anka/Service.Helpers.psm1

@@ -63,24 +63,24 @@ function Get-AvailableIPSWVersions {
     )
 
     if ($IsBeta) {
-        $command = { mist list firmware "$MacOSCodeNameOrVersion" --include-betas --latest --export "/Applications/export.json"}
+        $command = { mist list firmware "$MacOSCodeNameOrVersion" --compatible --include-betas --latest --export "/Applications/export.json" }
     } elseif ($IsLatest) {
-        $command = { mist list firmware "$MacOSCodeNameOrVersion" --latest  --export "/Applications/export.json" }
+        $command = { mist list firmware "$MacOSCodeNameOrVersion" --compatible --latest --export "/Applications/export.json" }
     } else {
-        $command = { mist list firmware "$MacOSCodeNameOrVersion"  --export "/Applications/export.json" }
+        $command = { mist list firmware "$MacOSCodeNameOrVersion" --compatible --export "/Applications/export.json" }
     }
 
     $condition = { $LASTEXITCODE -eq 0 }
     Invoke-WithRetry -Command $command -BreakCondition $condition | Out-Null
     $softwareList = get-content -Path "/Applications/export.json"
-    $turgetVersion = ($softwareList | ConvertFrom-Json).version
-    if ($null -eq $turgetVersion) {
+    $availableBuilds = ($softwareList | ConvertFrom-Json).build
+    if ($null -eq $availableBuilds) {
         Write-Host "Requested macOS '$MacOSCodeNameOrVersion' version not found in the list of available installers."
         $command = { mist list firmware "$($MacOSCodeNameOrVersion.split('.')[0])" }
         Invoke-WithRetry -Command $command -BreakCondition $condition
         exit 1
     }
-    return $turgetVersion
+    return $availableBuilds
 }
 
 function Get-MacOSIPSWInstaller {
@@ -96,21 +96,26 @@ function Get-MacOSIPSWInstaller {
         $MacOSName = "macOS Monterey"
     } elseif ($MacOSVersion -eq [version] "13.0") {
         $MacOSName = "macOS Ventura"
+    } elseif ($MacOSVersion -eq [version] "14.0") {
+        $MacOSName = "macOS Sonoma"
     } else {
         $MacOSName = $MacOSVersion.ToString()
     }
 
-
     Write-Host "`t[*] Finding available full installers"
     if ($DownloadLatestVersion -eq $true) {
-        $targetVersion = Get-AvailableIPSWVersions -IsLatest $true -MacOSCodeNameOrVersion $MacOSName
-        Write-host "`t[*] The 'DownloadLatestVersion' flag is set to true. Latest macOS version is '$MacOSName' - '$targetVersion' now"
+        $targetBuild = Get-AvailableIPSWVersions -IsLatest $true -MacOSCodeNameOrVersion $MacOSName
+        Write-Host "`t[*] The 'DownloadLatestVersion' flag is set to true. Latest compatible macOS build of '$MacOSName' is '$targetBuild'"
     } elseif ($BetaSearch -eq $true) {
-        $targetVersion = Get-AvailableIPSWVersions -IsBeta $true -MacOSCodeNameOrVersion $MacOSName
-        Write-host "`t[*] The 'BetaSearch' flag is set to true. Latestbeta macOS version is '$MacOSName' - '$targetVersion' now"
+        $targetBuild = Get-AvailableIPSWVersions -IsBeta $true -MacOSCodeNameOrVersion $MacOSName
+        Write-Host "`t[*] The 'BetaSearch' flag is set to true. Latest compatible beta macOS build of '$MacOSName' is '$targetBuild'"
     } else {
-        $targetVersion = Get-AvailableIPSWVersions -MacOSCodeNameOrVersion $MacOSName -IsLatest $false
-        Write-host "`t[*] The exact version was specified - '$MacOSName' "
+        $targetBuild = Get-AvailableIPSWVersions -MacOSCodeNameOrVersion $MacOSName -IsLatest $false
+        Write-Host "`t[*] Available compatible macOS builds of '$MacOSName' are: $($targetBuild -join ', ')"
+        if ($targetBuild.Count -gt 1) {
+            Write-Error "`t[*] Please specify the exact build number of macOS you want to install"
+            exit 1
+        }
     }
 
     $installerPathPattern = "/Applications/Install ${macOSName}*.ipsw"
@@ -123,12 +128,12 @@ function Get-MacOSIPSWInstaller {
     # Download macOS installer
     $installerDir = "/Applications/"
     $installerName = "Install ${macOSName}.ipsw"
-    Write-Host "`t[*] Requested macOS '$targetVersion' version installer found, fetching it from mist database"
-    Invoke-WithRetry { mist download firmware "$targetVersion" --output-directory $installerDir --firmware-name "$installerName" } {$LASTEXITCODE -eq 0} | Out-Null
+    Write-Host "`t[*] Requested macOS '$targetBuild' version installer found, fetching it from mist database"
+    Invoke-WithRetry { mist download firmware "$targetBuild" --output-directory $installerDir --firmware-name "$installerName" } { $LASTEXITCODE -eq 0 } | Out-Null
     if (Test-Path "$installerDir$installerName") {
         $result = "$installerDir$installerName"
     } else {
-        Write-host "`t[*] Requested macOS '$targetVersion' version installer failed to download"
+        Write-Error "`t[*] Requested macOS '$targetBuild' version installer failed to download"
         exit 1
     }
     return $result
@@ -169,7 +174,7 @@ function Get-MacOSInstaller {
             exit 1
         }
         Show-StringWithFormat $filteredVersions
-        $osVersions = $filteredVersions.OSVersion | Sort-Object {[version]$_}
+        $osVersions = $filteredVersions.OSVersion | Sort-Object { [version]$_ }
         $MacOSVersion = $osVersions | Select-Object -Last 1
         Write-Host "`t[*] The 'DownloadLatestVersion' flag is set. Latest macOS version is '$MacOSVersion' now"
     }
@@ -248,15 +253,15 @@ function Install-SoftwareUpdate {
     $command = "sw_vers"
     $guestMacosVersion = Invoke-SSHPassCommand -HostName $HostName -Command $command
     if ($guestMacosVersion[1] -match "12") {
-        foreach ($update in $listOfUpdates){
+        foreach ($update in $listOfUpdates) {
             # Filtering updates that contain "Ventura" word
             if ($update -notmatch "Ventura") {
                 $command = "sudo /usr/sbin/softwareupdate --restart --verbose --install '$($update.trim())'"
                 Invoke-SSHPassCommand -HostName $HostName -Command $command
             }
         }
     } elseif ($guestMacosVersion[1] -match "13") {
-        foreach ($update in $listOfUpdates){
+        foreach ($update in $listOfUpdates) {
             # Filtering updates that contain "Sonoma" word
             if ($update -notmatch "Sonoma") {
                 $command = "sudo /usr/sbin/softwareupdate --restart --verbose --install '$($update.trim())'"
@@ -302,7 +307,11 @@ function Invoke-SSHPassCommand {
         "${env:SSHUSER}@${HostName}"
     )
     $sshPassOptions = $sshArg -join " "
-    $result = bash -c "$sshPassOptions \""$Command\"" 2>&1"
+    if ($PSVersionTable.PSVersion.Major -eq 7 -and $PSVersionTable.PSVersion.Minor -le 2) {
+        $result = bash -c "$sshPassOptions \""$Command\"" 2>&1"
+    } else {
+        $result = bash -c "$sshPassOptions `"$Command`" 2>&1"
+    }
     if ($LASTEXITCODE -ne 0) {
         Write-Error "There is an error during command execution:`n$result"
         exit 1

images/linux/scripts/tests/Android.Tests.ps1

@@ -33,7 +33,6 @@ Describe "Android" {
             # Convert 'm2repository;com;android;support;constraint;constraint-layout-solver;1.0.0-beta1' ->
             #         'm2repository/com/android/support/constraint/constraint-layout-solver/1.0.0-beta1'
             #         'cmake;3.6.4111459' -> 'cmake/3.6.4111459'
-            #         'patcher;v4' -> 'patcher/v4'
             $PackageName = $PackageName.Replace(";", "/")
             $targetPath = Join-Path $env:ANDROID_HOME $PackageName
             $targetPath | Should -Exist
@@ -65,4 +64,4 @@ Describe "Android" {
             Validate-AndroidPackage $PackageName
         }
     }
-}
+}

images/linux/toolsets/toolset-2004.json

@@ -97,7 +97,7 @@
         "ndk": {
             "default": "25",
             "versions": [
-                "23", "24", "25", "26"
+                "24", "25", "26"
             ]
         }
     },

images/linux/toolsets/toolset-2204.json

@@ -91,7 +91,7 @@
         "ndk": {
             "default": "25",
             "versions": [
-                "23", "24", "25", "26"
+                "24", "25", "26"
             ]
         }
     },