Reboot the OSX machine when cannot find Apple Simulator

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.apple.sh

@@ -37,14 +37,17 @@ sudo launchctl asuser "$helix_runner_uid" sh ./xharness-runner.apple.sh \
 
 exit_code=$?
 
-# This handles an issue where Simulators get reeaally slow and they start failing to install apps
-# The only solution is to reboot the machine, so we request a work item retry + MacOS reboot when this happens
-# 123 - timeout in installation on Simulators
-installation_timeout_exit_code=123
-if [ $exit_code -eq $installation_timeout_exit_code ]; then
-    # Since we run the payload script using launchctl, env vars are not set there and we have to do this part here
-    "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because iOS Simulator application install hung')"
-    "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('Rebooting because iOS Simulator application install hung ')"
+# For some of the issues such as Simulators get reeaally slow and they start failing to install apps,
+# the only solution is to reboot the machine.
+# We usually also ask the work item to be re-tried on a different machine
+# Since we run the payload script using launchctl, env vars such as PYTHON_PATH are not set there and we have to do this part here
+# We signal this by creating files
+if [ -f './.retry' ]; then
+    "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_infra_retry; request_infra_retry('Retrying because XHarness ended with $exit_code')"
+fi
+
+if [ -f './.reboot' ]; then
+    "$HELIX_PYTHONPATH" -c "from helix.workitemutil import request_reboot; request_reboot('Rebooting because XHarness ended with $exit_code')"
 fi
 
 exit $exit_code

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh

@@ -145,16 +145,27 @@ alias xharness="dotnet exec $xharness_cli_path"
 source command.sh
 exit_code=$?
 
+# Exit code values - https://github.com/dotnet/xharness/blob/main/src/Microsoft.DotNet.XHarness.Common/CLI/ExitCode.cs
+
 # Kill the simulator just in case when we fail to launch the app
 # 80 - app crash
 if [ $exit_code -eq 80 ]; then
     sudo pkill -9 -f "$simulator_app"
 fi
 
+# If we fail to find a simulator and we are not targeting a specific version (e.g. `ios-simulator_13.5`), it is probably an issue because Xcode should always have at least one runtime version inside
+# 81 - simulator/device not found
+if [ $exit_code -eq 81 ] && [[ "$targets" =~ "simulator" ]] && [[ ! "$targets" =~ "_" ]]; then
+    touch './.retry'
+    touch './.reboot'
+fi
+
 # If we have a launch failure AND we are on simulators, we need to signal that we want a reboot+retry
 # The script that is running this one will notice and request Helix to do it
+# 83 - app launch failure
 if [ $exit_code -eq 83 ] && [[ "$targets" =~ "simulator" ]]; then
-    exit_code=123
+    touch './.retry'
+    touch './.reboot'
 fi
 
 # The simulator logs comming from the sudo-spawned Simulator.app are not readable by the helix uploader
@@ -163,20 +174,9 @@ chmod 0644 "$output_directory"/*.log
 # Remove empty files
 find "$output_directory" -name "*.log" -maxdepth 1 -size 0 -print -delete
 
-if [ "$command" == 'test' ]; then
-    test_results=$(ls "$output_directory"/xunit-*.xml)
-
-    if [ ! -f "$test_results" ]; then
-        echo "Failed to find xUnit tests results in the output directory. Existing files:"
-        ls -la "$output_directory"
-
-        if [ $exit_code -eq 0 ]; then
-            exit_code=5
-        fi
-
-        exit $exit_code
-    fi
-
+# Rename test result XML so that AzDO reporter recognizes it
+test_results=$(ls "$output_directory"/xunit-*.xml)
+if [ -f "$test_results" ]; then
     echo "Found test results in $output_directory/$test_results. Renaming to testResults.xml to prepare for Helix upload"
 
     # Prepare test results for Helix to pick up

tests/XHarness/XHarness.Simulator.AppleRun.proj

@@ -21,7 +21,7 @@
     <ItemGroup>
       <XHarnessAppBundleToTest Include="$(ArtifactsTmpDir)XHarness.RunAppBundle/$(XHarnessRunAppBundleName)">
         <Targets>ios-simulator-64</Targets>
-        <WorkItemTimeout>00:10:00</WorkItemTimeout>
+        <WorkItemTimeout>00:12:00</WorkItemTimeout>
         <TestTimeout>00:08:00</TestTimeout>
         <LaunchTimeout>00:05:00</LaunchTimeout>
         <IncludesTestRunner>false</IncludesTestRunner>

tests/XHarness/XHarness.Simulator.AppleTest.proj

@@ -22,7 +22,7 @@
       <XHarnessAppBundleToTest Include="$(ArtifactsTmpDir)XHarness.TestAppBundle/$(XHarnessTestAppBundleName)">
         <Targets>ios-simulator-64</Targets>
         <TestTimeout>00:05:00</TestTimeout>
-        <WorkItemTimeout>00:12:00</WorkItemTimeout>
+        <WorkItemTimeout>00:15:00</WorkItemTimeout>
         <LaunchTimeout>00:06:00</LaunchTimeout>
       </XHarnessAppBundleToTest>
     </ItemGroup>

tests/XHarness/XHarness.Simulator.CustomCommands.proj

@@ -21,7 +21,7 @@
     <ItemGroup>
       <XHarnessAppBundleToTest Include="$(ArtifactsTmpDir)XHarness.CustomCommandsAppBundle/$(XHarnessAppBundleName).app">
         <Targets>ios-simulator-64</Targets>
-        <WorkItemTimeout>00:12:00</WorkItemTimeout>
+        <WorkItemTimeout>00:15:00</WorkItemTimeout>
         <TestTimeout>00:05:00</TestTimeout>
         <LaunchTimeout>00:05:00</LaunchTimeout>
         <CustomCommands>