From 8ca453f8cd1102aada94dd09f7a7ac8b34ee18f9 Mon Sep 17 00:00:00 2001
From: Edward Chen <18449977+edgchen1@users.noreply.github.com>
Date: Tue, 3 Oct 2023 00:32:50 -0500
Subject: [PATCH] Address iOS CI build flakiness (#568)

* Add separate step to wait for simulator to boot.

* Add -parallel-testing-enabled NO xcodebuild option.

* Apply changes to ios_packaging.yml.

* quote other display names in same job for consistency
---
 .pipelines/ci.yml            | 24 ++++++++++++++++++------
 .pipelines/ios_packaging.yml | 18 +++++++++++++++---
 2 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml
index 891d297d4..8fe89cf26 100644
--- a/.pipelines/ci.yml
+++ b/.pipelines/ci.yml
@@ -622,35 +622,47 @@ stages:
           --ios_deployment_target 13.0 \
           -- \
           --enable_cxx_tests
-      displayName: Build xcframework for iphonesimulator x86_64
+      displayName: "Build xcframework for iphonesimulator x86_64"
 
     - script: |
         python ./tools/ios/assemble_pod_package.py \
           --staging-dir $(Build.BinariesDirectory)/pod_staging \
           --xcframework-output-dir $(Build.BinariesDirectory)/xcframework_out \
           --pod-version ${ORT_EXTENSIONS_POD_VERSION}
-      displayName: Assemble pod
+      displayName: "Assemble pod"
 
     - script: |
         ORT_EXTENSIONS_LOCAL_POD_PATH=$(Build.BinariesDirectory)/pod_staging \
           pod install
-      displayName: Install pods for OrtExtensionsUsage
+      displayName: "Install pods for OrtExtensionsUsage"
       workingDirectory: $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage
 
     - script: |
         set -e
 
-        SIMULATOR_DEVICE_INFO=$(python tools/ios/get_simulator_device_info.py)
+        SIMULATOR_DEVICE_INFO=$(python ./tools/ios/get_simulator_device_info.py)
+
         echo "Simulator device info:"
         echo "${SIMULATOR_DEVICE_INFO}"
 
         SIMULATOR_DEVICE_ID=$(jq --raw-output '.device_udid' <<< "${SIMULATOR_DEVICE_INFO}")
 
+        # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+        set +x
+        echo "##vso[task.setvariable variable=ORT_EXTENSIONS_SIMULATOR_DEVICE_ID]${SIMULATOR_DEVICE_ID}"
+      displayName: "Get simulator device info"
+
+    - script: |
+        xcrun simctl bootstatus ${ORT_EXTENSIONS_SIMULATOR_DEVICE_ID} -b
+      displayName: "Wait for simulator device to boot"
+
+    - script: |
         xcrun xcodebuild \
           -sdk iphonesimulator \
           -configuration Debug \
+          -parallel-testing-enabled NO \
           -workspace $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage/OrtExtensionsUsage.xcworkspace \
           -scheme OrtExtensionsUsage \
-          -destination "platform=iOS Simulator,id=${SIMULATOR_DEVICE_ID}" \
+          -destination "platform=iOS Simulator,id=${ORT_EXTENSIONS_SIMULATOR_DEVICE_ID}" \
           test CODE_SIGNING_ALLOWED=NO
-      displayName: Build and test OrtExtensionsUsage
+      displayName: "Build and test OrtExtensionsUsage"
diff --git a/.pipelines/ios_packaging.yml b/.pipelines/ios_packaging.yml
index 4a4fecc45..ea7ff0f41 100644
--- a/.pipelines/ios_packaging.yml
+++ b/.pipelines/ios_packaging.yml
@@ -70,20 +70,32 @@ jobs:
   - script: |
       set -e
 
-      SIMULATOR_DEVICE_INFO=$(python tools/ios/get_simulator_device_info.py)
+      SIMULATOR_DEVICE_INFO=$(python ./tools/ios/get_simulator_device_info.py)
+
       echo "Simulator device info:"
       echo "${SIMULATOR_DEVICE_INFO}"
 
       SIMULATOR_DEVICE_ID=$(jq --raw-output '.device_udid' <<< "${SIMULATOR_DEVICE_INFO}")
 
+      # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+      set +x
+      echo "##vso[task.setvariable variable=ORT_EXTENSIONS_SIMULATOR_DEVICE_ID]${SIMULATOR_DEVICE_ID}"
+    displayName: "Get simulator device info"
+
+  - script: |
+      xcrun simctl bootstatus ${ORT_EXTENSIONS_SIMULATOR_DEVICE_ID} -b
+    displayName: "Wait for simulator device to boot"
+
+  - script: |
       xcrun xcodebuild \
         -sdk iphonesimulator \
         -configuration Debug \
+        -parallel-testing-enabled NO \
         -workspace $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage/OrtExtensionsUsage.xcworkspace \
         -scheme OrtExtensionsUsage \
-        -destination "platform=iOS Simulator,id=${SIMULATOR_DEVICE_ID}" \
+        -destination "platform=iOS Simulator,id=${ORT_EXTENSIONS_SIMULATOR_DEVICE_ID}" \
         test CODE_SIGNING_ALLOWED=NO
-    displayName: Build and test OrtExtensionsUsage
+    displayName: "Build and test OrtExtensionsUsage"
 
   - task: InstallAppleCertificate@2
     inputs: