Address iOS CI build flakiness (microsoft#568)

* Add separate step to wait for simulator to boot. * Add -parallel-testing-enabled NO xcodebuild option. * Apply changes to ios_packaging.yml. * quote other display names in same job for consistency
xadupre · Oct 3, 2023 · 8ca453f · 8ca453f
1 parent 9abcda7
commit 8ca453f
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 9 deletions.
diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml
@@ -622,35 +622,47 @@ stages:
           --ios_deployment_target 13.0 \
           -- \
           --enable_cxx_tests
-      displayName: Build xcframework for iphonesimulator x86_64
+      displayName: "Build xcframework for iphonesimulator x86_64"
 
     - script: |
         python ./tools/ios/assemble_pod_package.py \
           --staging-dir $(Build.BinariesDirectory)/pod_staging \
           --xcframework-output-dir $(Build.BinariesDirectory)/xcframework_out \
           --pod-version ${ORT_EXTENSIONS_POD_VERSION}
-      displayName: Assemble pod
+      displayName: "Assemble pod"
 
     - script: |
         ORT_EXTENSIONS_LOCAL_POD_PATH=$(Build.BinariesDirectory)/pod_staging \
           pod install
-      displayName: Install pods for OrtExtensionsUsage
+      displayName: "Install pods for OrtExtensionsUsage"
       workingDirectory: $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage
 
     - script: |
         set -e
 
-        SIMULATOR_DEVICE_INFO=$(python tools/ios/get_simulator_device_info.py)
+        SIMULATOR_DEVICE_INFO=$(python ./tools/ios/get_simulator_device_info.py)
+
         echo "Simulator device info:"
         echo "${SIMULATOR_DEVICE_INFO}"
 
         SIMULATOR_DEVICE_ID=$(jq --raw-output '.device_udid' <<< "${SIMULATOR_DEVICE_INFO}")
 
+        # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+        set +x
+        echo "##vso[task.setvariable variable=ORT_EXTENSIONS_SIMULATOR_DEVICE_ID]${SIMULATOR_DEVICE_ID}"
+      displayName: "Get simulator device info"
+
+    - script: |
+        xcrun simctl bootstatus ${ORT_EXTENSIONS_SIMULATOR_DEVICE_ID} -b
+      displayName: "Wait for simulator device to boot"
+
+    - script: |
         xcrun xcodebuild \
           -sdk iphonesimulator \
           -configuration Debug \
+          -parallel-testing-enabled NO \
           -workspace $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage/OrtExtensionsUsage.xcworkspace \
           -scheme OrtExtensionsUsage \
-          -destination "platform=iOS Simulator,id=${SIMULATOR_DEVICE_ID}" \
+          -destination "platform=iOS Simulator,id=${ORT_EXTENSIONS_SIMULATOR_DEVICE_ID}" \
           test CODE_SIGNING_ALLOWED=NO
-      displayName: Build and test OrtExtensionsUsage
+      displayName: "Build and test OrtExtensionsUsage"
diff --git a/.pipelines/ios_packaging.yml b/.pipelines/ios_packaging.yml
@@ -70,20 +70,32 @@ jobs:
   - script: |
       set -e
 
-      SIMULATOR_DEVICE_INFO=$(python tools/ios/get_simulator_device_info.py)
+      SIMULATOR_DEVICE_INFO=$(python ./tools/ios/get_simulator_device_info.py)
+
       echo "Simulator device info:"
       echo "${SIMULATOR_DEVICE_INFO}"
 
       SIMULATOR_DEVICE_ID=$(jq --raw-output '.device_udid' <<< "${SIMULATOR_DEVICE_INFO}")
 
+      # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+      set +x
+      echo "##vso[task.setvariable variable=ORT_EXTENSIONS_SIMULATOR_DEVICE_ID]${SIMULATOR_DEVICE_ID}"
+    displayName: "Get simulator device info"
+
+  - script: |
+      xcrun simctl bootstatus ${ORT_EXTENSIONS_SIMULATOR_DEVICE_ID} -b
+    displayName: "Wait for simulator device to boot"
+
+  - script: |
       xcrun xcodebuild \
         -sdk iphonesimulator \
         -configuration Debug \
+        -parallel-testing-enabled NO \
         -workspace $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage/OrtExtensionsUsage.xcworkspace \
         -scheme OrtExtensionsUsage \
-        -destination "platform=iOS Simulator,id=${SIMULATOR_DEVICE_ID}" \
+        -destination "platform=iOS Simulator,id=${ORT_EXTENSIONS_SIMULATOR_DEVICE_ID}" \
         test CODE_SIGNING_ALLOWED=NO
-    displayName: Build and test OrtExtensionsUsage
+    displayName: "Build and test OrtExtensionsUsage"
 
   - task: InstallAppleCertificate@2
     inputs: