Extract integration tests to the separate CircleCI job (#1870)

* Rename job to build host tests * Rename job to build tests * Rename job to build firebase tests * Rename Build tests job * Run integration tests * Code coverage command * Enable codecoverage by default * Run integration tests on nightly bases * Utilize default values for run-integration-tests job * Remove unused arg
mapbox · Jan 24, 2023 · 4b54016 · 4b54016
1 parent 72c364e
commit 4b54016
Showing 1 changed file with 138 additions and 79 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -61,13 +61,14 @@ workflows:
           matrix:
             parameters:
               xcode: ["13.1.0", "13.2.1"]
-      - build-unit-tests:
+      - build-tests:
           context:
             - Slack Orb
             - SDK Registry Token
             - Maps SDK/Public Mapbox Token
           configuration: "Release"
           report_failure: true
+          name: Build tests (nightly)
       - run-unit-tests:
           context:
             - Slack Orb
@@ -76,7 +77,7 @@ workflows:
             parameters:
               destination: ["OS=15.5,name=iPhone 13", "OS=16.1,name=iPhone 14"]
           requires:
-            - build-unit-tests
+            - Build tests (nightly)
       - run-unit-tests:
           context:
             - Slack Orb
@@ -85,16 +86,22 @@ workflows:
             parameters:
               destination: ["OS=13.7,name=iPhone 11", "OS=14.5,name=iPhone 11"]
           requires:
-            - build-unit-tests
-      - build-tests:
+            - Build tests (nightly)
+      - run-integration-tests:
+          context:
+            - Slack Orb
+          requires:
+              - Build tests (nightly)
+          name: Run integration tests (nightly)
+      - build-host-tests:
           context:
             - Slack Orb
             - Fastlane Match
             - SDK Registry Token
             - Apple Machine account
             - Maps SDK/Public Mapbox Token
           name: Build MapboxTestHost tests
-      - run-tests:
+      - run-firebase-tests:
           context:
             - Slack Orb
             - Maps SDK/Google Cloud credentials
@@ -110,7 +117,7 @@ workflows:
             - Apple Machine account
             - Maps SDK/Public Mapbox Token
           name: Build Examples tests
-      - run-tests:
+      - run-firebase-tests:
           context:
             - Slack Orb
             - Maps SDK/Google Cloud credentials
@@ -134,7 +141,7 @@ workflows:
           filters:
             branches:
               ignore: main
-      - build-tests:
+      - build-host-tests:
           context:
             - Slack Orb
             - Fastlane Match
@@ -144,7 +151,7 @@ workflows:
           name: Build MapboxTestHost tests [non-main]
           requires:
             - run-tests?
-      - run-tests:
+      - run-firebase-tests:
           context:
             - Slack Orb
             - Maps SDK/Google Cloud credentials
@@ -169,7 +176,7 @@ workflows:
           name: Build Examples tests [non-main]
           requires:
             - run-examples-tests?
-      - run-tests:
+      - run-firebase-tests:
           context:
             - Slack Orb
             - Maps SDK/Google Cloud credentials
@@ -251,38 +258,44 @@ workflows:
               only: main
 
       # Run unit tests on CI simulator
-      - build-unit-tests:
+      - build-tests:
           context:
             - Slack Orb
             - SDK Registry Token
             - Maps SDK/Public Mapbox Token
-          name: Build unit tests
+          name: Build tests
       - run-unit-tests:
           context:
             - Slack Orb
           xcode: 14.1.0
-          codecoverage: true
           matrix:
             parameters:
               destination: ["OS=15.5,name=iPhone 13", "OS=16.1,name=iPhone 14"]
           requires:
-            - Build unit tests
+            - Build tests
       - run-unit-tests:
           context:
             - Slack Orb
           xcode: 12.5.1
+          disable_codecoverage: true
           matrix:
             parameters:
               destination: ["OS=13.7,name=iPhone 11", "OS=14.5,name=iPhone 11"]
           requires:
-            - Build unit tests
+            - Build tests
+      - run-integration-tests:
+          context:
+            - Slack Orb
+          requires:
+              - Build tests
+          name: Run integration tests
 
-      - build-unit-tests:
+      - build-tests:
           context:
             - Slack Orb
             - SDK Registry Token
             - Maps SDK/Public Mapbox Token
-          name: build-unit-tests-min-turf
+          name: build-tests-min-turf
           turf-revision: v2.0.0
 
       - run-unit-tests:
@@ -292,20 +305,20 @@ workflows:
           xcode: 13.1.0
           destination: OS=15.0,name=iPhone 12
           requires:
-            - build-unit-tests-min-turf
+            - build-tests-min-turf
 
       # Build all tests with host application
       # Run on main by default
       # Can be approved to run on other branches
-      - build-tests:
+      - build-host-tests:
           context:
             - Slack Orb
             - Fastlane Match
             - SDK Registry Token
             - Apple Machine account
             - Maps SDK/Public Mapbox Token
           name: Build MapboxTestHost tests
-      - run-tests:
+      - run-firebase-tests:
           context:
             - Slack Orb
             - Maps SDK/Google Cloud credentials
@@ -328,7 +341,7 @@ workflows:
             - Apple Machine account
             - Maps SDK/Public Mapbox Token
           name: Build Examples tests
-      - run-tests:
+      - run-firebase-tests:
           context:
             - Slack Orb
             - Maps SDK/Google Cloud credentials
@@ -739,7 +752,7 @@ jobs:
                           scripts/ci-trigger/ci-e2e-compatibility-start-pipeline.py --config mapbox-maps-ios=${CIRCLE_SHA1} --platform all --versions ${E2E_VERSION_CONFIG}
                       name: Trigger E2E SDK test validation pipeline
 
-  build-unit-tests:
+  build-tests:
     parameters:
       configuration:
         type: string
@@ -792,29 +805,72 @@ jobs:
         type: string
       xcode:
         type: string
-      codecoverage:
+      disable_codecoverage:
         type: boolean
         default: false
+      resource_class:
+        type: string
+        default: macos.x86.medium.gen2
+      xctestrun_prefix:
+        type: string
+        default: MapboxMaps_UnitTests
     macos:
       xcode: << parameters.xcode >>
-    resource_class: macos.x86.medium.gen2
+    resource_class: << parameters.resource_class >>
     steps:
       - checkout
       - attach_workspace:
           at: build
       - run:
           name: Run MapboxMaps unit tests on << parameters.destination >>  simulator
           command: |
-            XCTESTRUN_PATH=$(find build -type f -name 'MapboxMaps_UnitTests*.xctestrun' | head -1)
+            XCTESTRUN_PATH=$(find build -type f -name '<< parameters.xctestrun_prefix >>*.xctestrun' | head -1)
             xcodebuild \
               test-without-building \
               -xctestrun "$XCTESTRUN_PATH" \
               -parallel-testing-enabled NO \
               -destination 'platform=iOS Simulator,<< parameters.destination >>' \
-              -resultBundlePath MapboxMapsTests.xcresult | tee xcodebuild.log | xcpretty --report junit --output junit/unitTests/junit.xml
+              -resultBundlePath MapboxMapsTests.xcresult | tee xcodebuild.log | xcpretty --report junit --output junit.xml
 
-            find "${HOME}/Library/Developer/Xcode/DerivedData" -name '*.profdata' -exec mv {} Coverage-unittests.profdata \;
-            find "${HOME}/Library/Developer/Xcode/DerivedData" -name '*.profraw' -exec mv {} . \;
+            find "${HOME}/Library/Developer/Xcode/DerivedData" -name '*.profdata' -exec cp {} . \;
+            find "${HOME}/Library/Developer/Xcode/DerivedData" -name '*.profraw' -exec cp {} . \;
+      - unless:
+          condition: << parameters.disable_codecoverage >>
+          steps:
+            - collect-code-coverage
+      - run:
+          name: Compress XCResult
+          command: zip -r MapboxMapsTests.xcresult.zip MapboxMapsTests.xcresult
+          when: always
+      - store_test_results:
+          path: junit.xml
+      - store_artifacts:
+          path: MapboxMapsTests.xcresult.zip
+      - store_artifacts:
+          path: xcodebuild.log
+      - store_artifacts:
+          path: ~/Library/Logs/DiagnosticReports
+      - report-failure:
+                message: "Unit test failed: << parameters.destination >>, Xcode: << parameters.xcode >>"
+
+  run-integration-tests:
+    parameters:
+      destination:
+        type: string
+        default: "name=iPhone 14"
+      xcode:
+        type: string
+        default: "14.0.1"
+      resource_class:
+        type: string
+        default: macos.m1.large.gen1
+    macos:
+      xcode: << parameters.xcode >>
+    resource_class: << parameters.resource_class >>
+    steps:
+      - checkout
+      - attach_workspace:
+          at: build
       - run:
           name: Run MapboxMaps integration tests on << parameters.destination >>  simulator
           command: |
@@ -824,53 +880,14 @@ jobs:
               -xctestrun "$XCTESTRUN_PATH" \
               -parallel-testing-enabled NO \
               -destination 'platform=iOS Simulator,<< parameters.destination >>' \
-              -resultBundlePath MapboxMapsTests-integration.xcresult | tee -a xcodebuild.log | xcpretty --report junit --output junit/integration/junit.xml
+              -resultBundlePath MapboxMapsTests.xcresult | tee -a xcodebuild.log | xcpretty --report junit --output junit.xml
 
-            find "${HOME}/Library/Developer/Xcode/DerivedData" -name '*.profdata' -exec mv {} Coverage-integrationtests.profdata \;
-          when: always
-      - when:
-          condition: << parameters.codecoverage >>
-          steps:
-            - run:
-                name: Generate code coverage report
-                command: |
-                  BINARY_PATH="$(find build -name "MapboxMaps.o" | head -1)"
-
-                  xcrun llvm-cov export -arch $(uname -m) -format="lcov" \
-                    -instr-profile="Coverage-unittests.profdata" \
-                    "$BINARY_PATH" > coverage.lcov
-                  xcrun llvm-cov export -arch $(uname -m) -format="lcov" \
-                    -instr-profile="Coverage-integrationtests.profdata" \
-                    "$BINARY_PATH" > coverage-integrationtests.lcov
-                  xcrun llvm-cov export -arch $(uname -m) -format=text \
-                    -instr-profile="Coverage-unittests.profdata" \
-                    "$BINARY_PATH" > coverage.txt
-                  xcrun llvm-cov export -arch $(uname -m) -format=text \
-                    -instr-profile="Coverage-integrationtests.profdata" \
-                    "$BINARY_PATH" > coverage-integrationtests.txt
-
-                    zip -r coverage.zip coverage.lcov coverage-integrationtests.lcov coverage.txt coverage-integrationtests.txt "$BINARY_PATH" "$COVERAGE_PATH" junit
-            - install-python-dependencies
-            - install-mbx-ci
-            - run:
-                name: Send code coverage
-                command: |
-                  curl -Os https://uploader.codecov.io/latest/macos/codecov
-                  chmod +x codecov
-                  ./codecov -f coverage.lcov -F unit
-                  ./codecov -f coverage-integrationtests.lcov -F integration
-
-                  echo "Uploading to S3" ; \
-                  python3 ./scripts/code-coverage/parse-code-coverage.py \
-                    -g . \
-                    -c MapboxMaps \
-                    --scheme MapboxMaps \
-                    --report coverage.txt
-            - store_artifacts:
-                path: coverage.zip
+            find "${HOME}/Library/Developer/Xcode/DerivedData" -name '*.profdata' -exec cp {} . \;
+      - collect-code-coverage:
+          codecov_flag_name: integration
       - run:
           name: Compress XCResult
-          command: zip -r MapboxMapsTests.xcresult.zip MapboxMapsTests.xcresult MapboxMapsTests-integration.xcresult
+          command: zip -r MapboxMapsTests.xcresult.zip MapboxMapsTests.xcresult
           when: always
       - store_test_results:
           path: junit
@@ -880,14 +897,10 @@ jobs:
           path: xcodebuild.log
       - store_artifacts:
           path: ~/Library/Logs/DiagnosticReports
-      - when:
-          condition:
-            equal: [ "main", << pipeline.git.branch >>]
-          steps:
-            - report-failure:
-                message: "Unit test failed: << parameters.destination >>, Xcode: << parameters.xcode >>"
+      - report-failure:
+          message: "Integration test failed: << parameters.destination >>, Xcode: << parameters.xcode >>"
 
-  build-tests:
+  build-host-tests:
     executor: xcode-latest
     resource_class: macos.x86.medium.gen2
     steps:
@@ -907,7 +920,7 @@ jobs:
             - build_products.zip
       - slack-notify-failure-if-any
 
-  run-tests:
+  run-firebase-tests:
     parameters:
       devices:
         default: |
@@ -1868,3 +1881,49 @@ commands:
             - << parameters.output >>
       - store_artifacts:
           path: << parameters.output >>
+
+  collect-code-coverage:
+    parameters:
+      profdata_path:
+        type: string
+        default: Coverage.profdata
+      binary_name:
+        type: string
+        default: MapboxMaps.o
+      codecov_flag_name:
+        type: string
+        default: unit
+    steps:
+      - run:
+          name: Generate code coverage report
+          command: |
+            BINARY_PATH="$(find build -name "<< parameters.binary_name >>" | head -1)"
+
+            echo "Generating LCOV code coverage report for $BINARY_PATH"
+            xcrun llvm-cov export -arch $(uname -m) -format="lcov" \
+              -instr-profile="<< parameters.profdata_path >>" \
+              "$BINARY_PATH" > coverage.lcov
+
+            echo "Generating TXT code coverage report for $BINARY_PATH"
+            xcrun llvm-cov export -arch $(uname -m) -format=text \
+              -instr-profile="<< parameters.profdata_path >>" \
+              "$BINARY_PATH" > coverage.txt
+
+              zip -r coverage.zip coverage.lcov coverage.txt "<< parameters.profdata_path >>" "$BINARY_PATH" junit*
+      - install-python-dependencies
+      - install-mbx-ci
+      - run:
+          name: Send code coverage
+          command: |
+            curl -Os https://uploader.codecov.io/latest/macos/codecov
+            chmod +x codecov
+            ./codecov -f coverage.lcov -F << parameters.codecov_flag_name >>
+
+            echo "Uploading to S3" ; \
+            python3 ./scripts/code-coverage/parse-code-coverage.py \
+              -g . \
+              -c MapboxMaps \
+              --scheme MapboxMaps \
+              --report coverage.txt
+      - store_artifacts:
+          path: coverage.zip