test.yml

name: Test
on:
  push:
    branches:
      - main
      - release/**

  pull_request:
    paths:
      - "Sources/**"
      - "Tests/**"
      - "SentryTestUtils/**"
      - "test-server/**"
      - "Samples/**"
      - ".github/workflows/test.yml"
      - "fastlane/**"
      - "scripts/tests-with-thread-sanitizer.sh"
      - "scripts/ci-select-xcode.sh"
      - "scripts/xcode-test.sh"
      - ".codecov.yml"
      - Sentry.xcodeproj

      # run the workflow any time an Xcode scheme changes for a sample app
      - "Samples/tvOS-Swift/tvOS-Swift.xcodeproj/xcshareddata/xcschemes/tvOS-Swift.xcscheme"
      - "Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS13-Swift.xcscheme"
      - "Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS-SwiftUITests.xcscheme"
      - "Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS-Swift.xcscheme"
      - "Samples/macOS-Swift/macOS-Swift.xcodeproj/xcshareddata/xcschemes/macOS-Swift.xcscheme"
      - "Samples/iOS-ObjectiveC/iOS-ObjectiveC.xcodeproj/xcshareddata/xcschemes/iOS-ObjectiveC.xcscheme"

jobs:
  build-test-server:
    name: Build test server
    runs-on: macos-13
    steps:
      - uses: actions/checkout@v4
      - name: Cache for Test Server
        id: cache_test_server
        uses: actions/cache@v3
        with:
          path: ./test-server/.build
          key: test-server-${{ hashFiles('./test-server') }}
          restore-keys: |
            test-server-${{ hashFiles('./test-server') }}
            test-server-

      - name: Build Test Server
        if: steps.cache_test_server.outputs.cache-hit != 'true'
        working-directory: test-server
        run: >-
          swift build -c release 2>&1 | tee test-server-build.log

      - name: Copy exec
        working-directory: test-server
        run: cp $(swift build --show-bin-path -c release)/Run test-server-exec

      - name: Archiving DerivedData
        uses: actions/upload-artifact@v4
        with:
          name: test-server
          path: |
            ./test-server/test-server-exec

      - name: Archiving Raw Test Logs
        uses: actions/upload-artifact@v4
        if: ${{  failure() || cancelled() }}
        with:
          name: test-server-build-log-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
          path: |
            raw-test-output.log

  unit-tests:
    name: Unit ${{matrix.platform}} - Xcode ${{matrix.xcode}} - OS ${{matrix.test-destination-os}}
    runs-on: ${{matrix.runs-on}}
    timeout-minutes: 20
    needs: build-test-server

    strategy:
      fail-fast: false
      matrix:
        # Can't run tests on watchOS because XCTest is not available
        include:
          # iOS 13.7
          - runs-on: macos-11
            platform: "iOS"
            xcode: "13.2.1"
            test-destination-os: "13.7"
            device: "iPhone 8"

          # iOS 14
          - runs-on: macos-11
            platform: "iOS"
            xcode: "13.2.1"
            test-destination-os: "14.5"
            device: "iPhone 8"

          # iOS 15
          - runs-on: macos-12
            platform: "iOS"
            xcode: "13.4.1"
            test-destination-os: "latest"
            device: "iPhone 8"

          # iOS 16
          - runs-on: macos-13
            platform: "iOS"
            xcode: "14.3"
            test-destination-os: "16.4"
            device: "iPhone 14"

          # macOS 11
          - runs-on: macos-11
            platform: "macOS"
            xcode: "13.2.1"
            test-destination-os: "latest"

          # macOS 12
          - runs-on: macos-12
            platform: "macOS"
            xcode: "13.4.1"
            test-destination-os: "latest"

          # macOS 13
          - runs-on: macos-13
            platform: "macOS"
            xcode: "14.3"
            test-destination-os: "latest"

          # Catalyst. We only test the latest version, as
          # the risk something breaking on Catalyst and not
          # on an older iOS or macOS version is low.
          - runs-on: macos-13
            platform: "Catalyst"
            xcode: "14.3"
            test-destination-os: "latest"

          # tvOS 15
          - runs-on: macos-12
            platform: "tvOS"
            xcode: "13.4.1"
            test-destination-os: "latest"

          # tvOS 16
          - runs-on: macos-13
            platform: "tvOS"
            xcode: "14.3"
            test-destination-os: "latest"

    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: test-server

      - name: Print hardware info
        run: system_profiler SPHardwareDataType

      - name: Allow test-server to run
        run: chmod +x ./test-server-exec
      - run: ./test-server-exec &

      - name: Check test-server runs
        run: curl http://localhost:8080/echo-baggage-header

      - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}}

      # Workaround with a symlink pointed out in: https://github.com/actions/virtual-environments/issues/551#issuecomment-637344435
      - name: Prepare iOS 13.7 simulator
        if: ${{ matrix.platform == 'iOS' && matrix.test-destination-os == '13.7'}}
        run: |
          sudo mkdir -p /Library/Developer/CoreSimulator/Profiles/Runtimes
          sudo ln -s /Applications/Xcode_11.7.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime /Library/Developer/CoreSimulator/Profiles/Runtimes/iOS\ 13.7.simruntime
          xcrun simctl create custom-test-device "iPhone 8" "com.apple.CoreSimulator.SimRuntime.iOS-13-7"

      # Workaround with a symlink pointed out in: https://github.com/actions/virtual-environments/issues/551#issuecomment-637344435
      - name: Prepare iOS 14.5 simulator
        if: ${{ matrix.platform == 'iOS' && matrix.test-destination-os == '14.5'}}
        run: |
          sudo mkdir -p /Library/Developer/CoreSimulator/Profiles/Runtimes
          sudo ln -s /Applications/Xcode_12.5.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime /Library/Developer/CoreSimulator/Profiles/Runtimes/iOS\ 14.5.simruntime
          xcrun simctl create custom-test-device "iPhone 8" "com.apple.CoreSimulator.SimRuntime.iOS-14-5"

      - name: Install Slather
        run: gem install slather

      # We split building and running tests in two steps so we know how long running the tests takes.
      - name: Build tests
        id: build_tests
        run: ./scripts/xcode-test.sh ${{matrix.platform}} ${{matrix.test-destination-os}} $GITHUB_REF_NAME ci build-for-testing "${{matrix.device}}" TestCI

      - name: Run tests
        # We call a script with the platform so the destination
        # passed to xcodebuild doesn't end up in the job name,
        # because GitHub Actions don't provide an easy way of
        # manipulating string in expressions.
        run: ./scripts/xcode-test.sh ${{matrix.platform}} ${{matrix.test-destination-os}} $GITHUB_REF_NAME ci test-without-building "${{matrix.device}}" TestCI

      - name: Slowest Tests
        if: ${{ always() }}
        run: ./scripts/xcode-slowest-tests.sh

      - name: Archiving DerivedData Logs
        uses: actions/upload-artifact@v4
        if: steps.build_tests.outcome == 'failure'
        with:
          name: derived-data-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
          path: |
            /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/**

      - name: Archiving Raw Test Logs
        uses: actions/upload-artifact@v4
        if: ${{  failure() || cancelled() }}
        with:
          name: raw-test-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
          path: |
            raw-test-output.log

      - name: Archiving Crash Logs
        uses: actions/upload-artifact@v4
        if: ${{  failure() || cancelled() }}
        with:
          name: crash-logs-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
          path: |
            ~/Library/Logs/DiagnosticReports/**

      # We can upload all coverage reports, because codecov merges them.
      # See https://docs.codecov.io/docs/merging-reports
      # Checkout .codecov.yml to see the config of Codecov
      # We don't upload codecov for release branches, as we don't want a failing coverage check to block a release.
      # We don't upload codecov for scheduled runs as CodeCov only accepts a limited amount of uploads per commit.
      - name: Push code coverage to codecov
        id: codecov_1
        uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # pin@v3.1.4
        if: ${{ contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }}
        with:
          # Although public repos should not have to specify a token there seems to be a bug with the Codecov GH action, which can
          # be solved by specifying the token, see https://github.com/codecov/codecov-action/issues/557#issuecomment-1224970469
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: false
          verbose: true

      # Sometimes codecov uploads etc can fail. Retry one time to rule out e.g. intermittent network failures.
      - name: Push code coverage to codecov
        id: codecov_2
        uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # pin@v3.1.4
        if: ${{ steps.codecov_1.outcome == 'failure' && contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: true
          verbose: true

  # We don't run all unit tests with Thread Sanitizer enabled because
  # that adds a significant overhead.
  thread-sanitizer:
    name: Unit iOS - Thread Sanitizer
    runs-on: macos-13
    timeout-minutes: 20
    needs: [build-test-server]
    
    # There are several ways this test can flake. Sometimes threaded tests will just hang and the job will time out, other times waiting on expectations will take much longer than in a non-TSAN run and the test case will fail. We're making this nonfailable and will grep the logs to extract any actual thread sanitizer warnings to push to the PR, and ignore everything else.
    continue-on-error: true

    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          name: test-server

      - name: Allow test-server to run
        run: chmod +x ./test-server-exec

      - run: ./test-server-exec &

      - run: ./scripts/ci-select-xcode.sh

      - name: Running tests with ThreadSanitizer
        run: ./scripts/tests-with-thread-sanitizer.sh

      - name: Archiving Test Logs
        uses: actions/upload-artifact@v4
        if: ${{  failure() || cancelled() }}
        with:
          name: thread-sanitizer
          path: thread-sanitizer.log

  ui-tests:
    name: UI Tests for ${{matrix.target}} on Simulators
    runs-on: macos-13
    strategy:
      matrix:
        target: ["ios_objc", "tvos_swift"]

    steps:
      - uses: actions/checkout@v4
      - run: ./scripts/ci-select-xcode.sh

      # GitHub Actions sometimes fail to launch the UI tests. Therefore we retry
      - name: Run Fastlane
        run: for i in {1..2}; do fastlane ui_tests_${{matrix.target}} && break ; done
        shell: sh

      - name: Archiving Raw Test Logs
        uses: actions/upload-artifact@v4
        if: ${{  failure() || cancelled() }}
        with:
          name: raw-uitest-output-${{matrix.target}}
          path: |
            ~/Library/Logs/scan/*.log
            ./fastlane/test_output/**

  # SwiftUI only supports iOS 14+ so we run it in a separate matrix here
  ui-tests-swift-ui:
    name: UI Tests for SwiftUI on ${{matrix.device}} Simulator
    runs-on: ${{matrix.runs-on}}
    strategy:
      fail-fast: false
      matrix:
        include:
          - runs-on: macos-12
            xcode: "13.4.1"
            device: "iPhone 8 (15.2)"

          - runs-on: macos-13
            xcode: "14.3"
            device: "iPhone 8 (16.1)"

    steps:
      - uses: actions/checkout@v4
      - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}}

      # GitHub Actions sometimes fail to launch the UI tests. Therefore we retry
      - name: Run Fastlane
        run: for i in {1..2}; do fastlane ui_tests_ios_swiftui device:"${{matrix.device}}" && break ; done
        shell: sh

      - name: Archiving Raw Test Logs
        uses: actions/upload-artifact@v4
        if: ${{  failure() || cancelled() }}
        with:
          name: raw-swiftui-test-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
          path: |
            ~/Library/Logs/scan/*.log
            ./fastlane/test_output/**

  ui-tests-address-sanitizer:
    name: UI Tests with Address Sanitizer
    runs-on: macos-13

    steps:
      - uses: actions/checkout@v4
      - run: ./scripts/ci-select-xcode.sh

      # GitHub Actions sometimes fail to launch the UI tests. Therefore we retry
      - name: Run Fastlane
        run: for i in {1..2}; do fastlane ui_tests_ios_swift device:"$iPhone 8 (15.5)" address_sanitizer:true && break ; done
        shell: sh

      - name: Archiving Raw Test Logs
        uses: actions/upload-artifact@v4
        if: ${{  failure() || cancelled() }}
        with:
          name: raw-uitest-output-asan
          path: |
            ~/Library/Logs/scan/*.log
            ./fastlane/test_output/**