fix: gather metric samples at profile start/end #6387

Workflow file for this run

	name: Test
	on:
	push:
	branches:
	- main
	- release/**

	pull_request:
	paths:
	- "Sources/**"
	- "Tests/**"
	- "test-server/**"
	- "Samples/**"
	- ".github/workflows/test.yml"
	- "fastlane/**"
	- "scripts/tests-with-thread-sanitizer.sh"
	- "scripts/ci-select-xcode.sh"
	- "scripts/xcode-test.sh"
	- ".codecov.yml"
	- Sentry.xcodeproj

	# run the workflow any time an Xcode scheme changes for a sample app
	- "Samples/tvOS-Swift/tvOS-Swift.xcodeproj/xcshareddata/xcschemes/tvOS-Swift.xcscheme"
	- "Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS13-Swift.xcscheme"
	- "Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS-SwiftUITests.xcscheme"
	- "Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS-Swift.xcscheme"
	- "Samples/macOS-Swift/macOS-Swift.xcodeproj/xcshareddata/xcschemes/macOS-Swift.xcscheme"
	- "Samples/iOS-ObjectiveC/iOS-ObjectiveC.xcodeproj/xcshareddata/xcschemes/iOS-ObjectiveC.xcscheme"

	jobs:
	build-test-server:
	name: Build test server
	runs-on: macos-13
	steps:
	- uses: actions/checkout@v4
	- name: Cache for Test Server
	id: cache_test_server
	uses: actions/cache@v3
	with:
	path: ./test-server/.build
	key: test-server-${{ hashFiles('./test-server') }}
	restore-keys: \|
	test-server-${{ hashFiles('./test-server') }}
	test-server-

	- name: Build Test Server
	if: steps.cache_test_server.outputs.cache-hit != 'true'
	working-directory: test-server
	run: >-
	swift build -c release 2>&1 \| tee test-server-build.log

	- name: Copy exec
	working-directory: test-server
	run: cp $(swift build --show-bin-path -c release)/Run test-server-exec

	- name: Archiving DerivedData
	uses: actions/upload-artifact@v3
	with:
	name: test-server
	path: \|
	./test-server/test-server-exec

	- name: Archiving Raw Test Logs
	uses: actions/upload-artifact@v3
	if: ${{ failure() \|\| cancelled() }}
	with:
	name: test-server-build-log-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
	path: \|
	raw-test-output.log

	unit-tests:
	name: Unit ${{matrix.platform}} - Xcode ${{matrix.xcode}} - OS ${{matrix.test-destination-os}}
	runs-on: ${{matrix.runs-on}}
	timeout-minutes: 20
	needs: build-test-server

	strategy:
	fail-fast: false
	matrix:
	# Can't run tests on watchOS because XCTest is not available
	include:
	# iOS 13.7
	- runs-on: macos-11
	platform: "iOS"
	xcode: "13.2.1"
	test-destination-os: "13.7"

	# iOS 14
	- runs-on: macos-11
	platform: "iOS"
	xcode: "13.2.1"
	test-destination-os: "14.5"

	# iOS 15
	- runs-on: macos-12
	platform: "iOS"
	xcode: "13.4.1"
	test-destination-os: "latest"

	# iOS 16
	- runs-on: macos-13
	platform: "iOS"
	xcode: "14.3"
	test-destination-os: "latest"

	# macOS 11
	- runs-on: macos-11
	platform: "macOS"
	xcode: "13.2.1"
	test-destination-os: "latest"

	# macOS 12
	- runs-on: macos-12
	platform: "macOS"
	xcode: "13.4.1"
	test-destination-os: "latest"

	# macOS 13
	- runs-on: macos-13
	platform: "macOS"
	xcode: "14.3"
	test-destination-os: "latest"

	# Catalyst. We only test the latest version, as
	# the risk something breaking on Catalyst and not
	# on an older iOS or macOS version is low.
	- runs-on: macos-13
	platform: "Catalyst"
	xcode: "14.3"
	test-destination-os: "latest"

	# tvOS 15
	- runs-on: macos-12
	platform: "tvOS"
	xcode: "13.4.1"
	test-destination-os: "latest"

	# tvOS 16
	- runs-on: macos-13
	platform: "tvOS"
	xcode: "14.3"
	test-destination-os: "latest"

	steps:
	- uses: actions/checkout@v4
	- uses: actions/download-artifact@v3
	with:
	name: test-server

	- name: Print hardware info
	run: system_profiler SPHardwareDataType

	- name: Allow test-server to run
	run: chmod +x ./test-server-exec
	- run: ./test-server-exec &

	- run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}}

	# Workaround with a symlink pointed out in: https://github.com/actions/virtual-environments/issues/551#issuecomment-637344435
	- name: Prepare iOS 13.7 simulator
	if: ${{ matrix.platform == 'iOS' && matrix.test-destination-os == '13.7'}}
	run: \|
	sudo mkdir -p /Library/Developer/CoreSimulator/Profiles/Runtimes
	sudo ln -s /Applications/Xcode_11.7.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime /Library/Developer/CoreSimulator/Profiles/Runtimes/iOS\ 13.7.simruntime
	xcrun simctl create custom-test-device "iPhone 8" "com.apple.CoreSimulator.SimRuntime.iOS-13-7"

	# Workaround with a symlink pointed out in: https://github.com/actions/virtual-environments/issues/551#issuecomment-637344435
	- name: Prepare iOS 14.5 simulator
	if: ${{ matrix.platform == 'iOS' && matrix.test-destination-os == '14.5'}}
	run: \|
	sudo mkdir -p /Library/Developer/CoreSimulator/Profiles/Runtimes
	sudo ln -s /Applications/Xcode_12.5.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime /Library/Developer/CoreSimulator/Profiles/Runtimes/iOS\ 14.5.simruntime
	xcrun simctl create custom-test-device "iPhone 8" "com.apple.CoreSimulator.SimRuntime.iOS-14-5"

	- name: Install Slather
	run: gem install slather

	# We split building and running tests in two steps so we know how long running the tests takes.
	- name: Build tests
	id: build_tests
	run: ./scripts/xcode-test.sh ${{matrix.platform}} ${{matrix.test-destination-os}} $GITHUB_REF_NAME ci build-for-testing

	- name: Run tests
	# We call a script with the platform so the destination
	# passed to xcodebuild doesn't end up in the job name,
	# because GitHub Actions don't provide an easy way of
	# manipulating string in expressions.
	run: ./scripts/xcode-test.sh ${{matrix.platform}} ${{matrix.test-destination-os}} $GITHUB_REF_NAME ci test-without-building

	- name: Slowest Tests
	if: ${{ always() }}
	run: ./scripts/xcode-slowest-tests.sh

	- name: Archiving DerivedData Logs
	uses: actions/upload-artifact@v3
	if: steps.build_tests.outcome == 'failure'
	with:
	name: derived-data-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
	path: \|
	/Users/runner/Library/Developer/Xcode/DerivedData//Logs/

	- name: Archiving Raw Test Logs
	uses: actions/upload-artifact@v3
	if: ${{ failure() \|\| cancelled() }}
	with:
	name: raw-test-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
	path: \|
	raw-test-output.log

	- name: Archiving Crash Logs
	uses: actions/upload-artifact@v3
	if: ${{ failure() \|\| cancelled() }}
	with:
	name: crash-logs-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
	path: \|
	~/Library/Logs/DiagnosticReports/**

	# We can upload all coverage reports, because codecov merges them.
	# See https://docs.codecov.io/docs/merging-reports
	# Checkout .codecov.yml to see the config of Codecov
	# We don't upload codecov for release branches, as we don't want a failing coverage check to block a release.
	# We don't upload codecov for scheduled runs as CodeCov only accepts a limited amount of uploads per commit.
	- name: Push code coverage to codecov
	id: codecov_1
	uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # pin@v3.1.4
	if: ${{ contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }}
	with:
	# Although public repos should not have to specify a token there seems to be a bug with the Codecov GH action, which can
	# be solved by specifying the token, see https://github.com/codecov/codecov-action/issues/557#issuecomment-1224970469
	token: ${{ secrets.CODECOV_TOKEN }}
	fail_ci_if_error: false
	verbose: true

	# Sometimes codecov uploads etc can fail. Retry one time to rule out e.g. intermittent network failures.
	- name: Push code coverage to codecov
	id: codecov_2
	uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # pin@v3.1.4
	if: ${{ steps.codecov_1.outcome == 'failure' && contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }}
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	fail_ci_if_error: true
	verbose: true

	# We don't run all unit tests with Thread Sanitizer enabled because
	# that adds a significant overhead.
	thread-sanitizer:
	# Disabled for now, see https://github.com/getsentry/sentry-cocoa/issues/3200
	if: false
	name: Unit iOS - Thread Sanitizer
	runs-on: macos-13
	# When there are threading issues the tests sometimes keep hanging
	timeout-minutes: 20
	needs: [build-test-server]

	steps:
	- uses: actions/checkout@v4

	- uses: actions/download-artifact@v3
	with:
	name: test-server

	- name: Allow test-server to run
	run: chmod +x ./test-server-exec

	- run: ./test-server-exec &

	- run: ./scripts/ci-select-xcode.sh

	- name: Running tests with ThreadSanitizer
	run: ./scripts/tests-with-thread-sanitizer.sh

	- name: Archiving Test Logs
	uses: actions/upload-artifact@v3
	if: ${{ failure() \|\| cancelled() }}
	with:
	name: thread-sanitizer
	path: thread-sanitizer.log

	ui-tests:
	name: UI Tests for ${{matrix.target}} on Simulators
	runs-on: macos-13
	strategy:
	matrix:
	target: ["ios_swift", "ios_objc", "tvos_swift"]

	steps:
	- uses: actions/checkout@v4
	- run: ./scripts/ci-select-xcode.sh

	# GitHub Actions sometimes fail to launch the UI tests. Therefore we retry
	- name: Run Fastlane
	run: for i in {1..2}; do fastlane ui_tests_${{matrix.target}} && break ; done
	shell: sh

	- name: Archiving Raw Test Logs
	uses: actions/upload-artifact@v3
	if: ${{ failure() \|\| cancelled() }}
	with:
	name: raw-uitest-output-${{matrix.target}}
	path: \|
	~/Library/Logs/scan/*.log
	./fastlane/test_output/**

	# SwiftUI only supports iOS 14+ so we run it in a separate matrix here
	ui-tests-swift-ui:
	name: UI Tests for SwiftUI on ${{matrix.device}} Simulator
	runs-on: ${{matrix.runs-on}}
	strategy:
	fail-fast: false
	matrix:
	include:
	- runs-on: macos-12
	xcode: "13.4.1"
	device: "iPhone 8 (15.2)"

	- runs-on: macos-13
	xcode: "14.3"
	device: "iPhone 8 (16.1)"

	steps:
	- uses: actions/checkout@v4
	- run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}}

	# GitHub Actions sometimes fail to launch the UI tests. Therefore we retry
	- name: Run Fastlane
	run: for i in {1..2}; do fastlane ui_tests_ios_swiftui device:"${{matrix.device}}" && break ; done
	shell: sh

	- name: Archiving Raw Test Logs
	uses: actions/upload-artifact@v3
	if: ${{ failure() \|\| cancelled() }}
	with:
	name: raw-swiftui-test-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
	path: \|
	~/Library/Logs/scan/*.log
	./fastlane/test_output/**

	ui-tests-address-sanitizer:
	name: UI Tests with Address Sanitizer
	runs-on: macos-13

	steps:
	- uses: actions/checkout@v4
	- run: ./scripts/ci-select-xcode.sh

	# GitHub Actions sometimes fail to launch the UI tests. Therefore we retry
	- name: Run Fastlane
	run: for i in {1..2}; do fastlane ui_tests_ios_swift device:"$iPhone 8 (15.5)" address_sanitizer:true && break ; done
	shell: sh

	- name: Archiving Raw Test Logs
	uses: actions/upload-artifact@v3
	if: ${{ failure() \|\| cancelled() }}
	with:
	name: raw-uitest-output-asan
	path: \|
	~/Library/Logs/scan/*.log
	./fastlane/test_output/**

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fix: gather metric samples at profile start/end #6387

Workflow file

fix: gather metric samples at profile start/end #6387

Jobs

Run details

Workflow file for this run