-
-
Notifications
You must be signed in to change notification settings - Fork 315
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into feat/nesting-spans-during-lifecycle
- Loading branch information
Showing
62 changed files
with
1,372 additions
and
274 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
name: Benchmarking | ||
on: | ||
schedule: | ||
- cron: '0 0 * * *' # every night at midnight UTC | ||
push: | ||
branches: | ||
- master | ||
- release/** | ||
|
||
pull_request: | ||
paths: | ||
# test changes to Sentry SDK sources | ||
- 'Sources/**' | ||
|
||
# test changes to benchmarking implementation | ||
- 'Samples/iOS-Swift/iOS-Swift/**' | ||
- 'Samples/iOS-Swift/PerformanceBenchmarks/**' | ||
- '.github/workflows/benchmarking.yml' | ||
- '.sauce/benchmarking-config.yml' | ||
- 'fastlane/**' | ||
|
||
jobs: | ||
build-benchmark-test-target: | ||
name: Build UITests with Xcode ${{matrix.xcode}} | ||
runs-on: macos-11 | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- run: ./scripts/ci-select-xcode.sh | ||
- run: git apply ./scripts/set-device-tests-environment.patch | ||
- run: fastlane build_ios_benchmark_test | ||
env: | ||
APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }} | ||
APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }} | ||
APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }} | ||
FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }} | ||
MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }} | ||
MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }} | ||
MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }} | ||
- name: Archiving DerivedData | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
name: DerivedData-Xcode | ||
path: | | ||
**/Debug-iphoneos/iOS-Swift.app | ||
**/Debug-iphoneos/PerformanceBenchmarks-Runner.app | ||
run-ui-tests-with-sauce: | ||
name: Run benchmarks on Sauce Labs | ||
runs-on: ubuntu-latest | ||
needs: build-benchmark-test-target | ||
strategy: | ||
fail-fast: false | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- uses: actions/download-artifact@v3 | ||
with: | ||
name: DerivedData-Xcode | ||
- run: npm install -g saucectl@0.99.4 | ||
- name: Run Benchmarks in SauceLab | ||
env: | ||
SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} | ||
SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} | ||
run: saucectl run --select-suite "High-end device" --config .sauce/benchmarking-config.yml --tags benchmark --retries 5 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
apiVersion: v1alpha | ||
kind: xcuitest | ||
sauce: | ||
region: us-west-1 | ||
concurrency: 2 | ||
|
||
defaults: | ||
timeout: 30m # empirically observed; job usually takes 20-25 minutes on iPad Pro 12.9 2021 | ||
|
||
xcuitest: | ||
app: ./DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app | ||
testApp: ./DerivedData/Build/Products/Debug-iphoneos/PerformanceBenchmarks-Runner.app | ||
|
||
suites: | ||
- name: "High-end device" | ||
devices: | ||
- name: "iPad Pro 12.9 2021" | ||
platformVersion: "15.5" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
22 changes: 22 additions & 0 deletions
22
Samples/iOS-Swift/PerformanceBenchmarks/PerformanceBenchmarks-Info.plist
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
<?xml version="1.0" encoding="UTF-8"?> | ||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> | ||
<plist version="1.0"> | ||
<dict> | ||
<key>CFBundleDevelopmentRegion</key> | ||
<string>$(DEVELOPMENT_LANGUAGE)</string> | ||
<key>CFBundleExecutable</key> | ||
<string>$(EXECUTABLE_NAME)</string> | ||
<key>CFBundleIdentifier</key> | ||
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string> | ||
<key>CFBundleInfoDictionaryVersion</key> | ||
<string>6.0</string> | ||
<key>CFBundleName</key> | ||
<string>$(PRODUCT_NAME)</string> | ||
<key>CFBundlePackageType</key> | ||
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string> | ||
<key>CFBundleShortVersionString</key> | ||
<string>1.0</string> | ||
<key>CFBundleVersion</key> | ||
<string>1</string> | ||
</dict> | ||
</plist> |
17 changes: 17 additions & 0 deletions
17
Samples/iOS-Swift/PerformanceBenchmarks/SentryProcessInfo.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
#import <Foundation/Foundation.h> | ||
|
||
NS_ASSUME_NONNULL_BEGIN | ||
|
||
/** | ||
* @return YES if the process has a debugger attached. | ||
* @see https://developer.apple.com/library/archive/qa/qa1361/_index.html | ||
*/ | ||
BOOL isDebugging(void); | ||
|
||
/** | ||
* @return YES if the process is running in a simulator. | ||
* @see https://stackoverflow.com/a/45329149 | ||
*/ | ||
BOOL isSimulator(void); | ||
|
||
NS_ASSUME_NONNULL_END |
44 changes: 44 additions & 0 deletions
44
Samples/iOS-Swift/PerformanceBenchmarks/SentryProcessInfo.m
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
#import "SentryProcessInfo.h" | ||
#import <UIKit/UIKit.h> | ||
#import <sys/sysctl.h> | ||
#import <unistd.h> | ||
|
||
BOOL | ||
isDebugging() | ||
{ | ||
struct kinfo_proc info; | ||
|
||
// Initialize the flags so that, if sysctl fails for some bizarre | ||
// reason, we get a predictable result. | ||
info.kp_proc.p_flag = 0; | ||
|
||
// Initialize mib, which tells sysctl the info we want, in this case | ||
// we're looking for information about a specific process ID. | ||
int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid() }; | ||
|
||
// Call sysctl. | ||
size_t size = sizeof(info); | ||
int junk = sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0); | ||
if (junk != 0) { | ||
printf("sysctl failed while trying to get kinfo_proc\n"); | ||
return false; | ||
} | ||
|
||
// We're being debugged if the P_TRACED flag is set. | ||
return (info.kp_proc.p_flag & P_TRACED) != 0; | ||
} | ||
|
||
BOOL | ||
isSimulator() | ||
{ | ||
NSOperatingSystemVersion ios9 = { 9, 0, 0 }; | ||
NSProcessInfo *processInfo = [NSProcessInfo processInfo]; | ||
if ([processInfo isOperatingSystemAtLeastVersion:ios9]) { | ||
NSDictionary<NSString *, NSString *> *environment = [processInfo environment]; | ||
NSString *simulator = [environment objectForKey:@"SIMULATOR_DEVICE_NAME"]; | ||
return simulator != nil; | ||
} else { | ||
UIDevice *currentDevice = [UIDevice currentDevice]; | ||
return ([currentDevice.model rangeOfString:@"Simulator"].location != NSNotFound); | ||
} | ||
} |
138 changes: 138 additions & 0 deletions
138
Samples/iOS-Swift/PerformanceBenchmarks/SentrySDKPerformanceBenchmarkTests.m
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,138 @@ | ||
#import "SentryProcessInfo.h" | ||
#import <XCTest/XCTest.h> | ||
#import <objc/runtime.h> | ||
|
||
// To get around the 15 minute timeout per test case on Sauce Labs. See develop-docs/README.md. | ||
static NSUInteger SentrySDKPerformanceBenchmarkTestCases = 5; | ||
static NSUInteger SentrySDKPerformanceBenchmarkIterationsPerTestCase = 4; | ||
|
||
// All results are aggregated to analyse after completing the separate, | ||
// dynamically generated test cases | ||
static NSMutableArray *allResults; | ||
static BOOL checkedAssertions = NO; | ||
|
||
@interface SentrySDKPerformanceBenchmarkTests : XCTestCase | ||
|
||
@end | ||
|
||
@implementation SentrySDKPerformanceBenchmarkTests | ||
|
||
/** | ||
* Dynamically add a test method to an XCTestCase class. | ||
* @see https://www.gaige.net/dynamic-xctests.html | ||
*/ | ||
+ (BOOL)addInstanceMethodWithSelectorName:(NSString *)selectorName block:(void (^)(id))block | ||
{ | ||
NSParameterAssert(selectorName); | ||
NSParameterAssert(block); | ||
|
||
// See | ||
// http://stackoverflow.com/questions/6357663/casting-a-block-to-a-void-for-dynamic-class-method-resolution | ||
id impBlockForIMP = (__bridge id)(__bridge void *)(block); | ||
IMP myIMP = imp_implementationWithBlock(impBlockForIMP); | ||
SEL selector = NSSelectorFromString(selectorName); | ||
return class_addMethod(self, selector, myIMP, "v@:"); | ||
} | ||
|
||
+ (void)initialize | ||
{ | ||
allResults = [NSMutableArray array]; | ||
for (NSUInteger i = 0; i < SentrySDKPerformanceBenchmarkTestCases; i++) { | ||
[self addInstanceMethodWithSelectorName:[NSString stringWithFormat:@"testCPUBenchmark%lu", | ||
(unsigned long)i] | ||
block:^(XCTestCase *testCase) { | ||
[allResults | ||
addObjectsFromArray:[self _testCPUBenchmark]]; | ||
}]; | ||
} | ||
} | ||
|
||
- (void)tearDown | ||
{ | ||
if (allResults.count | ||
== SentrySDKPerformanceBenchmarkTestCases | ||
* SentrySDKPerformanceBenchmarkIterationsPerTestCase) { | ||
NSUInteger index = (NSUInteger)ceil(0.9 * allResults.count); | ||
NSNumber *p90 = | ||
[allResults sortedArrayUsingComparator:^NSComparisonResult(NSNumber *a, NSNumber *b) { | ||
return [a compare:b]; | ||
}][index >= allResults.count ? allResults.count - 1 : index]; | ||
XCTAssertLessThanOrEqual( | ||
p90.doubleValue, 5, @"Profiling P90 overhead should remain under 5%%."); | ||
checkedAssertions = YES; | ||
} | ||
|
||
[super tearDown]; | ||
} | ||
|
||
+ (void)tearDown | ||
{ | ||
if (!checkedAssertions) { | ||
@throw @"Did not perform assertion checks, might not have completed all benchmark trials."; | ||
} | ||
} | ||
|
||
+ (NSArray<NSNumber *> *)_testCPUBenchmark | ||
{ | ||
XCTSkipIf(isSimulator() && !isDebugging()); | ||
|
||
NSMutableArray *results = [NSMutableArray array]; | ||
for (NSUInteger j = 0; j < SentrySDKPerformanceBenchmarkIterationsPerTestCase; j++) { | ||
XCUIApplication *app = [[XCUIApplication alloc] init]; | ||
app.launchArguments = | ||
[app.launchArguments arrayByAddingObject:@"--io.sentry.test.benchmarking"]; | ||
[app launch]; | ||
[app.buttons[@"Performance scenarios"] tap]; | ||
|
||
XCUIElement *startButton = app.buttons[@"Start test"]; | ||
if (![startButton waitForExistenceWithTimeout:5.0]) { | ||
XCTFail(@"Couldn't find benchmark retrieval button."); | ||
} | ||
[startButton tap]; | ||
|
||
// after hitting the start button, the test app will do CPU intensive work until hitting the | ||
// stop button. wait 15 seconds so that work can be done while the profiler does its thing, | ||
// and the benchmarking observation in the test app will record how much CPU time is used by | ||
// everything | ||
sleep(15); | ||
|
||
XCUIElement *stopButton = app.buttons[@"Stop test"]; | ||
if (![stopButton waitForExistenceWithTimeout:5.0]) { | ||
XCTFail(@"Couldn't find benchmark retrieval button."); | ||
} | ||
[stopButton tap]; | ||
|
||
XCUIElement *textField = app.textFields[@"io.sentry.benchmark.value-marshaling-text-field"]; | ||
if (![textField waitForExistenceWithTimeout:5.0]) { | ||
XCTFail(@"Couldn't find benchmark value marshaling text field."); | ||
} | ||
|
||
NSString *benchmarkValueString = textField.value; | ||
// SentryBenchmarking.retrieveBenchmarks returns nil if there aren't at least 2 samples to | ||
// use for calculating deltas | ||
XCTAssertFalse([benchmarkValueString isEqualToString:@"nil"], | ||
@"Failure to record enough CPU samples to calculate benchmark."); | ||
if (benchmarkValueString == nil) { | ||
XCTFail(@"No benchmark value received from the app."); | ||
} | ||
|
||
NSArray *values = [benchmarkValueString componentsSeparatedByString:@","]; | ||
|
||
NSInteger profilerSystemTime = [values[0] integerValue]; | ||
NSInteger profilerUserTime = [values[1] integerValue]; | ||
NSInteger appSystemTime = [values[2] integerValue]; | ||
NSInteger appUserTime = [values[3] integerValue]; | ||
|
||
NSLog(@"[Sentry Benchmark] %ld,%ld,%ld,%ld", profilerSystemTime, profilerUserTime, | ||
appSystemTime, appUserTime); | ||
|
||
double usagePercentage | ||
= 100.0 * (profilerUserTime + profilerSystemTime) / (appUserTime + appSystemTime); | ||
|
||
[results addObject:@(usagePercentage)]; | ||
} | ||
|
||
return results; | ||
} | ||
|
||
@end |
Oops, something went wrong.