From e6d200a2f967cd27852ea873e879d0de91a965d1 Mon Sep 17 00:00:00 2001 From: Ravi Kumar Mandala Date: Thu, 16 Apr 2020 13:17:01 -0700 Subject: [PATCH] Merge exit code fixes from PR#432 and PR#430 to Xcode 11.2 branch (#435) * make retry checks more resilient to prevent infinite retries (#432) Also, log when retry count exceeded max retries. Co-authored-by: Mansfield Mark * Added a few more tests to mock test failure scenarios and fixed a few final Exit Status issues (#430) * Adding the ability to mock different test executions in Bluepill * Add a few more tests and fixed the exit status issues Changing the Exit Status to powers of 2 to make them mergeable. Easy consolidation/aggregation of exit code. In case of failure reporting, report all exit status that happened over multiple attempts, if any. Prevent extra attempt, which does nothing, by eliminating the use of `hasRemainingTestsInContext` which is not accurate. * Re-enabling some disabled/skipped BP tests and fixed a fragile packer test A fragile packer test broke when the total number of tests changed. So, fixed the testing method to fix the test. Also, re-enabled a few tests in BluepillTests that were being skipped/disabled for some reason. * Renaming BPExitStatusTestsAllPassed to BPExitStatusAllTestsPassed Renaming the all-good exit status from BPExitStatusTestsAllPassed to BPExitStatusAllTestsPassed and incorporated comments. Tweaked the tests a bit to reduce the regression testing duration. Co-authored-by: Mansfield Mark Co-authored-by: Mansfield Mark --- .../BPSampleAppHangingTests.m | 87 +++++++- README.md | 89 +++++--- bluepill/tests/BPPackerTests.m | 18 +- .../xcshareddata/xcschemes/bp-tests.xcscheme | 12 -- bp/src/BPConfiguration.h | 1 + bp/src/BPConfiguration.m | 13 +- bp/src/BPExitStatus.h | 24 +-- bp/src/BPExitStatus.m | 34 ++- bp/src/BPSimulator.m | 5 +- bp/src/BPStats.m | 2 +- bp/src/BPUtils.m | 2 - bp/src/Bluepill.m | 139 ++++++------ bp/src/SimulatorMonitor.m | 2 +- bp/tests/BPUtilsTests.m | 38 ++++ bp/tests/BluepillTests.m | 204 ++++++++++++++++-- bp/tests/Resource Files/hanging_tests.xml | 9 +- 16 files changed, 502 insertions(+), 177 deletions(-) diff --git a/BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m b/BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m index 844146e0..0c990a08 100644 --- a/BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m +++ b/BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m @@ -15,10 +15,75 @@ @interface BPSampleAppHangingTests : XCTestCase @implementation BPSampleAppHangingTests +-(long)attemptFromSimulatorVersionInfo:(NSString *)simulatorVersionInfo { + // simulatorVersionInfo is something like + // CoreSimulator 587.35 - Device: BP93497-2-2 (7AB3D528-5473-401A-B23E-2E2E86C73861) - Runtime: iOS 12.2 (16E226) - DeviceType: iPhone 7 + NSLog(@"Dissecting version info %@ to extra attempt number.", simulatorVersionInfo); + NSArray *parts = [simulatorVersionInfo componentsSeparatedByString:@" - "]; + NSString *deviceString = parts[1]; + // Device: BP93497-2-2 (7AB3D528-5473-401A-B23E-2E2E86C73861) + parts = [deviceString componentsSeparatedByString:@" "]; + NSString *device = parts[1]; + // BP93497-2-2 + parts = [device componentsSeparatedByString:@"-"]; + NSString *attempt = parts[1]; + return [attempt longLongValue]; +} + +-(void)extractPlanAndExecuteActions:(int)index { + NSDictionary *env = [[NSProcessInfo processInfo] environment]; + NSString *simulatorVersionInfo = [env objectForKey:@"SIMULATOR_VERSION_INFO"]; + long attempt = [self attemptFromSimulatorVersionInfo:simulatorVersionInfo]; + NSString *executionPlan = [env objectForKey:@"_BP_TEST_EXECUTION_PLAN"]; + if (!executionPlan) { + NSLog(@"No execution plan found in attempt#%ld. Failing the test.", attempt); + XCTAssert(NO); + return; + } + NSLog(@"Received execution plan %@ on attempt#%ld for this test.", executionPlan, attempt); + NSArray *setsOfPlans = [executionPlan componentsSeparatedByString:@";"]; + if (index >= [setsOfPlans count]) { + NSLog(@"Not enough plans for test#%d in execution plan: '%@'.", index, executionPlan); + XCTAssert(YES); + return; + } + NSString *currentPlan = setsOfPlans[index]; + currentPlan = [currentPlan stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]]; + NSArray *array = [currentPlan componentsSeparatedByCharactersInSet:[NSCharacterSet whitespaceCharacterSet]]; + if (attempt > [array count]) { + NSLog(@"Passing on attempt#%ld, by default, as there is no action defined in the execution plan", (long)attempt); + XCTAssert(YES); + return; + } + NSString *action = array[attempt - 1]; + if ([action isEqualToString:@"TIMEOUT"]) { + NSLog(@"Entering into an infinite loop on attempt#%ld to timeout as per the execution plan", (long)attempt); + while(1) { + } + return; + } else if ([action isEqualToString:@"PASS"]) { + NSLog(@"Passing on attempt#%ld based on execution plan", (long)attempt); + XCTAssert(YES); + return; + } else if ([action isEqualToString:@"FAIL"]) { + NSLog(@"Failing on attempt#%ld based on execution plan", (long)attempt); + XCTAssert(NO); + return; + } else if ([action isEqualToString:@"CRASH"]) { + NSLog(@"Crashing on attempt#%ld based on execution plan", (long)attempt); + // ok, let's crash and burn + int *pointer = nil; + *pointer = 1; + return; + } + NSLog(@"Failing on attempt#%ld as an unidentified action is encountered in the execution plan", (long)attempt); + XCTAssert(NO); + return; +} + - (void)setUp { [super setUp]; // Put setup code here. This method is called before the invocation of each test method in the class. - } - (void)tearDown { @@ -26,8 +91,24 @@ - (void)tearDown { [super tearDown]; } -- (void)testAppHanging { - while(TRUE){}; +- (void)testASimpleTest { + XCTAssert(YES); +} + +- (void)testBasedOnExecutionPlan { + [self extractPlanAndExecuteActions:0]; +} + +- (void)testCaseFinal { + XCTAssert(YES); +} + +- (void)testDoubleBasedOnExecutionPlan { + [self extractPlanAndExecuteActions:1]; +} + +- (void)testEndFinal { + XCTAssert(YES); } @end diff --git a/README.md b/README.md index 86232ee2..832b4f13 100644 --- a/README.md +++ b/README.md @@ -54,39 +54,60 @@ $ bluepill -c config.json A full list supported options are listed here. -| Config Arguments | Command Line Arguments | Explanation | Required | Default value | -|:----------------------:|:----------------------:|------------------------------------------------------------------------------------|:--------:|:----------------:| -| `app` | -a | The path to the host application to execute (your .app) | N | n/a | -| `xctestrun-path` | | The path to the `.xctestrun` file that xcode leaves when you `build-for-testing`. | Y | n/a | -| `output-dir` | -o | Directory where to put output log files (bluepill only) | Y | n/a | -| config | -c | Read options from the specified configuration file instead of the command line | N | n/a | -| device | -d | On which device to run the app. | N | iPhone 6 | -| exclude | -x | Exclude a testcase in the set of tests to run (takes priority over `include`). | N | empty | -| headless | -H | Run in headless mode (no GUI). | N | off | -| clone-simulator | -L | Spawn simulator by clone from simulator template. | N | off | -| xcode-path | -X | Path to xcode. | N | xcode-select -p | -| include | -i | Include a testcase in the set of tests to run (unless specified in `exclude`). | N | all tests | -| list-tests | -l | Only list tests in bundle | N | false | -| num-sims | -n | Number of simulators to run in parallel. (bluepill only) | N | 4 | -| printf-config | -P | Print a configuration file suitable for passing back using the `-c` option. | N | n/a | -| error-retries | -R | Number of times to recover from simulator/app crashing/hanging and continue running| N | 5 | -| failure-tolerance | -f | Number of times to retry on test failures | N | 0 | -| only-retry-failed | -F | When `failure-tolerance` > 0, only retry tests that failed | N | false | -| runtime | -r | What runtime to use. | N | iOS 11.1 | -| stuck-timeout | -S | Timeout in seconds for a test that seems stuck (no output). | N | 300s | -| test-timeout | -T | Timeout in seconds for a test that is producing output. | N | 300s | -| test-bundle-path | -t | The path to the test bundle to execute (single .xctest). | N | n/a | -| additional-unit-xctests| n/a | Additional XCTest bundles that is not Plugin folder | N | n/a | -| additional-ui-xctests | n/a | Additional XCTUITest bundles that is not Plugin folder | N | n/a | -| repeat-count | -C | Number of times we'll run the entire test suite (used for load testing). | N | 1 | -| no-split | -N | Test bundles you don't want to be packed into different groups to run in parallel. | N | n/a | -| quiet | -q | Turn off all output except fatal errors. | N | YES | -| diagnostics | n/a | Enable collection of diagnostics in outputDir in case of test failures | N | NO | -| help | -h | Help. | N | n/a | -| runner-app-path | -u | The test runner for UI tests. | N | n/a | -| screenshots-directory | n/a | Directory where simulator screenshots for failed ui tests will be stored | N | n/a | -| video-paths | -V | A list of videos that will be saved in the simulators | N | n/a | -| image-paths | -I | A list of images that will be saved in the simulators | N | n/a | +| Config Arguments | Command Line Arguments | Explanation | Required | Default value | +|:----------------------:|:----------------------:|-------------------------------------------------------------------------------------|:--------:|:----------------:| +| app | -a | The path to the host application to execute (your `.app`) | N | n/a | +| xctestrun-path | | The path to the `.xctestrun` file that xcode leaves when you `build-for-testing`. | Y | n/a | +| output-dir | -o | Directory where to put output log files. **(bluepill only)** | Y | n/a | +| config | -c | Read options from the specified configuration file instead of the command line. | N | n/a | +| device | -d | On which device to run the app. | N | iPhone 8 | +| exclude | -x | Exclude a testcase in the set of tests to run (takes priority over `include`). | N | empty | +| headless | -H | Run in headless mode (no GUI). | N | off | +| clone-simulator | -L | Spawn simulator by clone from simulator template. | N | off | +| xcode-path | -X | Path to xcode. | N | xcode-select -p | +| include | -i | Include a testcase in the set of tests to run (unless specified in `exclude`). | N | all tests | +| list-tests | -l | Only list tests and exit without executing tests. | N | false | +| num-sims | -n | Number of simulators to run in parallel. **(bluepill only)** | N | 4 | +| printf-config | -P | Print a configuration file suitable for passing back using the `-c` option. | N | n/a | +| error-retries | -R | Number of times to recover from simulator/app crashing/hanging and continue running.| N | 4 | +| failure-tolerance | -f | Number of times to retry on test failures | N | 0 | +| only-retry-failed | -F | Only retry failed tests instead of all. Also retry test that timed-out/crashed. | N | false | +| runtime | -r | What runtime to use. | N | iOS 13.2 | +| stuck-timeout | -S | Timeout in seconds for a test that seems stuck (no output). | N | 300s | +| test-timeout | -T | Timeout in seconds for a test that is producing output. | N | 300s | +| test-bundle-path | -t | The path to the test bundle to execute (single `.xctest`). | N | n/a | +| additional-unit-xctests| n/a | Additional XCTest bundles that is not Plugin folder | N | n/a | +| additional-ui-xctests | n/a | Additional XCTUITest bundles that is not Plugin folder | N | n/a | +| repeat-count | -C | Number of times we'll run the entire test suite (used for load testing). | N | 1 | +| no-split | -N | Test bundles you don't want to be packed into different groups to run in parallel. | N | n/a | +| quiet | -q | Turn off all output except fatal errors. | N | YES | +| diagnostics | n/a | Enable collection of diagnostics in output directory in case of test failures. | N | NO | +| help | -h | Help. | N | n/a | +| runner-app-path | -u | The test runner for UI tests. | N | n/a | +| screenshots-directory | n/a | Directory where simulator screenshots for failed ui tests will be stored. | N | n/a | +| video-paths | -V | A list of videos that will be saved in the simulators. | N | n/a | +| image-paths | -I | A list of images that will be saved in the simulators. | N | n/a | + +## Exit Status + +Here is a list of Bluepill exit codes. If a Bluepill execution has multiple exit codes from same or different test bundles, the final exit code is a combination of all exit codes. Note that app crashes are fatal even if the test passes on retry. + +```shell + BPExitStatusAllTestsPassed = 0, + BPExitStatusTestsFailed = 1 << 0, + BPExitStatusSimulatorCreationFailed = 1 << 1, + BPExitStatusInstallAppFailed = 1 << 2, + BPExitStatusInterrupted = 1 << 3, + BPExitStatusSimulatorCrashed = 1 << 4, + BPExitStatusLaunchAppFailed = 1 << 5, + BPExitStatusTestTimeout = 1 << 6, + BPExitStatusAppCrashed = 1 << 7, + BPExitStatusSimulatorDeleted = 1 << 8, + BPExitStatusUninstallAppFailed = 1 << 9, + BPExitStatusSimulatorReuseFailed = 1 << 10 +``` +**Note:** Please refer to `bp/src/BPExitStatus.h` for the latest/exact exit codes. + ## Demo @@ -138,7 +159,7 @@ If you're using [Bitrise.io](https://bitrise.io) as your CI/CD, you can start us Latest [release](https://github.com/linkedin/bluepill/releases/). -- How to test Bluepill in Xcode +- How to test Bluepill in Xcode? Select BPSampleApp scheme and build it first. Then you can switch back to `bluepill` or `bluepill-cli` scheme to run their tests. diff --git a/bluepill/tests/BPPackerTests.m b/bluepill/tests/BPPackerTests.m index 0f5c3326..44149cd2 100644 --- a/bluepill/tests/BPPackerTests.m +++ b/bluepill/tests/BPPackerTests.m @@ -187,7 +187,7 @@ - (void)testPacking { // Make sure we don't split when we don't want to self.config.numSims = @4; self.config.noSplit = @[@"BPSampleAppTests"]; - bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];// withNoSplitList:@[@"BPSampleAppTests"] intoBundles:4 andError:nil]; + bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil]; // withNoSplitList:@[@"BPSampleAppTests"] intoBundles:4 andError:nil]; // When we prevent BPSampleTests from splitting, BPSampleAppFatalErrorTests and BPAppNegativeTests gets split in two want = [[want arrayByAddingObject:@"BPSampleAppFatalErrorTests"] sortedArrayUsingSelector:@selector(compare:)]; XCTAssertEqual(bundles.count, app.testBundles.count + 2); @@ -195,9 +195,10 @@ - (void)testPacking { XCTAssertEqual([bundles[0].skipTestIdentifiers count], 0); XCTAssertEqual([bundles[1].skipTestIdentifiers count], 0); XCTAssertEqual([bundles[2].skipTestIdentifiers count], 0); - XCTAssertEqual([bundles[3].skipTestIdentifiers count], 2); - XCTAssertEqual([bundles[4].skipTestIdentifiers count], 3); + XCTAssertEqual([bundles[3].skipTestIdentifiers count], 1); + XCTAssertEqual([bundles[4].skipTestIdentifiers count], 4); XCTAssertEqual([bundles[5].skipTestIdentifiers count], 1); + XCTAssertEqual([bundles[6].skipTestIdentifiers count], 4); self.config.numSims = @4; self.config.noSplit = nil; @@ -209,14 +210,17 @@ - (void)testPacking { long testsPerBundle = [allTests count] / numSims; long skipTestsPerBundle = 0; long skipTestsInFinalBundle = 0; + long testCount = 0; for (int i = 0; i < bundles.count; ++i) { skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle); - skipTestsInFinalBundle = testsPerBundle * (numSims - 1); if (i < 4) { XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0); + testCount += [[bundles[i] allTestCases] count]; } else if (i < bundles.count-1) { XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle); + testCount += testsPerBundle; } else { /* last bundle */ + skipTestsInFinalBundle = [[bundles[i] allTestCases] count] - ([allTests count] - testCount); XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle); } } @@ -231,15 +235,19 @@ - (void)testPacking { numSims = [self.config.numSims integerValue]; testsPerBundle = [allTests count] / numSims; + testCount = 0; for (int i = 0; i < bundles.count; ++i) { skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle); - skipTestsInFinalBundle = testsPerBundle * (numSims - 1); if (i < 4) { XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0); + testCount += [[bundles[i] allTestCases] count]; } else if (i < bundles.count-1) { XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle); + testCount += testsPerBundle; } else { /* last bundle */ + skipTestsInFinalBundle = [[bundles[i] allTestCases] count] - ([allTests count] - testCount); XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle); + } } diff --git a/bp/bp.xcodeproj/xcshareddata/xcschemes/bp-tests.xcscheme b/bp/bp.xcodeproj/xcshareddata/xcschemes/bp-tests.xcscheme index ca5a1f5d..56188853 100644 --- a/bp/bp.xcodeproj/xcshareddata/xcschemes/bp-tests.xcscheme +++ b/bp/bp.xcodeproj/xcshareddata/xcschemes/bp-tests.xcscheme @@ -61,12 +61,6 @@ ReferencedContainer = "container:bp.xcodeproj"> - - - - @@ -76,12 +70,6 @@ - - - - diff --git a/bp/src/BPConfiguration.h b/bp/src/BPConfiguration.h index 0b0f8a79..41879bd4 100644 --- a/bp/src/BPConfiguration.h +++ b/bp/src/BPConfiguration.h @@ -116,6 +116,7 @@ typedef NS_ENUM(NSInteger, BPProgram) { @property (nonatomic) BOOL testing_HangAppOnLaunch; @property (nonatomic) BOOL testing_NoAppWillRun; @property (nonatomic) NSNumber *testing_crashOnAttempt; +@property (nonatomic, strong) NSString *testing_ExecutionPlan; // Generated fields @property (nonatomic, strong) NSString *xcodePath; diff --git a/bp/src/BPConfiguration.m b/bp/src/BPConfiguration.m index 8d85b7c3..93832c30 100644 --- a/bp/src/BPConfiguration.m +++ b/bp/src/BPConfiguration.m @@ -18,10 +18,10 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { - BP_VALUE = 1, // Single value - BP_LIST = 1 << 1, // List value - BP_PATH = 1 << 2, // Single value, CWD will be prepended - BP_BOOL = 1 << 3, // Boolean value + BP_VALUE = 1 << 0, // Single value + BP_LIST = 1 << 1, // List value + BP_PATH = 1 << 2, // Single value, CWD will be prepended + BP_BOOL = 1 << 3, // Boolean value BP_INTEGER = 1 << 4, // Integer value }; @@ -103,9 +103,9 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { {'q', "quiet", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "quiet", "Turn off all output except fatal errors."}, {'F', "only-retry-failed", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "onlyRetryFailed", - "If `failure-`tolerance` is > 0, only retry tests that failed."}, + "Only retry failed tests instead of all. Also retry test that timed-out/crashed. Note that app crashes are fatal even if the test passes on retry."}, {'l', "list-tests", BP_MASTER, NO, NO, no_argument, NULL, BP_VALUE | BP_BOOL, "listTestsOnly", - "Only list tests in bundle"}, + "Only list tests and exit without executing tests."}, {'v', "verbose", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "verboseLogging", "Enable verbose logging"}, {'k', "keep-individual-test-reports", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "keepIndividualTestReports", @@ -371,6 +371,7 @@ - (id)mutableCopyWithZone:(NSZone *)zone { newConfig.testing_HangAppOnLaunch = self.testing_HangAppOnLaunch; newConfig.testing_NoAppWillRun = self.testing_NoAppWillRun; newConfig.testing_crashOnAttempt = self.testing_crashOnAttempt; + newConfig.testing_ExecutionPlan = self.testing_ExecutionPlan; newConfig.xcTestRunPath = self.xcTestRunPath; newConfig.testPlanPath = self.testPlanPath; newConfig.xcTestRunDict = self.xcTestRunDict; diff --git a/bp/src/BPExitStatus.h b/bp/src/BPExitStatus.h index 47573461..41f6110b 100644 --- a/bp/src/BPExitStatus.h +++ b/bp/src/BPExitStatus.h @@ -10,18 +10,18 @@ #import typedef NS_ENUM(NSInteger, BPExitStatus) { - BPExitStatusTestsAllPassed = 0, - BPExitStatusTestsFailed = 1, - BPExitStatusSimulatorCreationFailed = 2, - BPExitStatusSimulatorCrashed = 3, - BPExitStatusInstallAppFailed = 4, - BPExitStatusLaunchAppFailed = 5, - BPExitStatusTestTimeout = 6, - BPExitStatusAppCrashed = 7, - BPExitStatusInterrupted = 8, - BPExitStatusSimulatorDeleted = 9, - BPExitStatusUninstallAppFailed = 10, - BPExitStatusSimulatorReuseFailed = 11, + BPExitStatusAllTestsPassed = 0, + BPExitStatusTestsFailed = 1 << 0, + BPExitStatusSimulatorCreationFailed = 1 << 1, + BPExitStatusInstallAppFailed = 1 << 2, + BPExitStatusInterrupted = 1 << 3, + BPExitStatusSimulatorCrashed = 1 << 4, + BPExitStatusLaunchAppFailed = 1 << 5, + BPExitStatusTestTimeout = 1 << 6, + BPExitStatusAppCrashed = 1 << 7, + BPExitStatusSimulatorDeleted = 1 << 8, + BPExitStatusUninstallAppFailed = 1 << 9, + BPExitStatusSimulatorReuseFailed = 1 << 10 }; @protocol BPExitStatusProtocol diff --git a/bp/src/BPExitStatus.m b/bp/src/BPExitStatus.m index 1519254f..c066086e 100644 --- a/bp/src/BPExitStatus.m +++ b/bp/src/BPExitStatus.m @@ -13,15 +13,18 @@ @implementation BPExitStatusHelper -// Exit status to string -+ (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus { ++ (NSString *)simpleExitStatus:(BPExitStatus)exitStatus { switch (exitStatus) { - case BPExitStatusTestsAllPassed: - return @"BPExitStatusTestsAllPassed"; + case BPExitStatusAllTestsPassed: + return @"BPExitStatusAllTestsPassed"; case BPExitStatusTestsFailed: return @"BPExitStatusTestsFailed"; case BPExitStatusSimulatorCreationFailed: return @"BPExitStatusSimulatorCreationFailed"; + case BPExitStatusInstallAppFailed: + return @"BPExitStatusInstallAppFailed"; + case BPExitStatusInterrupted: + return @"BPExitStatusInterrupted"; case BPExitStatusSimulatorCrashed: return @"BPExitStatusSimulatorCrashed"; case BPExitStatusLaunchAppFailed: @@ -30,17 +33,30 @@ + (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus { return @"BPExitStatusTestTimeout"; case BPExitStatusAppCrashed: return @"BPExitStatusAppCrashed"; - case BPExitStatusInstallAppFailed: - return @"BPExitStatusInstallAppFailed"; - case BPExitStatusInterrupted: - return @"BPExitStatusInterrupted"; case BPExitStatusSimulatorDeleted: return @"BPExitStatusSimulatorDeleted"; + case BPExitStatusUninstallAppFailed: + return @"BPExitStatusUninstallAppFailed"; case BPExitStatusSimulatorReuseFailed: return @"BPExitStatusSimulatorReuseFailed"; default: - return @"UNKNOWN_BPEXITSTATUS"; + return [NSString stringWithFormat:@"UNKNOWN_BPEXITSTATUS - %ld", (long)exitStatus]; + } +} + +// Exit status to string ++ (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus { + if (exitStatus == BPExitStatusAllTestsPassed) + return @"BPExitStatusAllTestsPassed"; + + NSString *exitStatusString = @""; + while (exitStatus > 0) { + BPExitStatus prevExitStatus = exitStatus; + exitStatus = exitStatus & (exitStatus - 1); + exitStatusString = [exitStatusString stringByAppendingFormat:@"%@ ", [self simpleExitStatus:(prevExitStatus - exitStatus)]]; } + + return [exitStatusString stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]]; } @end diff --git a/bp/src/BPSimulator.m b/bp/src/BPSimulator.m index c65860e0..6af1d1a1 100644 --- a/bp/src/BPSimulator.m +++ b/bp/src/BPSimulator.m @@ -498,6 +498,9 @@ - (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCom if (self.config.testing_HangAppOnLaunch) { mutableAppLaunchEnv[@"_BP_TEST_HANG_ON_LAUNCH"] = @"YES"; } + if (self.config.testing_ExecutionPlan) { + mutableAppLaunchEnv[@"_BP_TEST_EXECUTION_PLAN"] = self.config.testing_ExecutionPlan; + } appLaunchEnvironment = [mutableAppLaunchEnv copy]; NSDictionary *options = @{ kOptionsArgumentsKey: argsAndEnv[@"args"], @@ -622,7 +625,7 @@ - (BOOL)didTestStart { - (BOOL)checkFinished { if ([self.monitor isExecutionComplete]) { switch ([self.monitor exitStatus]) { - case BPExitStatusTestsAllPassed: + case BPExitStatusAllTestsPassed: case BPExitStatusTestsFailed: return YES; default: diff --git a/bp/src/BPStats.m b/bp/src/BPStats.m index 90f84ab8..5bda6529 100644 --- a/bp/src/BPStats.m +++ b/bp/src/BPStats.m @@ -240,7 +240,7 @@ -(unsigned long)bundleID { } - (NSString *)resultToCname:(NSString *)result { - if ([result isEqualToString:@"PASSED"] || [result isEqualToString:@"BPExitStatusTestsAllPassed"]) { + if ([result isEqualToString:@"PASSED"] || [result isEqualToString:@"BPExitStatusAllTestsPassed"]) { return @"good"; } else if ([result isEqualToString:@"FAILED"] || [result isEqualToString:@"BPExitStatusTestsFailed"]) { return @"bad"; diff --git a/bp/src/BPUtils.m b/bp/src/BPUtils.m index c6e36e56..10fd8c47 100644 --- a/bp/src/BPUtils.m +++ b/bp/src/BPUtils.m @@ -379,7 +379,6 @@ + (NSDictionary *)loadSimpleJsonFile:(NSString *)filePath NSDictionary *testsToRunByFilePath = [BPUtils getTestsToRunByFilePathWithConfig:config andXCTestFiles:xcTestFiles]; for(NSString *filePath in testsToRunByFilePath) { - NSLog(@"filePath=%@ Tests to run in this filePath=%@", filePath, [testsToRunByFilePath objectForKey:filePath]); NSSet *bundleTestsToRun = [testsToRunByFilePath objectForKey:filePath]; double __block testBundleExecutionTime = 0.0; [bundleTestsToRun enumerateObjectsUsingBlock:^(id _Nonnull test, BOOL * _Nonnull stop) { @@ -419,7 +418,6 @@ + (double)getTotalTimeWithConfig:(BPConfiguration *)config NSDictionary *testsToRunByFilePath = [BPUtils getTestsToRunByFilePathWithConfig:config andXCTestFiles:xcTestFiles]; for(NSString *filePath in testsToRunByFilePath) { - NSLog(@"filePath=%@ Tests to run in this filePath=%@", filePath, [testsToRunByFilePath objectForKey:filePath]); NSSet *bundleTestsToRun = [testsToRunByFilePath objectForKey:filePath]; double __block testBundleExecutionTime = 0.0; [bundleTestsToRun enumerateObjectsUsingBlock:^(id _Nonnull test, BOOL * _Nonnull stop) { diff --git a/bp/src/Bluepill.m b/bp/src/Bluepill.m index ac88364f..9d466058 100644 --- a/bp/src/Bluepill.m +++ b/bp/src/Bluepill.m @@ -117,14 +117,16 @@ - (void)beginWithContext:(BPExecutionContext *)context { // Retry from the beginning (default) or failed tests only if onlyRetryFailed is true - (void)retry { - // There were test failures. If our failure tolerance is 0, then we're good with that. - if (self.failureTolerance == 0) { + // There were test failures. Check if it can be retried. + if (![self canRetryOnError] || self.failureTolerance <= 0) { // If there is no more retries, set the final exitCode to current context's exitCode - self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus; + self.finalExitStatus |= self.context.finalExitStatus; [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; self.exitLoop = YES; return; } + // Resetting the failed bit since the test is being retried + self.context.finalExitStatus &= ~self.context.exitStatus; [self.context.parser cleanup]; // Otherwise, reduce our failure tolerance count and retry self.failureTolerance -= 1; @@ -153,37 +155,37 @@ - (void)retry { // - BPExitStatusUninstallAppFailed // - BPExitStatusLaunchAppFailed - (void)recover { - // If error retry reach to the max, then return - if (self.retries == [self.config.errorRetriesCount integerValue]) { - self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus; - [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; - self.exitLoop = YES; - [BPUtils printInfo:ERROR withString:@"Too many retries have occurred. Giving up."]; - return; - } - - [self.context.parser cleanup]; - // If we're not retrying only failed tests, we need to get rid of our saved tests, so that we re-execute everything. Recopy config. - if (self.executionConfigCopy.onlyRetryFailed == NO) { - self.executionConfigCopy = [self.config copy]; - } - // Increment the retry count - self.retries += 1; - - // Log some useful information to the log - [BPUtils printInfo:INFO withString:@"Exit Status: %@", [BPExitStatusHelper stringFromExitStatus:self.context.exitStatus]]; - [BPUtils printInfo:INFO withString:@"Failure Tolerance: %lu", self.failureTolerance]; - [BPUtils printInfo:INFO withString:@"Retry count: %lu", self.retries]; - - // Then start again from the beginning - [BPUtils printInfo:INFO withString:@"Recovering from tooling problem"]; - NEXT([self begin]); + // If error retry reach to the max, then return + if (![self canRetryOnError]) { + self.finalExitStatus |= self.context.finalExitStatus; + [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; + self.exitLoop = YES; + [BPUtils printInfo:ERROR withString:@"Too many retries have occurred. Giving up."]; + return; + } + + [self.context.parser cleanup]; + // If we're not retrying only failed tests, we need to get rid of our saved tests, so that we re-execute everything. Recopy config. + if (self.executionConfigCopy.onlyRetryFailed == NO) { + self.executionConfigCopy = [self.config copy]; + } + // Increment the retry count + self.retries += 1; + + // Log some useful information to the log + [BPUtils printInfo:INFO withString:@"Exit Status: %@", [BPExitStatusHelper stringFromExitStatus:self.context.exitStatus]]; + [BPUtils printInfo:INFO withString:@"Failure Tolerance: %lu", self.failureTolerance]; + [BPUtils printInfo:INFO withString:@"Retry count: %lu", self.retries]; + + // Then start again from the beginning + [BPUtils printInfo:INFO withString:@"Recovering from tooling problem"]; + NEXT([self begin]); } // Proceed to next test case - (void)proceed { - if (self.retries == [self.config.errorRetriesCount integerValue]) { - self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus; + if (![self canRetryOnError]) { + self.finalExitStatus |= self.context.finalExitStatus; [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; self.exitLoop = YES; [BPUtils printInfo:ERROR withString:@"Too many retries have occurred. Giving up."]; @@ -194,7 +196,8 @@ - (void)proceed { [BPUtils printInfo:INFO withString:@"Failure Tolerance: %lu", self.failureTolerance]; [BPUtils printInfo:INFO withString:@"Retry count: %lu", self.retries]; self.context.attemptNumber = self.retries + 1; // set the attempt number - self.context.exitStatus = BPExitStatusTestsAllPassed; // reset exitStatus + self.context.exitStatus = BPExitStatusAllTestsPassed; // reset exitStatus + [BPUtils printInfo:INFO withString:@"Proceeding to next test"]; NEXT([self beginWithContext:self.context]); } @@ -502,13 +505,13 @@ - (void)runnerCompletedWithContext:(BPExecutionContext *)context { // If we crashed, we need to retry [self deleteSimulatorWithContext:context andStatus:BPExitStatusSimulatorCrashed]; } else if (self.config.keepSimulator - && (context.runner.exitStatus == BPExitStatusTestsAllPassed + && (context.runner.exitStatus == BPExitStatusAllTestsPassed || context.runner.exitStatus == BPExitStatusTestsFailed)) { context.exitStatus = [context.runner exitStatus]; NEXT([self finishWithContext:context]); } else { // If the tests failed, save as much debugging info as we can. XXX: Put this behind a flag - if (context.runner.exitStatus != BPExitStatusTestsAllPassed && _config.saveDiagnosticsOnError) { + if (context.runner.exitStatus != BPExitStatusAllTestsPassed && _config.saveDiagnosticsOnError) { [BPUtils printInfo:INFO withString:@"Saving Diagnostics for Debugging"]; [BPUtils saveDebuggingDiagnostics:_config.outputDirectory]; } @@ -578,36 +581,17 @@ - (void)deleteSimulatorOnlyTaskWithContext:(BPExecutionContext *)context { } } -- (BOOL)hasRemainingTestsInContext:(BPExecutionContext *)context { - // Make sure we're not doing unnecessary work on the next run. - NSMutableSet *testsRemaining = [[NSMutableSet alloc] initWithArray:context.config.allTestCases]; - NSSet *testsToSkip = [[NSSet alloc] initWithArray:context.config.testCasesToSkip]; - [testsRemaining minusSet:testsToSkip]; - return ([testsRemaining count] > 0); -} - /** Scenarios: - 1. crash/time out and proceed passes -> Crash/Timeout - 2. crash/time out and retry passes -> AllPass + 1. crash and proceed passes -> Crash + 2. time out and retry passes -> AllPass 3. failure and retry passes -> AllPass 4. happy all pass -> AllPassed 5. failure and still fails -> TestFailed */ - (void)finishWithContext:(BPExecutionContext *)context { - - // Because BPExitStatusTestsAllPassed is 0, we must check it explicitly against - // the run rather than the aggregate bitmask built with finalExitStatus - - if (![self hasRemainingTestsInContext:context] && (context.attemptNumber <= [context.config.errorRetriesCount integerValue])) { - [BPUtils printInfo:INFO withString:@"No more tests to run."]; - [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; - // TODO: Temporarily disabling the fix from PR#338 while the issue is being investigated - // self.finalExitStatus = context.exitStatus; - self.finalExitStatus = context.finalExitStatus | context.exitStatus; - self.exitLoop = YES; - return; - } + context.finalExitStatus |= context.exitStatus; + [BPUtils printInfo:INFO withString:@"Attempt's Exit Status: %@", [BPExitStatusHelper stringFromExitStatus:context.exitStatus]]; switch (context.exitStatus) { // BP exit handler @@ -615,23 +599,15 @@ - (void)finishWithContext:(BPExecutionContext *)context { self.exitLoop = YES; return; - // MARK: Test suite completed - // If there is no test crash/time out, we retry from scratch case BPExitStatusTestsFailed: NEXT([self retry]); return; - case BPExitStatusTestsAllPassed: - // Check previous result - if (context.finalExitStatus != BPExitStatusTestsAllPassed) { - // If there is a test crashed/timed out before, retry from scratch - NEXT([self retry]); - } else { - // If it is a real all pass, exit - self.exitLoop = YES; - return; - } + case BPExitStatusAllTestsPassed: + // Time to exit + self.finalExitStatus |= BPExitStatusAllTestsPassed; + self.exitLoop = YES; return; // Recover from scratch if there is tooling failure. @@ -645,21 +621,26 @@ - (void)finishWithContext:(BPExecutionContext *)context { // If it is test hanging or crashing, we set final exit code of current context and proceed. case BPExitStatusTestTimeout: - context.finalExitStatus = BPExitStatusTestTimeout; + if (!self.config.onlyRetryFailed) { + self.finalExitStatus |= context.exitStatus; + } NEXT([self proceed]); return; + case BPExitStatusAppCrashed: - context.finalExitStatus = BPExitStatusAppCrashed; + // Remember the app crash and report whether a retry passes or not + self.finalExitStatus |= context.exitStatus; NEXT([self proceed]); return; + case BPExitStatusSimulatorDeleted: case BPExitStatusSimulatorReuseFailed: - self.finalExitStatus = context.exitStatus; + self.finalExitStatus |= context.finalExitStatus; [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; self.exitLoop = YES; return; } - + [BPUtils printInfo:ERROR withString:@"%s:%d YOU SHOULDN'T BE HERE. exitStatus = %@, finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:context.exitStatus], [BPExitStatusHelper stringFromExitStatus:context.finalExitStatus]]; } // MARK: Helpers @@ -676,6 +657,20 @@ - (BPSimulator *)test_simulator { return self.context.runner; } +- (BOOL)canRetryOnError { + NSInteger maxErrorRetryCount = [self.config.errorRetriesCount integerValue]; + if (self.retries < maxErrorRetryCount) { + return true; + } + + if (self.retries > maxErrorRetryCount) { + // If retries strictly exceeds the max error retry, then we must have incremented it beyond the limit somehow. + // It is safe to halt retries here, but log to alert unexpected behavior. + [BPUtils printInfo:ERROR withString:@"Current retry count (%d) exceeded maximum retry count (%d)!", (int) self.retries, (int) maxErrorRetryCount]; + } + return false; +} + int __line; NSString *__function; NSString *__from; diff --git a/bp/src/SimulatorMonitor.m b/bp/src/SimulatorMonitor.m index 6d42f11b..06cf2938 100644 --- a/bp/src/SimulatorMonitor.m +++ b/bp/src/SimulatorMonitor.m @@ -68,7 +68,7 @@ - (void)onAllTestsEnded { if (self.failureCount) { self.exitStatus = BPExitStatusTestsFailed; } else { - self.exitStatus = BPExitStatusTestsAllPassed; + self.exitStatus = BPExitStatusAllTestsPassed; } [[BPStats sharedStats] endTimer:ALL_TESTS withResult:[BPExitStatusHelper stringFromExitStatus: self.exitStatus]]; [BPUtils printInfo:INFO withString:@"All Tests Completed."]; diff --git a/bp/tests/BPUtilsTests.m b/bp/tests/BPUtilsTests.m index 5a0b2165..566c5691 100644 --- a/bp/tests/BPUtilsTests.m +++ b/bp/tests/BPUtilsTests.m @@ -8,6 +8,7 @@ // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. #import +#import "BPExitStatus.h" #import "BPUtils.h" #import "BPXCTestFile.h" #import "BPTestHelper.h" @@ -110,4 +111,41 @@ - (void) testTrailingParanthesesInTestNames { XCTAssert([testCasesWithParantheses count] == 0); } +- (void) testExitStatus { + BPExitStatus exitCode; + + exitCode = 0; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusAllTestsPassed"]); + exitCode = 1; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestsFailed"]); + exitCode = 2; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCreationFailed"]); + exitCode = 4; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusInstallAppFailed"]); + exitCode = 8; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusInterrupted"]); + exitCode = 16; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCrashed"]); + exitCode = 32; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusLaunchAppFailed"]); + exitCode = 64; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestTimeout"]); + exitCode = 128; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusAppCrashed"]); + exitCode = 256; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorDeleted"]); + exitCode = 512; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusUninstallAppFailed"]); + exitCode = 1024; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorReuseFailed"]); + exitCode = 3; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestsFailed BPExitStatusSimulatorCreationFailed"]); + exitCode = 192; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestTimeout BPExitStatusAppCrashed"]); + exitCode = 2048; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"UNKNOWN_BPEXITSTATUS - 2048"]); + exitCode = 2050; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCreationFailed UNKNOWN_BPEXITSTATUS - 2048"]); +} + @end diff --git a/bp/tests/BluepillTests.m b/bp/tests/BluepillTests.m index f91cc2ca..a81fb2ab 100644 --- a/bp/tests/BluepillTests.m +++ b/bp/tests/BluepillTests.m @@ -42,12 +42,12 @@ - (void)setUp { self.config = [[BPConfiguration alloc] initWithProgram:BP_SLAVE]; self.config.testBundlePath = testBundlePath; self.config.appBundlePath = hostApplicationPath; - self.config.stuckTimeout = @80; + self.config.stuckTimeout = @40; self.config.xcodePath = [BPUtils runShell:@"/usr/bin/xcode-select -print-path"]; self.config.runtime = @BP_DEFAULT_RUNTIME; self.config.repeatTestsCount = @1; self.config.errorRetriesCount = @0; - self.config.testCaseTimeout = @40; + self.config.testCaseTimeout = @20; self.config.deviceType = @BP_DEFAULT_DEVICE_TYPE; self.config.headlessMode = YES; self.config.videoPaths = @[[BPTestHelper sampleVideoPath]]; @@ -90,7 +90,10 @@ - (void)testAppThatCrashesOnLaunch { self.config.testing_CrashAppOnLaunch = YES; self.config.stuckTimeout = @3; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; - XCTAssert(exitCode == BPExitStatusAppCrashed, @"Expected: %ld Got: %ld", (long)BPExitStatusAppCrashed, (long)exitCode); + XCTAssert(exitCode == BPExitStatusAppCrashed, + @"Expected: %@ Got: %@", + [BPExitStatusHelper stringFromExitStatus:BPExitStatusAppCrashed], + [BPExitStatusHelper stringFromExitStatus:exitCode]); } - (void)testAppThatHangsOnLaunch { @@ -99,7 +102,10 @@ - (void)testAppThatHangsOnLaunch { self.config.testing_HangAppOnLaunch = YES; self.config.stuckTimeout = @3; BPExitStatus exitCode = [[[Bluepill alloc] initWithConfiguration:self.config] run]; - XCTAssert(exitCode == BPExitStatusSimulatorCrashed); + XCTAssert(exitCode == BPExitStatusSimulatorCrashed, + @"Expected: %@ Got: %@", + [BPExitStatusHelper stringFromExitStatus:BPExitStatusSimulatorCrashed], + [BPExitStatusHelper stringFromExitStatus:exitCode]); } - (void)testRecoverSimulatorOnCrash { @@ -141,7 +147,7 @@ - (void)testRunningOnlyCertainTestcases { ]; BPExitStatus exitCode = [[[Bluepill alloc] initWithConfiguration:self.config] run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); NSString *reportPath = [outputDir stringByAppendingPathComponent:@"TEST-BPSampleAppTests-1-results.xml"]; NSError *error; @@ -171,7 +177,7 @@ - (void)testRunningAndIgnoringCertainTestCases { self.config.testCasesToSkip = @[@"BPSampleAppTests/testCase173"]; BPExitStatus exitCode = [[[Bluepill alloc] initWithConfiguration:self.config] run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); NSString *reportPath = [outputDir stringByAppendingPathComponent:@"TEST-BPSampleAppTests-1-results.xml"]; NSError *error; @@ -213,7 +219,6 @@ - (void)testReportWithAppCrashingAndRetryOnlyFailedTestsSet { NSString *tempDir = NSTemporaryDirectory(); NSError *error; NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppCrashingTestsSetTempDir", tempDir] withError:&error]; - // NSLog(@"output directory is %@", outputDir); self.config.outputDirectory = outputDir; self.config.errorRetriesCount = @1; self.config.failureTolerance = @1; @@ -246,7 +251,7 @@ - (void)DISABLE_testAppCrashingAndRetryReportsCorrectExitCode { self.config.onlyRetryFailed = YES; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; - XCTAssertTrue(exitCode == BPExitStatusTestsAllPassed); + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); } - (void)testReportWithFatalErrorTestsSet { @@ -258,7 +263,7 @@ - (void)testReportWithFatalErrorTestsSet { NSLog(@"output directory is %@", outputDir); self.config.outputDirectory = outputDir; self.config.errorRetriesCount = @2; - self.config.testCaseTimeout = @60; // make sure we don't time-out + self.config.testCaseTimeout = @30; // make sure we don't time-out BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssertTrue(exitCode == BPExitStatusAppCrashed); @@ -275,8 +280,10 @@ - (void)testReportWithFatalErrorTestsSet { - (void)testReportWithAppHangingTestsSet { // Testcase timeout should be set larger than the stuck timeout - self.config.stuckTimeout = @40; + self.config.testCaseTimeout = @20; + self.config.stuckTimeout = @15; self.config.errorRetriesCount = @0; + self.config.testing_ExecutionPlan = @"TIMEOUT"; NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; NSString *tempDir = NSTemporaryDirectory(); @@ -301,6 +308,7 @@ - (void)testReportWithAppHangingTestsShouldReturnFailure { self.config.stuckTimeout = @6; self.config.failureTolerance = @0; self.config.errorRetriesCount = @4; + self.config.testing_ExecutionPlan = @"TIMEOUT"; NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; NSString *tempDir = NSTemporaryDirectory(); @@ -317,6 +325,171 @@ - (void)testReportWithAppHangingTestsShouldReturnFailure { [self assertGotReport:junitReportPath isEqualToWantReport:expectedFilePath]; } +/** + Execution plan: TIMEOUT, CRASH, PASS + */ +- (void)testReportFailureOnTimeoutCrashAndPass { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAppCrashed); +} + +/** + Execution plan: CRASH, TIMEOUT, PASS + */ +- (void)testReportFailureOnCrashTimeoutAndPass { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"CRASH TIMEOUT PASS"; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAppCrashed); +} + +/** + Execution plan: FAIL, TIMEOUT, PASS + */ +- (void)testReportSuccessOnFailTimeoutAndPass { + self.config.stuckTimeout = @6; + self.config.failureTolerance = @1; + self.config.testing_ExecutionPlan = @"FAIL TIMEOUT PASS"; + self.config.errorRetriesCount = @3; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + +/** + Execution plan: FAIL, TIMEOUT, PASS + */ +- (void)testReportFailureOnFailTimeoutAndPass { + self.config.stuckTimeout = @6; + self.config.failureTolerance = @0; + self.config.testing_ExecutionPlan = @"FAIL TIMEOUT PASS"; + self.config.errorRetriesCount = @3; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusTestsFailed); +} + +/** + Execution plan: TIMEOUT, PASS + */ +- (void)testReportSuccessOnTimeoutAndPassOnRetry { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"TIMEOUT PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = YES; + self.config.failureTolerance = @0; // Not relevant + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + +/** + Execution plan: TIMEOUT (NO RETRY)) + */ +- (void)testReportFailureOnTimeoutAndNoRetry { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"TIMEOUT"; + self.config.errorRetriesCount = @2; + self.config.onlyRetryFailed = NO; + self.config.failureTolerance = @1; // Not relevant since it's not a test failure + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusTestTimeout); +} + +/** + Execution plan: FAIL and PASS on retry all + */ +- (void)testReportSuccessOnFailedTestAndPassOnRetryAll { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"FAIL PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = NO; // Indicates to retry all tests when a test fails + self.config.failureTolerance = @1; // Even though failureTolerance is non-zero it wouldn't retry because onlyRetryFailed = NO + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + +/** + Execution plan: FAIL, PASS + */ +- (void)testReportSuccessOnTestFailedAndPassOnRetry { + self.config.stuckTimeout = @6; + self.config.failureTolerance = @1; + self.config.testing_ExecutionPlan = @"FAIL PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + - (void)testReportWithFailingTestsSetAndDiagnostics { NSString *tempDir = NSTemporaryDirectory(); NSError *error; @@ -385,7 +558,7 @@ - (void)testRunWithPassingTestsSet { self.config.testCasesToSkip = @[@"BPSampleAppTests/testCase000"]; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); } - (void)testRunWithFailingTestsSet { @@ -421,6 +594,7 @@ - (void)testKeepSimulatorWithAppHangingTestsSet { NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; self.config.keepSimulator = YES; + self.config.testing_ExecutionPlan = @"TIMEOUT"; Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; @@ -434,7 +608,7 @@ - (void)testDeleteSimulatorOnly { Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); XCTAssertNotNil(bp.test_simulatorUDID); self.config.deleteSimUDID = bp.test_simulatorUDID; @@ -472,7 +646,7 @@ - (void)testRunUITest { self.config.outputDirectory = outputDir; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; self.config.testRunnerAppPath = nil; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); } @@ -485,7 +659,7 @@ - (void)testCopySimulatorPreferencesFile { Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); XCTAssertNotNil(bp.test_simulatorUDID); NSURL *preferencesFile = bp.test_simulator.preferencesFile; @@ -513,7 +687,7 @@ - (void)testRunScript { Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); XCTAssertNotNil(bp.test_simulatorUDID); NSString *devicePath = bp.test_simulator.device.devicePath; diff --git a/bp/tests/Resource Files/hanging_tests.xml b/bp/tests/Resource Files/hanging_tests.xml index 79e78efd..54c45afe 100644 --- a/bp/tests/Resource Files/hanging_tests.xml +++ b/bp/tests/Resource Files/hanging_tests.xml @@ -1,8 +1,9 @@ - - - - + + + + +