Skip to content

Commit

Permalink
Merge exit code fixes from PR#432 and PR#430 to Xcode 11.2 branch (#435)
Browse files Browse the repository at this point in the history 
* make retry checks more resilient to prevent infinite retries (#432)

Also, log when retry count exceeded max retries.

Co-authored-by: Mansfield Mark <mmark@pinterest.com>

* Added a few more tests to mock test failure scenarios and fixed a few final Exit Status issues  (#430)

* Adding the ability to mock different test executions in Bluepill

* Add a few more tests and fixed the exit status issues

Changing the Exit Status to powers of 2 to make them mergeable.
Easy consolidation/aggregation of exit code.
In case of failure reporting, report all exit status that happened over multiple attempts, if any.
Prevent extra attempt, which does nothing, by eliminating the use of `hasRemainingTestsInContext` which is not accurate.

* Re-enabling some disabled/skipped BP tests and fixed a fragile packer test

A fragile packer test broke when the total number of tests changed. So, fixed
the testing method to fix the test.

Also, re-enabled a few tests in BluepillTests that were being skipped/disabled
for some reason.

* Renaming BPExitStatusTestsAllPassed to BPExitStatusAllTestsPassed

Renaming the all-good exit status from BPExitStatusTestsAllPassed
to BPExitStatusAllTestsPassed and incorporated comments. Tweaked
the tests a bit to reduce the regression testing duration.

Co-authored-by: Mansfield Mark <RainNapper@users.noreply.github.com>
Co-authored-by: Mansfield Mark <mmark@pinterest.com>
  • Loading branch information
@ravimandala @RainNapper
3 people committed Apr 16, 2020
1 parent a5e6eae commit e6d200a
Show file tree
Hide file tree
Showing 16 changed files with 502 additions and 177 deletions.
87 changes: 84 additions & 3 deletions BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m
View file
Expand Up @@ -15,19 +15,100 @@ @interface BPSampleAppHangingTests : XCTestCase

@implementation BPSampleAppHangingTests

-(long)attemptFromSimulatorVersionInfo:(NSString *)simulatorVersionInfo {
// simulatorVersionInfo is something like
// CoreSimulator 587.35 - Device: BP93497-2-2 (7AB3D528-5473-401A-B23E-2E2E86C73861) - Runtime: iOS 12.2 (16E226) - DeviceType: iPhone 7
NSLog(@"Dissecting version info %@ to extra attempt number.", simulatorVersionInfo);
NSArray<NSString *> *parts = [simulatorVersionInfo componentsSeparatedByString:@" - "];
NSString *deviceString = parts[1];
// Device: BP93497-2-2 (7AB3D528-5473-401A-B23E-2E2E86C73861)
parts = [deviceString componentsSeparatedByString:@" "];
NSString *device = parts[1];
// BP93497-2-2
parts = [device componentsSeparatedByString:@"-"];
NSString *attempt = parts[1];
return [attempt longLongValue];
}

-(void)extractPlanAndExecuteActions:(int)index {
NSDictionary *env = [[NSProcessInfo processInfo] environment];
NSString *simulatorVersionInfo = [env objectForKey:@"SIMULATOR_VERSION_INFO"];
long attempt = [self attemptFromSimulatorVersionInfo:simulatorVersionInfo];
NSString *executionPlan = [env objectForKey:@"_BP_TEST_EXECUTION_PLAN"];
if (!executionPlan) {
NSLog(@"No execution plan found in attempt#%ld. Failing the test.", attempt);
XCTAssert(NO);
return;
}
NSLog(@"Received execution plan %@ on attempt#%ld for this test.", executionPlan, attempt);
NSArray *setsOfPlans = [executionPlan componentsSeparatedByString:@";"];
if (index >= [setsOfPlans count]) {
NSLog(@"Not enough plans for test#%d in execution plan: '%@'.", index, executionPlan);
XCTAssert(YES);
return;
}
NSString *currentPlan = setsOfPlans[index];
currentPlan = [currentPlan stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
NSArray *array = [currentPlan componentsSeparatedByCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
if (attempt > [array count]) {
NSLog(@"Passing on attempt#%ld, by default, as there is no action defined in the execution plan", (long)attempt);
XCTAssert(YES);
return;
}
NSString *action = array[attempt - 1];
if ([action isEqualToString:@"TIMEOUT"]) {
NSLog(@"Entering into an infinite loop on attempt#%ld to timeout as per the execution plan", (long)attempt);
while(1) {
}
return;
} else if ([action isEqualToString:@"PASS"]) {
NSLog(@"Passing on attempt#%ld based on execution plan", (long)attempt);
XCTAssert(YES);
return;
} else if ([action isEqualToString:@"FAIL"]) {
NSLog(@"Failing on attempt#%ld based on execution plan", (long)attempt);
XCTAssert(NO);
return;
} else if ([action isEqualToString:@"CRASH"]) {
NSLog(@"Crashing on attempt#%ld based on execution plan", (long)attempt);
// ok, let's crash and burn
int *pointer = nil;
*pointer = 1;
return;
}
NSLog(@"Failing on attempt#%ld as an unidentified action is encountered in the execution plan", (long)attempt);
XCTAssert(NO);
return;
}

- (void)setUp {
[super setUp];
// Put setup code here. This method is called before the invocation of each test method in the class.

}

- (void)tearDown {
// Put teardown code here. This method is called after the invocation of each test method in the class.
[super tearDown];
}

- (void)testAppHanging {
while(TRUE){};
- (void)testASimpleTest {
XCTAssert(YES);
}

- (void)testBasedOnExecutionPlan {
[self extractPlanAndExecuteActions:0];
}

- (void)testCaseFinal {
XCTAssert(YES);
}

- (void)testDoubleBasedOnExecutionPlan {
[self extractPlanAndExecuteActions:1];
}

- (void)testEndFinal {
XCTAssert(YES);
}

@end
89 changes: 55 additions & 34 deletions README.md
View file
Expand Up @@ -54,39 +54,60 @@ $ bluepill -c config.json
A full list supported options are listed here.


| Config Arguments | Command Line Arguments | Explanation | Required | Default value |
|:----------------------:|:----------------------:|------------------------------------------------------------------------------------|:--------:|:----------------:|
| `app` | -a | The path to the host application to execute (your .app) | N | n/a |
| `xctestrun-path` | | The path to the `.xctestrun` file that xcode leaves when you `build-for-testing`. | Y | n/a |
| `output-dir` | -o | Directory where to put output log files (bluepill only) | Y | n/a |
| config | -c | Read options from the specified configuration file instead of the command line | N | n/a |
| device | -d | On which device to run the app. | N | iPhone 6 |
| exclude | -x | Exclude a testcase in the set of tests to run (takes priority over `include`). | N | empty |
| headless | -H | Run in headless mode (no GUI). | N | off |
| clone-simulator | -L | Spawn simulator by clone from simulator template. | N | off |
| xcode-path | -X | Path to xcode. | N | xcode-select -p |
| include | -i | Include a testcase in the set of tests to run (unless specified in `exclude`). | N | all tests |
| list-tests | -l | Only list tests in bundle | N | false |
| num-sims | -n | Number of simulators to run in parallel. (bluepill only) | N | 4 |
| printf-config | -P | Print a configuration file suitable for passing back using the `-c` option. | N | n/a |
| error-retries | -R | Number of times to recover from simulator/app crashing/hanging and continue running| N | 5 |
| failure-tolerance | -f | Number of times to retry on test failures | N | 0 |
| only-retry-failed | -F | When `failure-tolerance` > 0, only retry tests that failed | N | false |
| runtime | -r | What runtime to use. | N | iOS 11.1 |
| stuck-timeout | -S | Timeout in seconds for a test that seems stuck (no output). | N | 300s |
| test-timeout | -T | Timeout in seconds for a test that is producing output. | N | 300s |
| test-bundle-path | -t | The path to the test bundle to execute (single .xctest). | N | n/a |
| additional-unit-xctests| n/a | Additional XCTest bundles that is not Plugin folder | N | n/a |
| additional-ui-xctests | n/a | Additional XCTUITest bundles that is not Plugin folder | N | n/a |
| repeat-count | -C | Number of times we'll run the entire test suite (used for load testing). | N | 1 |
| no-split | -N | Test bundles you don't want to be packed into different groups to run in parallel. | N | n/a |
| quiet | -q | Turn off all output except fatal errors. | N | YES |
| diagnostics | n/a | Enable collection of diagnostics in outputDir in case of test failures | N | NO |
| help | -h | Help. | N | n/a |
| runner-app-path | -u | The test runner for UI tests. | N | n/a |
| screenshots-directory | n/a | Directory where simulator screenshots for failed ui tests will be stored | N | n/a |
| video-paths | -V | A list of videos that will be saved in the simulators | N | n/a |
| image-paths | -I | A list of images that will be saved in the simulators | N | n/a |
| Config Arguments | Command Line Arguments | Explanation | Required | Default value |
|:----------------------:|:----------------------:|-------------------------------------------------------------------------------------|:--------:|:----------------:|
| app | -a | The path to the host application to execute (your `.app`) | N | n/a |
| xctestrun-path | | The path to the `.xctestrun` file that xcode leaves when you `build-for-testing`. | Y | n/a |
| output-dir | -o | Directory where to put output log files. **(bluepill only)** | Y | n/a |
| config | -c | Read options from the specified configuration file instead of the command line. | N | n/a |
| device | -d | On which device to run the app. | N | iPhone 8 |
| exclude | -x | Exclude a testcase in the set of tests to run (takes priority over `include`). | N | empty |
| headless | -H | Run in headless mode (no GUI). | N | off |
| clone-simulator | -L | Spawn simulator by clone from simulator template. | N | off |
| xcode-path | -X | Path to xcode. | N | xcode-select -p |
| include | -i | Include a testcase in the set of tests to run (unless specified in `exclude`). | N | all tests |
| list-tests | -l | Only list tests and exit without executing tests. | N | false |
| num-sims | -n | Number of simulators to run in parallel. **(bluepill only)** | N | 4 |
| printf-config | -P | Print a configuration file suitable for passing back using the `-c` option. | N | n/a |
| error-retries | -R | Number of times to recover from simulator/app crashing/hanging and continue running.| N | 4 |
| failure-tolerance | -f | Number of times to retry on test failures | N | 0 |
| only-retry-failed | -F | Only retry failed tests instead of all. Also retry test that timed-out/crashed. | N | false |
| runtime | -r | What runtime to use. | N | iOS 13.2 |
| stuck-timeout | -S | Timeout in seconds for a test that seems stuck (no output). | N | 300s |
| test-timeout | -T | Timeout in seconds for a test that is producing output. | N | 300s |
| test-bundle-path | -t | The path to the test bundle to execute (single `.xctest`). | N | n/a |
| additional-unit-xctests| n/a | Additional XCTest bundles that is not Plugin folder | N | n/a |
| additional-ui-xctests | n/a | Additional XCTUITest bundles that is not Plugin folder | N | n/a |
| repeat-count | -C | Number of times we'll run the entire test suite (used for load testing). | N | 1 |
| no-split | -N | Test bundles you don't want to be packed into different groups to run in parallel. | N | n/a |
| quiet | -q | Turn off all output except fatal errors. | N | YES |
| diagnostics | n/a | Enable collection of diagnostics in output directory in case of test failures. | N | NO |
| help | -h | Help. | N | n/a |
| runner-app-path | -u | The test runner for UI tests. | N | n/a |
| screenshots-directory | n/a | Directory where simulator screenshots for failed ui tests will be stored. | N | n/a |
| video-paths | -V | A list of videos that will be saved in the simulators. | N | n/a |
| image-paths | -I | A list of images that will be saved in the simulators. | N | n/a |

## Exit Status

Here is a list of Bluepill exit codes. If a Bluepill execution has multiple exit codes from same or different test bundles, the final exit code is a combination of all exit codes. Note that app crashes are fatal even if the test passes on retry.

```shell
BPExitStatusAllTestsPassed = 0,
BPExitStatusTestsFailed = 1 << 0,
BPExitStatusSimulatorCreationFailed = 1 << 1,
BPExitStatusInstallAppFailed = 1 << 2,
BPExitStatusInterrupted = 1 << 3,
BPExitStatusSimulatorCrashed = 1 << 4,
BPExitStatusLaunchAppFailed = 1 << 5,
BPExitStatusTestTimeout = 1 << 6,
BPExitStatusAppCrashed = 1 << 7,
BPExitStatusSimulatorDeleted = 1 << 8,
BPExitStatusUninstallAppFailed = 1 << 9,
BPExitStatusSimulatorReuseFailed = 1 << 10
```
**Note:** Please refer to `bp/src/BPExitStatus.h` for the latest/exact exit codes.
## Demo
Expand Down Expand Up @@ -138,7 +159,7 @@ If you're using [Bitrise.io](https://bitrise.io) as your CI/CD, you can start us
Latest [release](https://github.com/linkedin/bluepill/releases/).
- How to test Bluepill in Xcode
- How to test Bluepill in Xcode?
Select BPSampleApp scheme and build it first. Then you can switch back to `bluepill` or `bluepill-cli` scheme to run their tests.
Expand Down
18 changes: 13 additions & 5 deletions bluepill/tests/BPPackerTests.m
View file
Expand Up @@ -187,17 +187,18 @@ - (void)testPacking {
// Make sure we don't split when we don't want to
self.config.numSims = @4;
self.config.noSplit = @[@"BPSampleAppTests"];
bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];// withNoSplitList:@[@"BPSampleAppTests"] intoBundles:4 andError:nil];
bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil]; // withNoSplitList:@[@"BPSampleAppTests"] intoBundles:4 andError:nil];
// When we prevent BPSampleTests from splitting, BPSampleAppFatalErrorTests and BPAppNegativeTests gets split in two
want = [[want arrayByAddingObject:@"BPSampleAppFatalErrorTests"] sortedArrayUsingSelector:@selector(compare:)];
XCTAssertEqual(bundles.count, app.testBundles.count + 2);

XCTAssertEqual([bundles[0].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[1].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[2].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[3].skipTestIdentifiers count], 2);
XCTAssertEqual([bundles[4].skipTestIdentifiers count], 3);
XCTAssertEqual([bundles[3].skipTestIdentifiers count], 1);
XCTAssertEqual([bundles[4].skipTestIdentifiers count], 4);
XCTAssertEqual([bundles[5].skipTestIdentifiers count], 1);
XCTAssertEqual([bundles[6].skipTestIdentifiers count], 4);

self.config.numSims = @4;
self.config.noSplit = nil;
Expand All @@ -209,14 +210,17 @@ - (void)testPacking {
long testsPerBundle = [allTests count] / numSims;
long skipTestsPerBundle = 0;
long skipTestsInFinalBundle = 0;
long testCount = 0;
for (int i = 0; i < bundles.count; ++i) {
skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle);
skipTestsInFinalBundle = testsPerBundle * (numSims - 1);
if (i < 4) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0);
testCount += [[bundles[i] allTestCases] count];
} else if (i < bundles.count-1) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle);
testCount += testsPerBundle;
} else { /* last bundle */
skipTestsInFinalBundle = [[bundles[i] allTestCases] count] - ([allTests count] - testCount);
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle);
}
}
Expand All @@ -231,15 +235,19 @@ - (void)testPacking {

numSims = [self.config.numSims integerValue];
testsPerBundle = [allTests count] / numSims;
testCount = 0;
for (int i = 0; i < bundles.count; ++i) {
skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle);
skipTestsInFinalBundle = testsPerBundle * (numSims - 1);
if (i < 4) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0);
testCount += [[bundles[i] allTestCases] count];
} else if (i < bundles.count-1) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle);
testCount += testsPerBundle;
} else { /* last bundle */
skipTestsInFinalBundle = [[bundles[i] allTestCases] count] - ([allTests count] - testCount);
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle);

}
}

Expand Down

0 comments on commit e6d200a

Please sign in to comment.