Allow .xctestrun equivalent in config.json

BPSampleApp/BPSampleApp.xcodeproj/project.pbxproj

@@ -7,6 +7,7 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
+		01EA9E85237D290D00538FDF /* test_plan.json in Resources */ = {isa = PBXBuildFile; fileRef = 01EA9E84237D290D00538FDF /* test_plan.json */; };
 		32FEEBBD2012B93D005916EF /* BPSampleAppSwiftTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 32FEEBBC2012B93D005916EF /* BPSampleAppSwiftTests.swift */; };
 		BA4BCFC91DC47EC800592FA4 /* BPSampleAppHangingTests.m in Sources */ = {isa = PBXBuildFile; fileRef = BA4BCFC81DC47EC800592FA4 /* BPSampleAppHangingTests.m */; };
 		BA9C2DD51DD7F182007CB967 /* BPSampleAppFatalErrorTests.m in Sources */ = {isa = PBXBuildFile; fileRef = BA9C2DD41DD7F182007CB967 /* BPSampleAppFatalErrorTests.m */; };
@@ -70,6 +71,7 @@
 /* End PBXContainerItemProxy section */
 
 /* Begin PBXFileReference section */
+		01EA9E84237D290D00538FDF /* test_plan.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = test_plan.json; sourceTree = "<group>"; };
 		32FEEBBB2012B93D005916EF /* BPSampleAppTests-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "BPSampleAppTests-Bridging-Header.h"; sourceTree = "<group>"; };
 		32FEEBBC2012B93D005916EF /* BPSampleAppSwiftTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BPSampleAppSwiftTests.swift; sourceTree = "<group>"; };
 		BA4BCFC61DC47EC700592FA4 /* BPSampleAppHangingTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = BPSampleAppHangingTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -225,6 +227,7 @@
 				BAB24F3A1DB5D45E00867756 /* Main.storyboard */,
 				BAB24F3D1DB5D45E00867756 /* Assets.xcassets */,
 				BAB24F3F1DB5D45E00867756 /* LaunchScreen.storyboard */,
+				01EA9E84237D290D00538FDF /* test_plan.json */,
 				BAB24F421DB5D45E00867756 /* Info.plist */,
 				BAB24F311DB5D45E00867756 /* Supporting Files */,
 			);
@@ -500,6 +503,7 @@
 			isa = PBXResourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				01EA9E85237D290D00538FDF /* test_plan.json in Resources */,
 				BAB24F411DB5D45E00867756 /* LaunchScreen.storyboard in Resources */,
 				BAB24F3E1DB5D45E00867756 /* Assets.xcassets in Resources */,
 				BAB24F3C1DB5D45E00867756 /* Main.storyboard in Resources */,

BPSampleApp/BPSampleApp/test_plan.json

@@ -0,0 +1 @@
+

bluepill/src/BPApp.m

@@ -99,6 +99,21 @@ + (instancetype)appWithConfig:(BPConfiguration *)config
 
     BPApp *app = [[BPApp alloc] init];
     NSMutableArray<BPXCTestFile *> *allXCTestFiles = [[NSMutableArray alloc] init];
+
+    if (config.tests != nil && config.tests.count != 0) {
+        [BPUtils printInfo:INFO withString:@"Using test bundles"];
+        NSMutableArray<BPXCTestFile *> *loadedTests = [[NSMutableArray alloc] initWithCapacity:config.tests.count];
+        for (NSString *testName in config.tests) {
+            BPTestPlan *testPlan = [config.tests objectForKey:testName];
+            BPXCTestFile *xcTestFile = [BPXCTestFile BPXCTestFileFromBPTestPlan:testPlan withName:testName andError:errPtr];
+            [loadedTests addObject:xcTestFile];
+        }
+
+        app.testBundles = loadedTests;
+
+        return app;
+    }
+
     if (config.xcTestRunDict) {
         NSAssert(config.xcTestRunPath, @"");
         [BPUtils printInfo:INFO withString:@"Using xctestrun configuration"];

bluepill/tests/BPIntegrationTests.m

@@ -17,31 +17,33 @@
 #import "bp/src/BPConstants.h"
 
 @interface BPIntegrationTests : XCTestCase
-@property (nonatomic, strong) BPConfiguration* config;
 @end
 
 @implementation BPIntegrationTests
 
+- (BPConfiguration *)generateConfig {
+    NSString *hostApplicationPath = [BPTestHelper sampleAppPath];
+    NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath];
+    BPConfiguration *config = [[BPConfiguration alloc] initWithProgram:BP_MASTER];
+    config.testBundlePath = testBundlePath;
+    config.appBundlePath = hostApplicationPath;
+    config.stuckTimeout = @80;
+    config.xcodePath = [BPUtils runShell:@"/usr/bin/xcode-select -print-path"];
+    config.runtime = @BP_DEFAULT_RUNTIME;
+    config.repeatTestsCount = @1;
+    config.errorRetriesCount = @0;
+    config.failureTolerance = @0;
+    config.deviceType = @BP_DEFAULT_DEVICE_TYPE;
+    config.headlessMode = YES;
+    config.quiet = [BPUtils isBuildScript];
+    return config;
+}
+
 - (void)setUp {
     [super setUp];
-
     // Put setup code here. This method is called before the invocation of each test method in the class.
-    NSString *hostApplicationPath = [BPTestHelper sampleAppPath];
-    NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath];
-    self.config = [[BPConfiguration alloc] initWithProgram:BP_MASTER];
-    self.config.testBundlePath = testBundlePath;
-    self.config.appBundlePath = hostApplicationPath;
-    self.config.stuckTimeout = @80;
-    self.config.xcodePath = [BPUtils runShell:@"/usr/bin/xcode-select -print-path"];
-    self.config.runtime = @BP_DEFAULT_RUNTIME;
-    self.config.repeatTestsCount = @1;
-    self.config.errorRetriesCount = @0;
-    self.config.failureTolerance = @0;
-    self.config.deviceType = @BP_DEFAULT_DEVICE_TYPE;
-    self.config.headlessMode = YES;
     [BPUtils enableDebugOutput:![BPUtils isBuildScript]];
     [BPUtils quietMode:[BPUtils isBuildScript]];
-    self.config.quiet = [BPUtils isBuildScript];
 }
 
 - (void)tearDown {
@@ -50,89 +52,93 @@ - (void)tearDown {
 }
 
 - (void)testOneBPInstance {
-    self.config.numSims = @1;
+    BPConfiguration *config = [self generateConfig];
+    config.numSims = @1;
     NSError *err;
-    BPApp *app = [BPApp appWithConfig:self.config
+    BPApp *app = [BPApp appWithConfig:config
                             withError:&err];
 
     NSString *bpPath = [BPTestHelper bpExecutablePath];
-    BPRunner *runner = [BPRunner BPRunnerWithConfig:self.config withBpPath:bpPath];
+    BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
     XCTAssert(runner != nil);
     int rc = [runner runWithBPXCTestFiles:app.testBundles];
     XCTAssert(rc == 0, @"Wanted 0, got %d", rc);
     XCTAssert([runner.nsTaskList count] == 0);
 }
 
 - (void)testTwoBPInstances {
-    self.config.numSims = @2;
-    self.config.errorRetriesCount = @1;
-    self.config.failureTolerance = @0;
+    BPConfiguration *config = [self generateConfig];
+    config.numSims = @2;
+    config.errorRetriesCount = @1;
+    config.failureTolerance = @0;
     NSError *err;
-    BPApp *app = [BPApp appWithConfig:self.config
+    BPApp *app = [BPApp appWithConfig:config
                             withError:&err];
 
     NSString *bpPath = [BPTestHelper bpExecutablePath];
-    BPRunner *runner = [BPRunner BPRunnerWithConfig:self.config withBpPath:bpPath];
+    BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
     XCTAssert(runner != nil);
     int rc = [runner runWithBPXCTestFiles:app.testBundles];
     XCTAssert(rc == 0);
     XCTAssert([runner.nsTaskList count] == 0);
 }
 
 - (void)testClonedSimulators {
-    self.config.numSims = @2;
-    self.config.errorRetriesCount = @1;
-    self.config.failureTolerance = @0;
-    self.config.cloneSimulator = TRUE;
+    BPConfiguration *config = [self generateConfig];
+    config.numSims = @2;
+    config.errorRetriesCount = @1;
+    config.failureTolerance = @0;
+    config.cloneSimulator = TRUE;
     // need to validate the configuration to fill in simDevice and simRuntime
-    [self.config validateConfigWithError:nil];
+    [config validateConfigWithError:nil];
     NSError *err;
-    BPApp *app = [BPApp appWithConfig:self.config
+    BPApp *app = [BPApp appWithConfig:config
                             withError:&err];
 
     NSString *bpPath = [BPTestHelper bpExecutablePath];
-    BPRunner *runner = [BPRunner BPRunnerWithConfig:self.config withBpPath:bpPath];
+    BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
     XCTAssert(runner != nil);
     int rc = [runner runWithBPXCTestFiles:app.testBundles];
     XCTAssert(rc == 0);
     XCTAssert([runner.nsTaskList count] == 0);
 }
 
 - (void)testTwoBPInstancesWithUITests {
-    self.config.numSims = @2;
-    self.config.errorRetriesCount = @1;
-    self.config.failureTolerance = @0;
+    BPConfiguration *config = [self generateConfig];
+    config.numSims = @2;
+    config.errorRetriesCount = @1;
+    config.failureTolerance = @0;
     // This looks backwards but we want the main app to be the runner
     // and the sampleApp is launched from the callback.
-    self.config.testBundlePath = [BPTestHelper sampleAppUITestBundlePath];
-    self.config.testRunnerAppPath = [BPTestHelper sampleAppPath];
-    self.config.appBundlePath = [BPTestHelper sampleAppUITestRunnerPath];
-
+    config.testBundlePath = [BPTestHelper sampleAppUITestBundlePath];
+    config.testRunnerAppPath = [BPTestHelper sampleAppPath];
+    config.appBundlePath = [BPTestHelper sampleAppUITestRunnerPath];
 
     NSError *err;
-    BPApp *app = [BPApp appWithConfig:self.config
+    BPApp *app = [BPApp appWithConfig:config
                             withError:&err];
 
     NSString *bpPath = [BPTestHelper bpExecutablePath];
-    BPRunner *runner = [BPRunner BPRunnerWithConfig:self.config withBpPath:bpPath];
+    BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
     XCTAssert(runner != nil);
     int rc = [runner runWithBPXCTestFiles:app.testBundles];
     XCTAssert(rc == 0);
     XCTAssert([runner.nsTaskList count] == 0);
 }
 
 - (void)testTwoBPInstancesWithXCTestRunFile {
-    self.config.numSims = @2;
-    self.config.testBundlePath = nil;
-    self.config.testRunnerAppPath = nil;
+    BPConfiguration *config = [self generateConfig];
+    config.numSims = @2;
+    config.testBundlePath = nil;
+    config.testRunnerAppPath = nil;
     NSString *runtime = [[NSString stringWithUTF8String:BP_DEFAULT_RUNTIME] stringByReplacingOccurrencesOfString:@"iOS " withString:@""];
     NSString *xcTestRunFile = [NSString stringWithFormat:@"Build/Products/BPSampleApp_iphonesimulator%@-x86_64.xctestrun", runtime];
-    self.config.xcTestRunPath = [[[BPTestHelper derivedDataPath] stringByDeletingLastPathComponent] stringByAppendingPathComponent:xcTestRunFile];
+    config.xcTestRunPath = [[[BPTestHelper derivedDataPath] stringByDeletingLastPathComponent] stringByAppendingPathComponent:xcTestRunFile];
     NSError *err;
-    [self.config validateConfigWithError:&err];
-    BPApp *app = [BPApp appWithConfig:self.config withError:&err];
+    [config validateConfigWithError:&err];
+    BPApp *app = [BPApp appWithConfig:config withError:&err];
     NSString *bpPath = [BPTestHelper bpExecutablePath];
-    BPRunner *runner = [BPRunner BPRunnerWithConfig:self.config withBpPath:bpPath];
+    BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
     XCTAssert(runner != nil);
     int rc = [runner runWithBPXCTestFiles:app.testBundles];
     for (BPXCTestFile *testBundle in app.testBundles) {
@@ -146,17 +152,67 @@ - (void)testTwoBPInstancesWithXCTestRunFile {
 }
 
 - (void)testTwoBPInstancesTestCaseFail {
-    self.config.numSims = @2;
-    self.config.testBundlePath = [BPTestHelper sampleAppNegativeTestsBundlePath];
+    BPConfiguration *config = [self generateConfig];
+    config.numSims = @2;
+    config.testBundlePath = [BPTestHelper sampleAppNegativeTestsBundlePath];
     NSError *err;
-    BPApp *app = [BPApp appWithConfig:self.config
+    BPApp *app = [BPApp appWithConfig:config
                             withError:&err];
     NSString *bpPath = [BPTestHelper bpExecutablePath];
-    BPRunner *runner = [BPRunner BPRunnerWithConfig:self.config withBpPath:bpPath];
+    BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
     XCTAssert(runner != nil);
     int rc = [runner runWithBPXCTestFiles:app.testBundles];
     XCTAssert(rc != 0);
     XCTAssert([runner.nsTaskList count] == 0);
 }
 
+- (void)testTwoBPInstancesWithTestPlanJson {
+    [self writeTestPlan];
+    BPConfiguration *config = [self generateConfig];
+    config.numSims = @2;
+    config.testBundlePath = nil;
+    config.testRunnerAppPath = nil;
+    config.appBundlePath = nil;
+    config.testPlanPath = [BPTestHelper testPlanPath];
+
+    NSError *err;
+    [config validateConfigWithError:&err];
+    BPApp *app = [BPApp appWithConfig:config withError:&err];
+    NSString *bpPath = [BPTestHelper bpExecutablePath];
+    BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
+    XCTAssert(runner != nil);
+    int rc = [runner runWithBPXCTestFiles:app.testBundles];
+    XCTAssert(rc != 0); // this runs tests that fail
+    XCTAssertEqual(app.testBundles.count, 2);
+    XCTAssertTrue([app.testBundles[0].name isEqualToString:@"BPAppNegativeTests"]);
+    XCTAssertEqual(app.testBundles[0].numTests, 4);
+    XCTAssertEqual(app.testBundles[0].skipTestIdentifiers.count, 0);
+    XCTAssertTrue([app.testBundles[1].name isEqualToString:@"BPSampleAppTests"]);
+    XCTAssertEqual(app.testBundles[1].numTests, 207);
+    XCTAssertEqual(app.testBundles[1].skipTestIdentifiers.count, 0);
+}
+
+- (void)writeTestPlan {
+    NSDictionary *testPlan = @{
+        @"tests": @{
+            @"BPSampleAppTests": @{
+                @"test_host": [BPTestHelper sampleAppPath],
+                @"test_host_bundle_identifier": @"identifier",
+                @"test_bundle_path": [BPTestHelper sampleAppBalancingTestsBundlePath],
+                @"environment": @{},
+                @"arguments": @{}
+            },
+            @"BPAppNegativeTests": @{
+                @"test_host": [BPTestHelper sampleAppPath],
+                @"test_host_bundle_identifier": @"identifier",
+                @"test_bundle_path": [BPTestHelper sampleAppNegativeTestsBundlePath],
+                @"environment": @{},
+                @"arguments": @{}
+            }
+        }
+    };
+    NSData *jsonData = [NSJSONSerialization dataWithJSONObject:testPlan options:0 error:nil];
+    [jsonData writeToFile:[BPTestHelper testPlanPath] atomically:YES];
+}
+
 @end

bp/src/BPConfiguration.h

@@ -13,6 +13,28 @@
 @class SimDeviceType;
 @class SimRuntime;
 
+/**
+ TestPlans populated by an outside build system
+*/
+
+@interface BPTestPlan : NSObject<NSCopying>
+
+@property (nonatomic, strong) NSString *testHost;
+@property (nonatomic, strong) NSDictionary<NSString *, NSString *> *environment;
+@property (nonatomic, strong) NSDictionary<NSString *, NSString *> *arguments;
+@property (nonatomic, strong) NSString *testBundlePath;
+@property (nonatomic, strong) NSString *testHostBundleIdentifier;
+@property (nonatomic, strong) NSString *uiTargetAppPath;
+
+/**
+ isValid checks to make sure the testHost is valid
+
+ @param errPtr the error to fill if the test plan is not valid
+ */
+- (BOOL)isValid:(NSError **)errPtr;
+
+@end
+
 /**
  BPConfiguration stores necessary information for Simulator Runner to run
  */
@@ -45,6 +67,7 @@ typedef NS_ENUM(NSInteger, BPProgram) {
 @property (nonatomic, strong) NSString *runtime;
 @property (nonatomic, strong) NSString *configFile;
 @property (nonatomic, strong) NSString *xcTestRunPath;
+@property (nonatomic, strong) NSString *testPlanPath;
 @property (nonatomic, strong) NSDictionary *xcTestRunDict; // parsed copy of the path above.
 @property (nonatomic, strong) NSMutableArray *bpCmdLineArgs; // command line arguments passed to bluepill
 @property (nonatomic, strong) NSNumber *repeatTestsCount;
@@ -82,6 +105,7 @@ typedef NS_ENUM(NSInteger, BPProgram) {
 
 @property (nonatomic, strong) NSArray<NSString *> *commandLineArguments; // command line arguments for the app
 @property (nonatomic, strong) NSDictionary<NSString *, NSString *> *environmentVariables;
+@property (nonatomic, strong) NSDictionary<NSString *, BPTestPlan *> *tests;
 
 // Media Assets
 @property (nonatomic, strong) NSArray<NSString *> *videoPaths; // The videos to be pushed into each simulator.
@@ -199,3 +223,4 @@ typedef NS_ENUM(NSInteger, BPProgram) {
 - (id)mutableCopyWithZone: (NSZone *) zone;
 
 @end
+