Merge pull request #255 from inkling/jeff/234_apply_focus_first

Apply focus before filtering for platform/environment support.

Sources/Classes/Internal/SLTestController+Internal.h

@@ -37,13 +37,14 @@
  Given a set of tests, returns an array ordered by the specified seed 
  and filtered to those that should be run.
  
- The set of tests is sorted and randomized using the specified seed. 
+ The set of tests is sorted in ascending order of [group](+[SLTest runGroup]),
+ and then within each group, in an order randomized using the specified seed.
  The resulting array is then filtered:
  
- 1. to those tests that [are concrete](+[SLTest isAbstract]),
- 2. that [support the current platform](+[SLTest supportsCurrentPlatform]),
- 3. that [support the current environment](+[SLTest supportsCurrentEnvironment]),
- 4. and that [are focused](+[SLTest isFocused]) (if any remaining are focused).
+ 1. to those that [are focused](+[SLTest isFocused]) (if any),
+ 2. that [are concrete](+[SLTest isAbstract]),
+ 3. that [support the current platform](+[SLTest supportsCurrentPlatform]),
+ 4. and that [support the current environment](+[SLTest supportsCurrentEnvironment]).
  
  By sorting prior to filtering, the relative order of tests is maintained 
  regardless of focus.

Sources/Classes/SLTest.h

@@ -304,9 +304,7 @@
 + (BOOL)testCaseWithSelectorSupportsCurrentEnvironment:(SEL)testCaseSelector;
 
 /**
- Returns YES if the test has at least one test case which is focused
- and which supports the current [platform](+testCaseWithSelectorSupportsCurrentPlatform:)
- and [environment](+testCaseWithSelectorSupportsCurrentEnvironment:).
+ Returns YES if the test has at least one test case which is focused.
 
  When a test is run, if any of its test cases are focused, only those test cases will run.
  This may be useful when writing or debugging tests.

Sources/Classes/SLTest.m

@@ -134,16 +134,8 @@ + (BOOL)isAbstract {
     return ![[self testCases] count];
 }
 
-+ (BOOL)isFocused {    
-    for (NSString *testCaseName in [self focusedTestCases]) {
-        // pass the unfocused selector, as focus is temporary and shouldn't require modifying the test infrastructure
-        SEL unfocusedTestCaseSelector = NSSelectorFromString([self unfocusedTestCaseName:testCaseName]);
-        if ([self testCaseWithSelectorSupportsCurrentPlatform:unfocusedTestCaseSelector] &&
-            [self testCaseWithSelectorSupportsCurrentEnvironment:unfocusedTestCaseSelector]) {
-            return YES;
-        }
-    }
-    return NO;
++ (BOOL)isFocused {
+    return [[self focusedTestCases] count] > 0;
 }
 
 + (BOOL)supportsCurrentPlatform {

Sources/Classes/SLTestController.h

@@ -81,8 +81,9 @@
  If any tests fail, the test controller will log the seed that was used,
  so that the run order may be reproduced by invoking this method with that seed.
 
- Tests must support the current [platform](+[SLTest supportsCurrentPlatform])
- and [environment](+[SLTest supportsCurrentEnvironment]) in order to be run.
+ In order to be run, tests must [define test cases](+[SLTest isAbstract]),
+ support the current [platform](+[SLTest supportsCurrentPlatform]),
+ and support the current [environment](+[SLTest supportsCurrentEnvironment]).
  If any tests [are focused](+[SLTest isFocused]), only those tests will be run.
  
  When using a given seed, tests execute in the same relative order regardless of focus.

Sources/Classes/SLTestController.m

@@ -190,24 +190,23 @@ + (NSArray *)testsToRun:(NSSet *)tests usingSeed:(inout unsigned int *)seed with
         [testsToRun addObjectsFromArray:group];
     }
 
-    // now filter the tests to run:
-    [testsToRun filterUsingPredicate:[NSCompoundPredicate andPredicateWithSubpredicates:@[
-        // only run tests that are concrete...
-        [NSPredicate predicateWithFormat:@"isAbstract == NO"],
-        // ...that support the current platform...
-        [NSPredicate predicateWithFormat:@"supportsCurrentPlatform == YES"],
-        // ...that support the current environment...
-        [NSPredicate predicateWithFormat:@"supportsCurrentEnvironment == YES"]
-    ]]];
-
-    // ...and that are focused (if any remaining are focused)
+    // now filter the tests to run to those focused, if any...
     NSMutableArray *focusedTests = [testsToRun mutableCopy];
     [focusedTests filterUsingPredicate:[NSPredicate predicateWithFormat:@"isFocused == YES"]];
     BOOL runningWithFocus = ([focusedTests count] > 0);
     if (runningWithFocus) {
         testsToRun = focusedTests;
     }
     if (withFocus) *withFocus = runningWithFocus;
+
+    [testsToRun filterUsingPredicate:[NSCompoundPredicate andPredicateWithSubpredicates:@[
+        // ...then, only run tests that are concrete...
+        [NSPredicate predicateWithFormat:@"isAbstract == NO"],
+        // ...that support the current platform...
+        [NSPredicate predicateWithFormat:@"supportsCurrentPlatform == YES"],
+        // ...and that support the current environment.
+        [NSPredicate predicateWithFormat:@"supportsCurrentEnvironment == YES"]
+    ]]];
 
     return [testsToRun copy];
 }
@@ -349,14 +348,24 @@ - (void)runTests:(NSSet *)tests withCompletionBlock:(void (^)())completionBlock
 }
 
 - (void)runTests:(NSSet *)tests usingSeed:(unsigned int)seed withCompletionBlock:(void (^)())completionBlock {
+    // have to check this outside of the block below, wherein it will be used
+    static const char *const kMethodDescription = __PRETTY_FUNCTION__;
+
     dispatch_async(_runQueue, ^{
         _completionBlock = completionBlock;
 
         _runningWithPredeterminedSeed = (seed != SLTestControllerRandomSeed);
         _runSeed = seed;
         _testsToRun = [[self class] testsToRun:tests usingSeed:&_runSeed withFocus:&_runningWithFocus];
         if (![_testsToRun count]) {
-            SLLog(@"%@%@%@", @"There are no tests to run", (_runningWithFocus) ? @": no tests are focused" : @"", @".");
+            NSMutableString *noTestsToRunWarning = [@"There are no tests to run: " mutableCopy];
+            if ([tests count]) {
+                [noTestsToRunWarning appendFormat:@"none of the tests %@ meet the criteria to be run. See `%@`'s documentation.",
+                                                 (_runningWithFocus) ? @"focused" : @"passed", @(kMethodDescription)];
+            } else {
+                [noTestsToRunWarning appendString:@"no tests were passed."];
+            }
+            [[SLLogger sharedLogger] logWarning:noTestsToRunWarning];
             [self _finishTesting];
             return;
         }

Unit Tests/SLTestControllerTests.m

@@ -345,7 +345,12 @@ - (void)testTheUserIsNotifiedWhenRunningTaggedTests {
     [[_loggerMock expect] logMessage:[NSString stringWithFormat:@"Running test cases described by tags: %@.", tagDescriptionString]];
     [[_loggerMock expect] logTestingStart];
 
-    SLRunTestsAndWaitUntilFinished([SLTest allTests], nil);
+    // Ignore focused tests because they don't support the current environment
+    // (not being tagged with the above tags) and so will be filtered out,
+    // leaving no tests to be run--causing the run to immediately abort without
+    // logging the expected message.
+    NSSet *nonFocusedTests = [[SLTest allTests] filteredSetUsingPredicate:[NSPredicate predicateWithFormat:@"isFocused == NO"]];
+    SLRunTestsAndWaitUntilFinished(nonFocusedTests, nil);
     STAssertNoThrow([_loggerMock verify], @"Test was not run/messages were not logged as expected.");
 }
 
@@ -439,16 +444,16 @@ - (void)testAFocusedTestMustSupportTheCurrentPlatformInOrderToBeRun {
         nil
     ];
 
-     // While TestThatIsFocusedButDoesntSupportCurrentPlatform is focused,
+     // While `TestThatIsFocusedButDoesntSupportCurrentPlatform` is focused,
     // it doesn't support the current platform, thus isn't going to run.
-    // If it's not going to run, its focus is irrelevant, and so the other test should run after all.
+    // However, its being focused should exclude the other test from running too.
     id testThatIsFocusedButDoesntSupportCurrentPlatformClassMock = [OCMockObject partialMockForClass:testThatIsFocusedButDoesntSupportCurrentPlatformClass];
     [[testThatIsFocusedButDoesntSupportCurrentPlatformClassMock reject] runAndReportNumExecuted:[OCMArg anyPointer]
                                                                                          failed:[OCMArg anyPointer]
                                                                              failedUnexpectedly:[OCMArg anyPointer]];
 
     id testThatIsNotFocusedClassMock = [OCMockObject partialMockForClass:testThatIsNotFocusedClass];
-    [[testThatIsNotFocusedClassMock expect] runAndReportNumExecuted:[OCMArg anyPointer]
+    [[testThatIsNotFocusedClassMock reject] runAndReportNumExecuted:[OCMArg anyPointer]
                                                              failed:[OCMArg anyPointer]
                                                  failedUnexpectedly:[OCMArg anyPointer]];
 
@@ -475,14 +480,14 @@ - (void)testAFocusedTestMustSupportTheCurrentEnvironmentInOrderToBeRun {
 
     // While `TestThatIsFocusedButDoesntSupportCurrentEnvironment` is focused,
     // it doesn't support the current environment, thus isn't going to run.
-    // If it's not going to run, its focus is irrelevant, and so the other test should run after all.
+    // However, its being focused should exclude the other test from running too.
     id testThatIsFocusedButDoesntSupportCurrentEnvironmentClassMock = [OCMockObject partialMockForClass:testThatIsFocusedButDoesntSupportCurrentEnvironmentClass];
     [[testThatIsFocusedButDoesntSupportCurrentEnvironmentClassMock reject] runAndReportNumExecuted:[OCMArg anyPointer]
                                                                                             failed:[OCMArg anyPointer]
                                                                                 failedUnexpectedly:[OCMArg anyPointer]];
 
     id testThatIsNotFocusedClassMock = [OCMockObject partialMockForClass:testThatIsNotFocusedClass];
-    [[testThatIsNotFocusedClassMock expect] runAndReportNumExecuted:[OCMArg anyPointer]
+    [[testThatIsNotFocusedClassMock reject] runAndReportNumExecuted:[OCMArg anyPointer]
                                                              failed:[OCMArg anyPointer]
                                                  failedUnexpectedly:[OCMArg anyPointer]];
 
@@ -491,6 +496,7 @@ - (void)testAFocusedTestMustSupportTheCurrentEnvironmentInOrderToBeRun {
     STAssertNoThrow([testThatIsNotFocusedClassMock verify], @"Other test was not run as expected.");
 }
 
+// Focus always excludes other tests--but only if the user tries to run the focused test.
 - (void)testFocusedTestsAreNotAutomaticallyAddedToTheSetOfTestsToRun {
     Class testWithSomeTestCasesClass = [TestWithSomeTestCases class];
     STAssertFalse([testWithSomeTestCasesClass isFocused],
@@ -624,4 +630,28 @@ - (void)testMustUseSharedController {
 #pragma clang diagnostic pop
 }
 
+- (void)testTheUserIsWarnedIfTheyDidntPassTests {
+    [[_loggerMock expect] logWarning:@"There are no tests to run: no tests were passed."];
+
+    SLRunTestsAndWaitUntilFinished([NSSet set], nil);
+
+    STAssertNoThrow([_loggerMock verify], @"Expected warning was not logged.");
+}
+
+- (void)testTheUserIsWarnedIfThereAreNoRegularTestsToRun {
+    [[_loggerMock expect] logWarning:@"There are no tests to run: none of the tests passed meet the criteria to be run. See `-[SLTestController runTests:usingSeed:withCompletionBlock:]`'s documentation."];
+
+    SLRunTestsAndWaitUntilFinished([NSSet setWithObject:[TestNotSupportingCurrentPlatform class]], nil);
+
+    STAssertNoThrow([_loggerMock verify], @"Expected warning was not logged.");
+}
+
+- (void)testTheUserIsWarnedIfThereAreNoFocusedTestsToRun {
+    [[_loggerMock expect] logWarning:@"There are no tests to run: none of the tests focused meet the criteria to be run. See `-[SLTestController runTests:usingSeed:withCompletionBlock:]`'s documentation."];
+
+    SLRunTestsAndWaitUntilFinished([NSSet setWithObjects:[Focus_TestThatIsFocusedButDoesntSupportCurrentPlatform class], [TestWithSomeTestCases class], nil], nil);
+
+    STAssertNoThrow([_loggerMock verify], @"Expected warning was not logged.");
+}
+
 @end

Unit Tests/SLTestTests.m

@@ -666,10 +666,10 @@ - (void)testFocusedTestCasesMustSupportTheCurrentPlatformInOrderToRun {
     [[[deviceMock stub] andReturnValue:OCMOCK_VALUE(currentUserInterfaceIdiom)] userInterfaceIdiom];
 
     // While `testBar_iPad` is focused, it doesn't support the current platform, thus isn't going to run.
-    // If it's not going to run, its focus is irrelevant, and so the other test case should run after all.
+    // However, its being focused should exclude the other test case from running too.
     id testWithAFocusedPlatformSpecificTestCaseClassMock = [OCMockObject partialMockForClass:testWithAFocusedPlatformSpecificTestCaseClass];
     [[testWithAFocusedPlatformSpecificTestCaseClassMock reject] focus_testBar_iPad];
-    [[testWithAFocusedPlatformSpecificTestCaseClassMock expect] testFoo];
+    [[testWithAFocusedPlatformSpecificTestCaseClassMock reject] testFoo];
 
     SLRunTestsAndWaitUntilFinished([NSSet setWithObject:testWithAFocusedPlatformSpecificTestCaseClass], nil);
     STAssertNoThrow([testWithAFocusedPlatformSpecificTestCaseClassMock verify], @"Test cases did not execute as expected.");
@@ -684,10 +684,10 @@ - (void)testFocusedTestCasesMustSupportTheCurrentEnvironmentInOrderToRun {
     [[[deviceMock stub] andReturnValue:OCMOCK_VALUE(currentUserInterfaceIdiom)] userInterfaceIdiom];
 
     // While `testBar` is focused, it doesn't support the current environment, thus isn't going to run.
-    // If it's not going to run, its focus is irrelevant, and so the other test case should run after all.
+    // However, its being focused should exclude the other test from running too.
     id testWithAFocusedEnvironmentSpecificTestCaseClassMock = [OCMockObject partialMockForClass:testWithAFocusedEnvironmentSpecificTestCaseClass];
     [[testWithAFocusedEnvironmentSpecificTestCaseClassMock reject] focus_testBar];
-    [[testWithAFocusedEnvironmentSpecificTestCaseClassMock expect] testFoo];
+    [[testWithAFocusedEnvironmentSpecificTestCaseClassMock reject] testFoo];
 
     SLRunTestsAndWaitUntilFinished([NSSet setWithObject:testWithAFocusedEnvironmentSpecificTestCaseClass], nil);
     STAssertNoThrow([testWithAFocusedEnvironmentSpecificTestCaseClassMock verify], @"Test cases did not execute as expected.");