rgf script returns 1 if any Examples fail.

TODO

@@ -1,6 +1,4 @@
 # from future, import...
-# rgf script returns non-zero exit status if any specs failed
-# --> At this point we can be used by a CI bot :-)
 # ExampleGroups can be properly nested
 # before-functions are properly dealt with in nested contexts
 # Actually work out the language issues

fixture_specs/failed/a_spec.py

@@ -0,0 +1,6 @@
+from rgf.dsl import describe, it
+
+with describe("This isn't going to end well"):
+    @it('goes boom')
+    def s(w):
+        assert False

rgf/core/examples.py

@@ -31,9 +31,12 @@ def before(self, before_function):
         self.before_function = before_function
 
     def run(self, reporter):
+        all_examples_succeeded = True
         for example in self.examples:
             result = example.run(self)
+            if result.is_not_success(): all_examples_succeeded = False
             reporter.example_ran(example, result)
+        return all_examples_succeeded
 
     def run_before_each(self, example):
         if self.before_function:
@@ -76,8 +79,11 @@ def pop_current_example_group(self):
         self.current_example_group_stack.pop()
 
     def run(self, reporter):
-        [example_group.run(reporter) for example_group in self.example_groups]
+        results = [example_group.run(reporter) for example_group in self.example_groups]
         reporter.run_finished()
+        def all_succeeded(start_state, current_state):
+            return start_state and current_state
+        return reduce(all_succeeded, results, True)
 
 class ExampleResult(object):
     SUCCESS = 1
@@ -100,3 +106,15 @@ def __init__(self, kind, exception = None, traceback = None):
         self.kind = kind
         self.exception = exception
         self.traceback = traceback
+
+    def is_success(self):
+        return self.kind == self.SUCCESS
+
+    def is_failure(self):
+        return self.kind == self.FAILURE
+
+    def is_error(self):
+        return self.kind == self.ERROR
+
+    def is_not_success(self):
+        return self.kind != self.SUCCESS

rgf/core/runner.py

@@ -129,12 +129,16 @@ def __init__(self, reporter):
     def run(self, suite, path):
         collector = Collector(path)
         collector.collect_to(suite)
-        suite.run(self.reporter)
+        result = suite.run(self.reporter)
+        if result:
+            return 0
+        else:
+            return 1
 
 def main():
     formatter = ProgressFormatter(sys.stdout)
     reporter = Reporter(formatter)
     runner = Runner(reporter)
     suite = ExampleSuite.get_suite()
 
-    runner.run(suite, sys.argv[1])
+    return runner.run(suite, sys.argv[1])

spec/rgf/core/examples/example_group_spec.py

@@ -18,34 +18,46 @@ def run_finished(self):
 def first_test_function(world):
     world.has_been_run = True
 
+def failing_test_function(world):
+    assert False
+
 def before_func(world):
     world.before_was_run = True
 
 with describe('ExampleGroup'):
+    @before
+    def b(w):
+        w.eg = ExampleGroup(MockExampleSuite(), "A group of Examples")
+        w.eg.add_example(Example('All good', first_test_function))
+        w.eg.add_example(Example('Still good', first_test_function))
+
     @it('can be created and described')
-    def spec(world):
-        eg = ExampleGroup(MockExampleSuite(), "A group of Examples")
-        assert eg.description == "A group of Examples"
+    def spec(w):
+        assert w.eg.description == "A group of Examples"
 
-        # Examples can be grouped and run together
-        eg = ExampleGroup(MockExampleSuite(), "")
-        eg.add_example(Example('All good', first_test_function))
-        eg.add_example(Example('Still good', first_test_function))
-        eg.run(MockReporter())
+    @it('can group Examples and run them together')
+    def spec(w):
+        w.eg.run(MockReporter())
 
-        assert eg.examples[0].has_been_run
-        assert eg.examples[1].has_been_run
+        assert w.eg.examples[0].has_been_run
+        assert w.eg.examples[1].has_been_run
 
     @it('can have setup code to be run before examples added')
-    def spec(world):
-        eg = ExampleGroup(MockExampleSuite(), '')
-        eg.add_example(Example('All good', first_test_function))
-        eg.add_example(Example('Still good', first_test_function))
-        eg.before(before_func)
-        eg.run(MockReporter())
-
-        assert eg.examples[0].before_was_run
-        assert eg.examples[1].before_was_run
+    def spec(w):
+        w.eg.before(before_func)
+        w.eg.run(MockReporter())
+
+        assert w.eg.examples[0].before_was_run
+        assert w.eg.examples[1].before_was_run
+
+    @it('returns True if all Examples success')
+    def spec(w):
+        assert w.eg.run(MockReporter()) is True
+
+    @it('returns False if any Examples fail')
+    def spec(w):
+        w.eg.add_example(Example('Not good', failing_test_function))
+        assert w.eg.run(MockReporter()) is False
 
 with describe('ExampleGroup context manager API'):
     @it('sets itself as the current example group in the suite on __enter__()')

spec/rgf/core/examples/example_result_spec.py

@@ -0,0 +1,61 @@
+from rgf.dsl import describe, it, before
+
+from rgf.core.examples import ExampleResult
+
+class MockException(object):
+    pass
+
+class MockTraceback(object):
+    pass
+
+with describe('ExampleResult'):
+    @it('reports that it was a success')
+    def s(w):
+        result = ExampleResult.as_success()
+        assert result.is_success()
+
+with describe('ExampleResult failed'):
+    @before
+    def b(w):
+        w.mock_exception = MockException()
+        w.mock_traceback = MockTraceback()
+        w.result = ExampleResult.as_failure(w.mock_exception, w.mock_traceback)
+
+    @it('reports that it was a failure')
+    def s(w):
+        assert w.result.is_failure()
+
+    @it('reports that it was not a success')
+    def s(w):
+        assert w.result.is_not_success()
+
+    @it('returns the exception')
+    def s(w):
+        assert w.result.exception is w.mock_exception
+
+    @it('returns the traceback')
+    def s(w):
+        assert w.result.traceback is w.mock_traceback
+
+with describe('ExampleResult error'):
+    @before
+    def b(w):
+        w.mock_exception = MockException()
+        w.mock_traceback = MockTraceback()
+        w.result = ExampleResult.as_error(w.mock_exception, w.mock_traceback)
+
+    @it('reports that it was an error')
+    def s(w):
+        assert w.result.is_error()
+
+    @it('reports that it was not a success')
+    def s(w):
+        assert w.result.is_not_success()
+
+    @it('returns the exception')
+    def s(w):
+        assert w.result.exception is w.mock_exception
+
+    @it('returns the traceback')
+    def s(w):
+        assert w.result.traceback is w.mock_traceback

spec/rgf/core/examples/example_suite_spec.py

@@ -19,58 +19,64 @@ def before_func(world):
     world.before_was_run = True
 
 with describe('ExampleSuite'):
+    @before
+    def b(w):
+        w.suite = ExampleSuite()
+
     @it('can collect many ExampleGroups')
-    def spec(world):
-        suite = ExampleSuite()
-        example_group = suite.add_example_group('ExampleGroup description')
+    def spec(w):
+        example_group = w.suite.add_example_group('ExampleGroup description')
         assert type(example_group) is ExampleGroup
-        assert example_group in suite.example_groups
+        assert example_group in w.suite.example_groups
 
     @it('returns itself as the current ExampleGroup if there is none')
-    def spec(world):
-        suite = ExampleSuite()
-        assert suite.get_current_example_group() is suite
+    def spec(w):
+        assert w.suite.get_current_example_group() is w.suite
 
     @it('allows the current ExampleGroup to be set')
-    def spec(world):
-        suite = ExampleSuite()
-        example_group = suite.add_example_group('ExampleGroup description')
-        suite.set_current_example_group(example_group)
-        assert suite.get_current_example_group() is example_group
+    def spec(w):
+        example_group = w.suite.add_example_group('ExampleGroup description')
+        w.suite.set_current_example_group(example_group)
+        assert w.suite.get_current_example_group() is example_group
 
     @it('allows the current ExampleGroup to be popped off')
-    def spec(world):
-        suite = ExampleSuite()
-        example_group = suite.add_example_group('ExampleGroup description')
-        suite.set_current_example_group(example_group)
-        suite.pop_current_example_group()
-        assert suite.get_current_example_group() is suite
+    def spec(w):
+        example_group = w.suite.add_example_group('ExampleGroup description')
+        w.suite.set_current_example_group(example_group)
+        w.suite.pop_current_example_group()
+        assert w.suite.get_current_example_group() is w.suite
 
     @it('can create and return a single instance of itself')
-    def spec(world):
+    def spec(w):
         assert type(ExampleSuite.get_suite()) is ExampleSuite
         assert ExampleSuite.get_suite() is ExampleSuite.get_suite()
 
     @it('can set the ExampleSuite instance to be returned by get_suite()')
     def f(w):
-        temp_suite = ExampleSuite()
-        ExampleSuite.set_suite(temp_suite)
-        assert ExampleSuite.get_suite() is temp_suite
+        ExampleSuite.set_suite(w.suite)
+        assert ExampleSuite.get_suite() is w.suite
 
     @it('can run run all its ExampleGroups')
-    def spec(world):
-        example_suite = ExampleSuite()
-        example_group = example_suite.add_example_group('eg for explicit passing to it decorator')
+    def spec(w):
+        example_group = w.suite.add_example_group('eg for explicit passing to it decorator')
         @it('has explicit ExampleGroup', example_group = example_group)
         def f(world):
             world.has_been_run = True
 
-        example_suite.run(MockReporter())
+        w.suite.run(MockReporter())
         assert example_group.examples[0].has_been_run
 
     @it('tells the Reporter when the run has finished')
-    def spec(world):
-        example_suite = ExampleSuite()
+    def spec(w):
         reporter = MockReporter()
-        example_suite.run(reporter)
+        w.suite.run(reporter)
         assert reporter.run_finished_was_called
+
+    @it('returns True from run() if there were no failures')
+    def f(w):
+        example_group = w.suite.add_example_group('happy example group')
+        @it('succeeds', example_group = example_group)
+        def f(w):
+            pass
+        assert w.suite.run(MockReporter()) is True
+

spec/rgf/core/runner/runner_spec.py

@@ -39,3 +39,9 @@ def f(w):
     def f(w):
         return_val, output = run_spec_script('fixture_specs/success')
         assert return_val == 0
+
+    @it('returns exit status 1 if there are failures')
+    def f(w):
+        return_val, output = run_spec_script('fixture_specs/failed')
+        assert return_val == 1
+