Tests now fail when an error occurs.

.scrutinizer.yml

@@ -1,6 +1,7 @@
 filter:
   paths: [src/*]
   excluded_paths: [vendor/*, specs/*, src/Test/*]
+  dependency_paths: [fixtures/*]
 checks:
   php:
     parameter_doc_comments: false
@@ -23,7 +24,7 @@ tools:
       method_contract_checks:
         verify_documented_constraints: false
   sensiolabs_security_checker: true
-  php_code_sniffer: 
+  php_code_sniffer:
     enabled: true
     config:
       standard: "PSR2"

composer.json

@@ -30,5 +30,10 @@
       "Peridot\\": "src/"
     }
   },
+  "autoload-dev": {
+    "files": [
+      "fixtures/error-polyfill.php"
+    ]
+  },
   "bin": ["bin/peridot"]
 }

fixtures/error-polyfill.php

@@ -0,0 +1,8 @@
+<?php
+
+if (!interface_exists('Throwable')) {
+    interface Throwable {}
+}
+if (!class_exists('Error')) {
+    class Error extends Exception implements Throwable {}
+}

specs/runner.spec.php

@@ -160,51 +160,5 @@
                 assert($result->getTestCount() === 3, "spec count should be 3");
             });
         });
-
-        $behavesLikeErrorEmitter = function() {
-            $this->suite->addTest(new Test("my spec", function() {
-                trigger_error("This is a user notice", E_USER_NOTICE);
-            }));
-
-            $error = [];
-            $this->eventEmitter->on('error', function($errno, $errstr, $errfile, $errline) use (&$error) {
-                $error = array(
-                    'errno' => $errno,
-                    'errstr' => $errstr,
-                    'errfile' => $errfile,
-                    'errline' => $errline
-                );
-            });
-
-            $this->runner->run(new TestResult(new EventEmitter()));
-            assert($error['errno'] == E_USER_NOTICE, "error event should have passed error constant");
-            assert($error['errstr'] == "This is a user notice");
-        };
-
-        it("should emit an error event with error information", $behavesLikeErrorEmitter);
-
-        it("should invoke the previous error handler on error", function() use ($behavesLikeErrorEmitter) {
-            $handled = [];
-            $handler = function() use (&$handled) {
-                $handled = func_get_args();
-            };
-            set_error_handler($handler);
-            call_user_func(Closure::bind($behavesLikeErrorEmitter, $this, $this));
-            assert(count($handled) === 4, "runner should have invoked previous handler");
-            assert($handled[0] === E_USER_NOTICE, "runner should have invoked previous handler");
-            assert($handled[1] === "This is a user notice", "runner should have invoked previous handler");
-            assert($handled[2] === __FILE__, "runner should have invoked previous handler");
-            assert(is_int($handled[3]), "runner should have invoked previous handler");
-        });
-
-        it("should restore a previous error handler upon completion", function() use ($behavesLikeErrorEmitter) {
-            $handler = function($errno, $errstr, $errfile, $errline) {
-                //such errors handled. wow!
-            };
-            set_error_handler($handler);
-            call_user_func(Closure::bind($behavesLikeErrorEmitter, $this, $this));
-            $old = set_error_handler(function($n,$s,$f,$l) {});
-            assert($handler === $old, "runner should have restored previous handler");
-        });
     });
 });

specs/spec-reporter.spec.php

@@ -17,7 +17,7 @@
     context('when test.failed is emitted', function() {
         it('should include an error number and the test description', function() {
             $test = new Test("test", function() {});
-            $this->emitter->emit('test.failed', [$test]);
+            $this->emitter->emit('test.failed', [$test, new Exception()]);
             $contents = $this->output->fetch();
             assert(strstr($contents, '1) test') !== false, "error count and test description should be present");
         });

specs/test.spec.php

@@ -51,13 +51,67 @@
 
         it("should add failed results to result", function () {
             $test = new ItWasRun("this should return a failed result", function () {
-                throw new \Exception('blaaargh');
+                throw new Exception('blaaargh');
             });
             $result = new TestResult(new EventEmitter());
             $test->run($result);
             assert("1 run, 1 failed" == $result->getSummary(), "result summary should have shown 1 failed");
         });
 
+        $removeErrorHandlers = function () {
+            do {
+                if ($handler = set_error_handler(function () {})) {
+                    restore_error_handler();
+                }
+
+                restore_error_handler();
+            } while ($handler);
+        };
+
+        it("should add failed results to result on error", function () use ($removeErrorHandlers) {
+            $test = new ItWasRun("this should return a failed result", function () {
+                trigger_error("This is a user notice", E_USER_NOTICE);
+            });
+            $result = new TestResult(new EventEmitter());
+            $removeErrorHandlers();
+            error_reporting(-1);
+            $test->run($result);
+            assert("1 run, 1 failed" == $result->getSummary(), "result summary should have shown 1 failed");
+        });
+
+        it("should ignore errors that are excluded by error reporting", function () use ($removeErrorHandlers) {
+            $test = new ItWasRun("this should return a failed result", function () {
+                trigger_error("This is a user notice", E_USER_NOTICE);
+            });
+            $result = new TestResult(new EventEmitter());
+            $removeErrorHandlers();
+            error_reporting(E_ERROR);
+            $test->run($result);
+            assert("1 run, 0 failed" == $result->getSummary(), "result summary should have shown 0 failed");
+        });
+
+        it("should ignore errors that are excluded by the error control operator", function () use ($removeErrorHandlers) {
+            $test = new ItWasRun("this should return a failed result", function () {
+                $foo = [];
+                @strlen($foo['bar']);
+            });
+            $result = new TestResult(new EventEmitter());
+            $removeErrorHandlers();
+            error_reporting(-1);
+            $test->run($result);
+            assert("1 run, 0 failed" == $result->getSummary(), "result summary should have shown 0 failed");
+        });
+
+        it("should add failed results to result on engine error", function () use ($removeErrorHandlers) {
+            $test = new ItWasRun("this should return a failed result", function () {
+                throw new Error('blaaargh');
+            });
+            $result = new TestResult(new EventEmitter());
+            $removeErrorHandlers();
+            $test->run($result);
+            assert("1 run, 1 failed" == $result->getSummary(), "result summary should have shown 1 failed");
+        });
+
         it("should add pending results to result", function () {
             $test = new Test('shouldnt run', function() {});
             $test->setPending(true);
@@ -200,6 +254,19 @@
                 assert($expected == $actual, "expected $expected, got $actual");
             });
 
+            it('should continue if tear down fails with an engine error', function () {
+                $test = new Test('spec', function() {});
+                $test->addTearDownFunction(function() {
+                    throw new Error();
+                });
+
+                $result = new TestResult(new EventEmitter());
+                $test->run($result);
+                $expected = "1 run, 1 failed";
+                $actual = $result->getSummary();
+                assert($expected == $actual, "expected $expected, got $actual");
+            });
+
             it('should not result in a pass and fail if tear down fails', function() {
                 $test = new Test("passing", function() {});
                 $test->addTearDownFunction(function() {

src/Core/Test.php

@@ -2,7 +2,10 @@
 
 namespace Peridot\Core;
 
+use Error;
+use ErrorException;
 use Exception;
+use Throwable;
 
 /**
  * The main test fixture for Peridot.
@@ -52,13 +55,17 @@ public function run(TestResult $result)
     protected function executeTest(TestResult $result)
     {
         $action = ['passTest', $this];
+        $handler = $this->handleErrors($result, $action);
         try {
             $this->runSetup();
             call_user_func_array($this->getDefinition(), $this->getDefinitionArguments());
+        } catch (Throwable $e) {
+            $this->failIfPassing($action, $e);
         } catch (Exception $e) {
-            $action = ['failTest', $this, $e];
+            $this->failIfPassing($action, $e);
         }
         $this->runTearDown($result, $action);
+        $this->restoreErrorHandler($handler);
     }
 
     /**
@@ -81,18 +88,79 @@ protected function runSetup()
      * @param TestResult $result
      * @param array $action
      */
-    protected function runTearDown(TestResult $result, $action)
+    protected function runTearDown(TestResult $result, array $action)
     {
         $this->forEachNodeBottomUp(function (TestInterface $test) use ($result, &$action) {
             $tearDowns = $test->getTearDownFunctions();
             foreach ($tearDowns as $tearDown) {
                 try {
                     $tearDown();
+                } catch (Throwable $e) {
+                    $this->failIfPassing($action, $e);
                 } catch (Exception $e) {
-                    $action = ['failTest', $this, $e];
+                    $this->failIfPassing($action, $e);
                 }
             }
         });
         call_user_func_array([$result, $action[0]], array_slice($action, 1));
     }
+
+    /**
+     * Set an error handler to handle errors within the test
+     *
+     * @param TestResult $result
+     * @param array      &$action
+     *
+     * @return callable|null
+     */
+    protected function handleErrors(TestResult $result, array &$action)
+    {
+        $handler = null;
+        $handler = set_error_handler(function ($severity, $message, $path, $line) use ($result, &$action, &$handler) {
+            // if there is an existing error handler, call it and record the result
+            $isHandled = $handler && false !== $handler($severity, $message, $path, $line);
+
+            if (!$isHandled) {
+                $result->getEventEmitter()->emit('error', [$severity, $message, $path, $line]);
+
+                // honor the error reporting configuration - this also takes care of the error control operator (@)
+                $errorReporting = error_reporting();
+                $shouldHandle = $severity === ($severity & $errorReporting);
+
+                if ($shouldHandle) {
+                    $this->failIfPassing($action, new ErrorException($message, 0, $severity, $path, $line));
+                }
+            }
+        });
+
+        return $handler;
+    }
+
+    /**
+     * Restore the previous error handler
+     *
+     * @param callable|null $handler
+     */
+    protected function restoreErrorHandler($handler)
+    {
+        if ($handler) {
+            set_error_handler($handler);
+        } else {
+            // unfortunately, we can't pass null until PHP 5.5
+            set_error_handler(function () { return false; });
+        }
+    }
+
+    /**
+     * Fail the test, but do not overwrite existing failures
+     *
+     * @param array &$action
+     * @param mixed $error
+     */
+    protected function failIfPassing(array &$action, $error)
+    {
+        if ('passTest' === $action[0]) {
+            $action = ['failTest', $this, $error];
+        }
+    }
 }

src/Runner/Runner.php

@@ -45,8 +45,6 @@ public function __construct(Suite $suite, Configuration $configuration, EventEmi
      */
     public function run(TestResult $result)
     {
-        $this->handleErrors();
-
         $this->eventEmitter->on('test.failed', function () {
             if ($this->configuration->shouldStopOnFailure()) {
                 $this->eventEmitter->emit('suite.halt');
@@ -58,24 +56,5 @@ public function run(TestResult $result)
         $start = microtime(true);
         $this->suite->run($result);
         $this->eventEmitter->emit('runner.end', [microtime(true) - $start]);
-
-        restore_error_handler();
-    }
-
-    /**
-     * Set an error handler to broadcast an error event.
-     */
-    protected function handleErrors()
-    {
-        $handler = null;
-        $handler = set_error_handler(function ($errno, $errstr, $errfile, $errline) use (&$handler) {
-            $this->eventEmitter->emit('error', [$errno, $errstr, $errfile, $errline]);
-
-            if ($handler) {
-                return $handler($errno, $errstr, $errfile, $errline);
-            }
-
-            return false;
-        });
     }
 }