diff --git a/ChangeLog-5.5.md b/ChangeLog-5.5.md
index 0e1d74c3b9e..46eb9e92916 100644
--- a/ChangeLog-5.5.md
+++ b/ChangeLog-5.5.md
@@ -2,6 +2,12 @@
 
 All notable changes of the PHPUnit 5.5 release series are documented in this file using the [Keep a CHANGELOG](http://keepachangelog.com/) principles.
 
+## [5.5.6] - 2016-MM-DD
+
+### Fixed
+
+* Fixed [#2261](https://github.com/sebastianbergmann/phpunit/issues/2261): Invalid test listener configuration leads to confusing behavior
+
 ## [5.5.5] - 2016-09-21
 
 ### Fixed
@@ -46,6 +52,7 @@ New release of PHPUnit as PHAR with updated dependencies
 
 * An `AssertionError` raised by an `assert()` in the tested code now causes the test to be interpreted as a failure instead of an error
 
+[5.5.6]: https://github.com/sebastianbergmann/phpunit/compare/5.5.5...5.5.6
 [5.5.5]: https://github.com/sebastianbergmann/phpunit/compare/5.5.4...5.5.5
 [5.5.4]: https://github.com/sebastianbergmann/phpunit/compare/5.5.3...5.5.4
 [5.5.3]: https://github.com/sebastianbergmann/phpunit/compare/5.5.2...5.5.3
diff --git a/src/TextUI/TestRunner.php b/src/TextUI/TestRunner.php
index 216a916993d..67635d065bc 100644
--- a/src/TextUI/TestRunner.php
+++ b/src/TextUI/TestRunner.php
@@ -882,22 +882,35 @@ protected function handleConfiguration(array &$arguments)
                     require_once $listener['file'];
                 }
 
-                if (class_exists($listener['class'])) {
-                    if (count($listener['arguments']) == 0) {
-                        $listener = new $listener['class'];
-                    } else {
-                        $listenerClass = new ReflectionClass(
+                if (!class_exists($listener['class'])) {
+                    throw new PHPUnit_Framework_Exception(
+                        sprintf(
+                            'Class "%s" does not exist',
                             $listener['class']
-                        );
-                        $listener      = $listenerClass->newInstanceArgs(
-                            $listener['arguments']
-                        );
-                    }
+                        )
+                    );
+                }
 
-                    if ($listener instanceof PHPUnit_Framework_TestListener) {
-                        $arguments['listeners'][] = $listener;
-                    }
+                $listenerClass = new ReflectionClass($listener['class']);
+
+                if (!$listenerClass->implementsInterface(PHPUnit_Framework_TestListener::class)) {
+                    throw new PHPUnit_Framework_Exception(
+                        sprintf(
+                            'Class "%s" does not implement the PHPUnit_Framework_TestListener interface',
+                            $listener['class']
+                        )
+                    );
+                }
+
+                if (count($listener['arguments']) == 0) {
+                    $listener = new $listener['class'];
+                } else {
+                    $listener = $listenerClass->newInstanceArgs(
+                        $listener['arguments']
+                    );
                 }
+
+                $arguments['listeners'][] = $listener;
             }
 
             $loggingConfiguration = $arguments['configuration']->getLoggingConfiguration();