Fixed test engine reporting when there is an exception in either the … (

#840)

* Fixed test engine reporting when there is an exception in either the init block, the beforeSpec method or the afterSpec method #771

* Updated mocha console #787

kotlintest-runner/kotlintest-runner-junit5/src/main/kotlin/io/kotlintest/runner/junit5/JUnitTestRunnerListener.kt

@@ -152,6 +152,13 @@ class JUnitTestRunnerListener(private val listener: EngineExecutionListener,
     results.add(ResultState(testCase, result))
   }
 
+  override fun specInitialisationFailed(klass: KClass<out Spec>, t: Throwable) {
+    logger.trace("specInitialisationFailed $klass $t")
+    // we must "start" the spec to get it to show
+    beforeSpecClass(klass)
+    afterSpecClass(klass, t)
+  }
+
   override fun afterSpecClass(klass: KClass<out Spec>, t: Throwable?) {
     logger.trace("afterSpecClass [$klass]")
 

kotlintest-runner/kotlintest-runner-junit5/src/main/kotlin/io/kotlintest/runner/junit5/KotlinTestEngine.kt

@@ -5,7 +5,6 @@ import io.kotlintest.Spec
 import io.kotlintest.description
 import io.kotlintest.runner.jvm.IsolationTestEngineListener
 import io.kotlintest.runner.jvm.SpecFilter
-import io.kotlintest.runner.jvm.SynchronizedTestEngineListener
 import io.kotlintest.runner.jvm.TestDiscovery
 import org.junit.platform.engine.EngineDiscoveryRequest
 import org.junit.platform.engine.ExecutionRequest
@@ -39,7 +38,7 @@ class KotlinTestEngine : TestEngine {
   override fun execute(request: ExecutionRequest) {
     logger.debug("JUnit execution request [configurationParameters=${request.configurationParameters}; rootTestDescriptor=${request.rootTestDescriptor}]")
     val root = request.rootTestDescriptor as KotlinTestEngineDescriptor
-    val listener = SynchronizedTestEngineListener(IsolationTestEngineListener(JUnitTestRunnerListener(SynchronizedEngineExecutionListener(request.engineExecutionListener), root)))
+    val listener = IsolationTestEngineListener(JUnitTestRunnerListener(SynchronizedEngineExecutionListener(request.engineExecutionListener), root))
     val runner = io.kotlintest.runner.jvm.TestEngine(root.classes, emptyList(), Project.parallelism(), listener)
     runner.execute()
   }

kotlintest-runner/kotlintest-runner-jvm/src/main/kotlin/io/kotlintest/runner/jvm/IsolationTestEngineListener.kt

@@ -42,9 +42,13 @@ class IsolationTestEngineListener(val listener: TestEngineListener) : TestEngine
   }
 
   override fun specInitialisationFailed(klass: KClass<out Spec>, t: Throwable) {
-    runningSpec.compareAndSet(null, Description.spec(klass)) // otherwise the before & after spec functions do nothing
-    beforeSpecClass(klass)
-    afterSpecClass(klass, t)
+    if (runningSpec.compareAndSet(null, Description.spec(klass))) {
+      listener.specInitialisationFailed(klass, t)
+    } else {
+      queue {
+        specInitialisationFailed(klass, t)
+      }
+    }
   }
 
   override fun beforeSpecClass(klass: KClass<out Spec>) {

kotlintest-runner/kotlintest-runner-jvm/src/main/kotlin/io/kotlintest/runner/jvm/TestEngine.kt

@@ -9,7 +9,6 @@ import io.kotlintest.runner.jvm.spec.SpecExecutor
 import org.slf4j.LoggerFactory
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
-import java.util.concurrent.atomic.AtomicReference
 import kotlin.reflect.KClass
 
 class TestEngine(val classes: List<KClass<out Spec>>,
@@ -18,7 +17,6 @@ class TestEngine(val classes: List<KClass<out Spec>>,
                  val listener: TestEngineListener) {
 
   private val logger = LoggerFactory.getLogger(this.javaClass)
-  private val error = AtomicReference<Throwable?>(null)
 
   // the main executor is used to parallelize the execution of specs
   // inside a spec, tests themselves are executed as coroutines
@@ -41,26 +39,26 @@ class TestEngine(val classes: List<KClass<out Spec>>,
   }
 
   private fun submitAll() = Try {
-    logger.debug("Submitting ${classes.size} specs")
+    logger.trace("Submitting ${classes.size} specs")
 
     // the classes are ordered using an instance of SpecExecutionOrder before
     // being submitted in the order returned
     Project.specExecutionOrder().sort(classes).forEach { submitSpec(it) }
     executor.shutdown()
 
-    logger.debug("Waiting for spec execution to terminate")
-    try {
+    logger.trace("Waiting for spec execution to terminate")
+    val error = try {
       executor.awaitTermination(1, TimeUnit.DAYS)
+      null
     } catch (t: InterruptedException) {
-      error.compareAndSet(null, t)
+      t
     }
 
     // the executor may have terminated early because it was shutdown immediately
     // by an error in a submission. This will be reflected in the error reference
     // being set to a non null value
-    val t = error.get()
-    if (t != null)
-      throw t
+    if (error != null)
+      throw error
   }
 
   private fun end(t: Throwable?) = Try {
@@ -89,17 +87,14 @@ class TestEngine(val classes: List<KClass<out Spec>>,
   private fun submitSpec(klass: KClass<out Spec>) {
     executor.submit {
       createSpec(klass).fold(
-          // if there is an error creating the spec then we
-          // will add a placeholder spec so we can see the error in intellij/gradle
-          // otherwise it won't appear
           { t ->
             listener.specInitialisationFailed(klass, t)
-            error.compareAndSet(null, t)
             executor.shutdownNow()
           },
           { spec ->
             specExecutor.execute(spec).onFailure { t ->
-              error.compareAndSet(null, t)
+              // todo move this to a new listener method like specFailed(klass, t)
+              listener.specInitialisationFailed(klass, t)
               executor.shutdownNow()
             }
           }

kotlintest-runner/kotlintest-runner-jvm/src/main/kotlin/io/kotlintest/runner/jvm/TestEngineListener.kt

@@ -43,10 +43,7 @@ interface TestEngineListener {
   /**
    * Is invoked if a [Spec] throws an exception during initialisation
    */
-  fun specInitialisationFailed(klass: KClass<out Spec>, t: Throwable) {
-    beforeSpecClass(klass)
-    afterSpecClass(klass, t)
-  }
+  fun specInitialisationFailed(klass: KClass<out Spec>, t: Throwable) {}
 
   /**
    * Invoked each time a [TestCase] has been entered from a parent test.

kotlintest-runner/kotlintest-runner-jvm/src/main/kotlin/io/kotlintest/runner/jvm/jvm.kt

@@ -11,8 +11,8 @@ fun <T : Spec> instantiateSpec(clazz: KClass<T>): Try<Spec> = Try {
 
   val nullSpec: Spec? = null
 
-  val instance = Project.constructorExtensions().fold(nullSpec) { spec, ext -> spec ?: ext.instantiate(clazz) }
-      ?: clazz.createInstance()
+  val instance = Project.constructorExtensions()
+      .fold(nullSpec) { spec, ext -> spec ?: ext.instantiate(clazz) } ?: clazz.createInstance()
 
   // after the class is created we no longer allow new top level tests to be added
   if (instance is AbstractSpec) {