Skip to content
Permalink
Browse files

Error in spec initialization is not visible #1097

  • Loading branch information
sksamuel committed Jan 13, 2020
1 parent 776fc64 commit 5e0159eadc2296fbb56ad24ff3396d182468e53c
Showing with 239 additions and 154 deletions.
  1. +1 −0 CHANGELOG.md
  2. +9 −0 kotest-core/src/commonMain/kotlin/io/kotest/core/spec/parallelise.kt
  3. +5 −0 kotest-core/src/jsMain/kotlin/io/kotest/core/spec/isDoNotParallelize.kt
  4. +0 −3 kotest-core/src/jvmMain/kotlin/io/kotest/AbstractSpec.kt
  5. +6 −0 kotest-core/src/jvmMain/kotlin/io/kotest/core/spec/isDoNotParallelize.kt
  6. +43 −17 .../jvmMain/kotlin/io/kotest/runner/junit5/{JUnitTestRunnerListener.kt → JUnitTestEngineListener.kt}
  7. +4 −5 .../kotest-runner-junit5/src/jvmMain/kotlin/io/kotest/runner/junit5/KotestJunitPlatformTestEngine.kt
  8. +6 −1 kotest-runner/kotest-runner-junit5/src/jvmMain/kotlin/io/kotest/runner/junit5/discoveryRequest.kt
  9. +20 −29 kotest-runner/kotest-runner-junit5/src/jvmMain/kotlin/io/kotest/runner/junit5/tostring.kt
  10. +45 −47 ...runner-junit5/src/jvmTest/kotlin/com/sksamuel/kotest/runner/junit5/JUnitTestRunnerListenerTest.kt
  11. +50 −0 ...runner-junit5/src/jvmTest/kotlin/com/sksamuel/kotest/runner/junit5/SpecInitializationErrorTest.kt
  12. +15 −16 ...t-runner/kotest-runner-jvm/src/jvmMain/kotlin/io/kotest/runner/jvm/IsolationTestEngineListener.kt
  13. +9 −17 kotest-runner/kotest-runner-jvm/src/jvmMain/kotlin/io/kotest/runner/jvm/KotestEngine.kt
  14. +1 −1 kotest-runner/kotest-runner-jvm/src/jvmMain/kotlin/io/kotest/runner/jvm/TestDiscovery.kt
  15. +3 −6 kotest-runner/kotest-runner-jvm/src/jvmMain/kotlin/io/kotest/runner/jvm/TestEngineListener.kt
  16. 0 ...st-runner/kotest-runner-jvm/src/jvmMain/kotlin/io/kotest/runner/jvm/{jvm.kt → instantiateSpec.kt}
  17. +9 −3 kotest-runner/kotest-runner-jvm/src/jvmMain/kotlin/io/kotest/runner/jvm/spec/SpecExecutor.kt
  18. +0 −9 kotest-runner/kotest-runner-jvm/src/jvmMain/kotlin/io/kotest/runner/jvm/spec/SpecRunner.kt
  19. +13 −0 ...sts/kotest-tests-core/src/jvmTest/kotlin/com/sksamuel/kotest/specs/SpecInitializationErrorTest.kt
@@ -36,6 +36,7 @@ Changelog
* SkipTestException removed
* Fixed file matchers to check for null
* Test callbacks now coroutine enabled
* Bugfix: Error in spec initialization is cryptic with recent intellij #1097

#### 3.4.2

@@ -0,0 +1,9 @@
package io.kotest.core.spec

import kotlin.reflect.KClass

@Target(AnnotationTarget.CLASS)
@Retention(AnnotationRetention.RUNTIME)
annotation class DoNotParallelize

expect fun KClass<*>.isDoNotParallelize(): Boolean
@@ -0,0 +1,5 @@
package io.kotest.core.spec

import kotlin.reflect.KClass

actual fun KClass<*>.isDoNotParallelize(): Boolean = false
@@ -2,6 +2,3 @@ package io.kotest



@Target(AnnotationTarget.CLASS)
@Retention(AnnotationRetention.RUNTIME)
annotation class DoNotParallelize
@@ -0,0 +1,6 @@
package io.kotest.core.spec

import kotlin.reflect.KClass
import kotlin.reflect.full.findAnnotation

actual fun KClass<*>.isDoNotParallelize(): Boolean = findAnnotation<DoNotParallelize>() != null
@@ -1,6 +1,8 @@
package io.kotest.runner.junit5

import arrow.core.extensions.list.foldable.exists
import io.kotest.Project
import io.kotest.core.spec.FunSpec
import io.kotest.core.spec.SpecConfiguration
import io.kotest.core.spec.description
import io.kotest.core.test.*
@@ -60,7 +62,9 @@ import kotlin.reflect.KClass
* Must start tests after their parent or they can go missing.
* Sibling containers can start and finish in parallel.
*/
class JUnitTestRunnerListener(
private class NoTests : FunSpec()

class JUnitTestEngineListener(
private val listener: EngineExecutionListener,
val root: EngineDescriptor
) : TestEngineListener {
@@ -82,6 +86,10 @@ class JUnitTestRunnerListener(
override fun engineStarted(classes: List<KClass<out SpecConfiguration>>) {
logger.trace("Engine started; classes=[$classes]")
listener.executionStarted(root)
val desc = createSpecDescriptor(NoTests::class)
listener.dynamicTestRegistered(desc)
listener.executionStarted(desc)
listener.executionFinished(desc, TestExecutionResult.successful())
}

/**
@@ -115,7 +123,7 @@ class JUnitTestRunnerListener(
}

override fun specStarted(kclass: KClass<out SpecConfiguration>) {
logger.trace("beforeSpecClass [${kclass.qualifiedName}]")
logger.trace("specStarted [${kclass.qualifiedName}]")
try {
val descriptor = createSpecDescriptor(kclass)
logger.trace("Registering junit dynamic test: $descriptor")
@@ -128,21 +136,6 @@ class JUnitTestRunnerListener(
}
}

override fun specFailed(klass: KClass<out SpecConfiguration>, t: Throwable) {
logger.trace("beforeSpecClass [${klass.qualifiedName}]")
try {
val descriptor = createSpecDescriptor(klass)
logger.trace("Registering junit dynamic test: $descriptor")
listener.dynamicTestRegistered(descriptor)
logger.trace("Notifying junit that execution has started: $descriptor")
listener.executionStarted(descriptor)
listener.executionFinished(descriptor, TestExecutionResult.aborted(t))
} catch (t: Throwable) {
logger.error("Error in JUnit Platform listener", t)
specException = t
}
}

override fun specFinished(
klass: KClass<out SpecConfiguration>,
t: Throwable?,
@@ -156,6 +149,7 @@ class JUnitTestRunnerListener(
val nestedFailure = findChildFailure(klass.description())

val result = when {
t != null -> TestExecutionResult.failed(t)
specException != null -> TestExecutionResult.failed(specException)
nestedFailure != null -> nestedFailure.testExecutionResult()
else -> TestExecutionResult.successful()
@@ -164,9 +158,41 @@ class JUnitTestRunnerListener(
logger.trace("Notifying junit that execution has finished: $descriptor, $result")
// we are ignoring junit guidelines here and failing the spec if any of it's tests failed
// this is because in gradle and intellij nested errors are not very obvious
//ensureSpecVisible(klass)
listener.executionFinished(descriptor, result)
}

/**
* If the spec fails to be created, then there will be no tests, so we should insert an instantiation
* failed test so that the spec shows up.
*/
override fun specInstantiationError(kclass: KClass<out SpecConfiguration>, t: Throwable) {
val description = kclass.description()
val spec = descriptors[description]!!
val test = spec.append(description.append("Spec instantiation failed"), TestDescriptor.Type.TEST, null)
if (!isVisible(description)) {
listener.dynamicTestRegistered(test)
listener.executionStarted(test)
listener.executionFinished(test, TestExecutionResult.aborted(t))
}
}

// if no test events were received for a spec then we need to insert a dummy
// placeholder test so the ide shows the spec, otherwise it just disappears
// private fun ensureSpecVisible(kclass: KClass<out SpecConfiguration>) {
// val description = kclass.description()
// if (!isVisible(description)) {
// val spec = descriptors[description]!!
// val test = spec.append(description.append("No tests"), TestDescriptor.Type.TEST, null)
// listener.dynamicTestRegistered(test)
// listener.executionStarted(test)
// listener.executionFinished(test, TestExecutionResult.successful())
// }
// }

private fun isVisible(description: Description) =
results.exists { description.isAncestorOf(it.first) }

override fun testStarted(testCase: TestCase) {
val descriptor = createTestDescriptor(testCase)
logger.trace("Registering junit dynamic test: $descriptor")
@@ -29,10 +29,10 @@ class KotestJunitPlatformTestEngine : TestEngine {
override fun getId(): String = EngineId

override fun execute(request: ExecutionRequest) {
logger.debug("JUnit execution request [configurationParameters=${request.configurationParameters}; rootTestDescriptor=${request.rootTestDescriptor}]")
logger.debug("JUnit ExecutionRequest[${request::class.java.name}] [configurationParameters=${request.configurationParameters}; rootTestDescriptor=${request.rootTestDescriptor}]")
val root = request.rootTestDescriptor as KotestEngineDescriptor
val listener = IsolationTestEngineListener(
JUnitTestRunnerListener(
JUnitTestEngineListener(
SynchronizedEngineExecutionListener(request.engineExecutionListener),
root
)
@@ -52,9 +52,8 @@ class KotestJunitPlatformTestEngine : TestEngine {
request: EngineDiscoveryRequest,
uniqueId: UniqueId
): KotestEngineDescriptor {
logger.trace("configurationParameters=" + request.configurationParameters)
logger.trace("uniqueId=$uniqueId")
logger.trace(request.string())
logger.debug("uniqueId=$uniqueId")
logger.debug(request.string())

val postFilters = request.postFilters()

@@ -8,9 +8,11 @@ import org.junit.platform.engine.discovery.ClassSelector
import org.junit.platform.engine.discovery.ClasspathRootSelector
import org.junit.platform.engine.discovery.DirectorySelector
import org.junit.platform.engine.discovery.MethodSelector
import org.junit.platform.engine.discovery.NestedMethodSelector
import org.junit.platform.engine.discovery.PackageNameFilter
import org.junit.platform.engine.discovery.PackageSelector
import org.junit.platform.engine.discovery.UriSelector
import org.junit.platform.engine.discovery.UniqueIdSelector

/**
* Returns a Kotest [DiscoveryRequest] built from the selectors and filters present
@@ -30,7 +32,10 @@ import org.junit.platform.engine.discovery.UriSelector
*
* Unsupported selectors are:
*
* - [MethodSelector] - not supported because kotest does not define tests as methods/functions
* - [MethodSelector] - not supported because kotest does not define tests as methods
* - [NestedMethodSelector] - not supported becase kotest does not define tests as methods
* - [UniqueIdSelector]
* - [DirectorySelector]
*/
internal fun discoveryRequest(request: EngineDiscoveryRequest): DiscoveryRequest {

@@ -15,34 +15,25 @@ import org.junit.platform.engine.discovery.UniqueIdSelector
import org.junit.platform.engine.discovery.UriSelector
import org.junit.platform.launcher.LauncherDiscoveryRequest

fun EngineDiscoveryRequest.string() =
"EngineDiscoveryRequest [\n" +
"classpathRootSelectors=${this.getSelectorsByType(ClasspathRootSelector::class.java)}\n" +
"classpathResourceSelectors=${this.getSelectorsByType(ClasspathResourceSelector::class.java)}\n" +
"classSelectors=${this.getSelectorsByType(ClassSelector::class.java).map { it.className }}\n" +
"methodSelectors=${this.getSelectorsByType(MethodSelector::class.java)}\n" +
"directorySelectors=${this.getSelectorsByType(DirectorySelector::class.java)}\n" +
"fileSelectors=${this.getSelectorsByType(FileSelector::class.java)}\n" +
"moduleSelectors=${this.getSelectorsByType(ModuleSelector::class.java)}\n" +
"packageSelectors=${this.getSelectorsByType(PackageSelector::class.java)}\n" +
"uniqueIdSelectors=${this.getSelectorsByType(UniqueIdSelector::class.java)}\n" +
"uriSelectors=${this.getSelectorsByType(UriSelector::class.java)}\n" +
"classnameFilters=${this.getFiltersByType(ClassNameFilter::class.java)}\n" +
"packageNameFilters=${this.getFiltersByType(PackageNameFilter::class.java)}\n]"
fun EngineDiscoveryRequest.string() = when (this) {
is LauncherDiscoveryRequest -> this.string()
else -> "EngineDiscoveryRequest []"
}

fun LauncherDiscoveryRequest.string() =
"LauncherDiscoveryRequest [\n" +
"classpathRootSelectors=${this.getSelectorsByType(ClasspathRootSelector::class.java)}\n" +
"classpathResourceSelectors=${this.getSelectorsByType(ClasspathResourceSelector::class.java)}\n" +
"classSelectors=${this.getSelectorsByType(ClassSelector::class.java).map { it.className }}\n" +
"methodSelectors=${this.getSelectorsByType(MethodSelector::class.java)}\n" +
"directorySelectors=${this.getSelectorsByType(DirectorySelector::class.java)}\n" +
"fileSelectors=${this.getSelectorsByType(FileSelector::class.java)}\n" +
"moduleSelectors=${this.getSelectorsByType(ModuleSelector::class.java)}\n" +
"packageSelectors=${this.getSelectorsByType(PackageSelector::class.java)}\n" +
"uniqueIdSelectors=${this.getSelectorsByType(UniqueIdSelector::class.java)}\n" +
"uriSelectors=${this.getSelectorsByType(UriSelector::class.java)}\n" +
"engineFilters=${this.engineFilters}\n" +
"postDiscoveryFilters=${this.postDiscoveryFilters}\n" +
"classnameFilters=${this.getFiltersByType(ClassNameFilter::class.java)}\n" +
"packageNameFilters=${this.getFiltersByType(PackageNameFilter::class.java)}\n]"
"LauncherDiscoveryRequest [\n" +
"configurationParameters=${this.configurationParameters}\n" +
"classpathRootSelectors=${this.getSelectorsByType(ClasspathRootSelector::class.java)}\n" +
"classpathResourceSelectors=${this.getSelectorsByType(ClasspathResourceSelector::class.java)}\n" +
"classSelectors=${this.getSelectorsByType(ClassSelector::class.java).map { it.className }}\n" +
"methodSelectors=${this.getSelectorsByType(MethodSelector::class.java)}\n" +
"directorySelectors=${this.getSelectorsByType(DirectorySelector::class.java)}\n" +
"fileSelectors=${this.getSelectorsByType(FileSelector::class.java)}\n" +
"moduleSelectors=${this.getSelectorsByType(ModuleSelector::class.java)}\n" +
"packageSelectors=${this.getSelectorsByType(PackageSelector::class.java)}\n" +
"uniqueIdSelectors=${this.getSelectorsByType(UniqueIdSelector::class.java)}\n" +
"uriSelectors=${this.getSelectorsByType(UriSelector::class.java)}\n" +
"engineFilters=${this.engineFilters}\n" +
"postDiscoveryFilters=${this.postDiscoveryFilters}\n" +
"classnameFilters=${this.getFiltersByType(ClassNameFilter::class.java)}\n" +
"packageNameFilters=${this.getFiltersByType(PackageNameFilter::class.java)}\n]"
@@ -8,7 +8,7 @@ import io.kotest.core.test.TestCase
import io.kotest.core.test.TestResult
import io.kotest.core.test.TestType
import io.kotest.matchers.maps.shouldHaveSize
import io.kotest.runner.junit5.JUnitTestRunnerListener
import io.kotest.runner.junit5.JUnitTestEngineListener
import io.kotest.runner.junit5.KotestEngineDescriptor
import io.kotest.shouldBe
import org.junit.platform.engine.EngineExecutionListener
@@ -20,60 +20,58 @@ import kotlin.time.Duration
import kotlin.time.ExperimentalTime

@UseExperimental(ExperimentalTime::class)
val tests = funSpec {
context("foo") {
test("failed test should fail parent and spec") {
val childFailsParentTest = funSpec {
test("failed test should fail parent and spec") {

val root = KotestEngineDescriptor(UniqueId.forEngine("kotest"), emptyList())
val root = KotestEngineDescriptor(UniqueId.forEngine("kotest"), emptyList())

val finished = mutableMapOf<String, TestExecutionResult.Status>()
val finished = mutableMapOf<String, TestExecutionResult.Status>()

val engineListener = object : EngineExecutionListener {
override fun executionFinished(testDescriptor: TestDescriptor, testExecutionResult: TestExecutionResult) {
finished[testDescriptor.displayName] = testExecutionResult.status
}

override fun reportingEntryPublished(testDescriptor: TestDescriptor?, entry: ReportEntry?) {}
override fun executionSkipped(testDescriptor: TestDescriptor?, reason: String?) {}
override fun executionStarted(testDescriptor: TestDescriptor?) {}
override fun dynamicTestRegistered(testDescriptor: TestDescriptor?) {}
val engineListener = object : EngineExecutionListener {
override fun executionFinished(testDescriptor: TestDescriptor, testExecutionResult: TestExecutionResult) {
finished[testDescriptor.displayName] = testExecutionResult.status
}

val test1 = TestCase(
JUnitTestRunnerListenerTests::class.description().append("test1"),
JUnitTestRunnerListenerTests(),
{ },
sourceRef(),
TestType.Container
)
override fun reportingEntryPublished(testDescriptor: TestDescriptor?, entry: ReportEntry?) {}
override fun executionSkipped(testDescriptor: TestDescriptor?, reason: String?) {}
override fun executionStarted(testDescriptor: TestDescriptor?) {}
override fun dynamicTestRegistered(testDescriptor: TestDescriptor?) {}
}

val test1 = TestCase(
JUnitTestRunnerListenerTests::class.description().append("test1"),
JUnitTestRunnerListenerTests(),
{ },
sourceRef(),
TestType.Container
)

val test2 = TestCase(
test1.description.append("test2"),
JUnitTestRunnerListenerTests(),
{ },
sourceRef(),
TestType.Container
)
val test2 = TestCase(
test1.description.append("test2"),
JUnitTestRunnerListenerTests(),
{ },
sourceRef(),
TestType.Container
)

val listener = JUnitTestRunnerListener(engineListener, root)
listener.engineStarted(emptyList())
listener.specStarted(JUnitTestRunnerListenerTests::class)
listener.testStarted(test1)
listener.testStarted(test2)
listener.testFinished(test2, TestResult.failure(AssertionError("boom"), Duration.ZERO))
listener.testFinished(test1, TestResult.success(Duration.ZERO))
listener.specFinished(JUnitTestRunnerListenerTests::class, null, emptyMap())
listener.engineFinished(null)
val listener = JUnitTestEngineListener(engineListener, root)
listener.engineStarted(emptyList())
listener.specStarted(JUnitTestRunnerListenerTests::class)
listener.testStarted(test1)
listener.testStarted(test2)
listener.testFinished(test2, TestResult.failure(AssertionError("boom"), Duration.ZERO))
listener.testFinished(test1, TestResult.success(Duration.ZERO))
listener.specFinished(JUnitTestRunnerListenerTests::class, null, emptyMap())
listener.engineFinished(null)

finished.shouldHaveSize(4)
finished.toMap() shouldBe mapOf(
"test1" to TestExecutionResult.Status.FAILED,
"test2" to TestExecutionResult.Status.FAILED,
"com.sksamuel.kotest.runner.junit5.JUnitTestRunnerListenerTests" to TestExecutionResult.Status.FAILED,
"Kotest" to TestExecutionResult.Status.SUCCESSFUL
)
}
finished.shouldHaveSize(4)
finished.toMap() shouldBe mapOf(
"test1" to TestExecutionResult.Status.FAILED,
"test2" to TestExecutionResult.Status.FAILED,
"com.sksamuel.kotest.runner.junit5.JUnitTestRunnerListenerTests" to TestExecutionResult.Status.FAILED,
"Kotest" to TestExecutionResult.Status.SUCCESSFUL
)
}
}

class JUnitTestRunnerListenerTests : CompositeSpec(tests)
class JUnitTestRunnerListenerTests : CompositeSpec(childFailsParentTest)

0 comments on commit 5e0159e

Please sign in to comment.
You can’t perform that action at this time.