Skip to content
Permalink
Browse files

Introduced isolation mode for specs, reimplemented test runner to run…

… a node to completion before starting the next test #379
  • Loading branch information...
sksamuel committed Jul 28, 2018
1 parent e210f93 commit 30e6f8ecbd19c974f5c3275c27cf7bc284011e48
@@ -31,3 +31,4 @@ kotlintest-tests-junit-report.log
kotlintest-tests-core.log

local.properties
.kotlintest
@@ -39,8 +39,11 @@ interface Spec : TestListener {
* per test. This is due to implementation trickery required
* with nested closures and junit test discovery.
*/
@Deprecated("Instead of this function, override specIsolationMode() which should return a SpecIsolationMode value indicating how the isolation level should be set for this spec")
fun isInstancePerTest(): Boolean

fun specIsolationMode(): SpecIsolationMode? = null

/**
* Override this function to register extensions
* which will be invoked during execution of this spec.
@@ -0,0 +1,7 @@
package io.kotlintest

enum class SpecIsolationMode {
SharedInstance,
InstancePerNode,
InstancePerLeaf
}
@@ -23,6 +23,8 @@ interface TestListener {
*/
fun afterTest(description: Description, result: TestResult): Unit = Unit

fun beforeSpecStarted(description: Description, spec: Spec): Unit = Unit

/**
* Is invoked each time a [Spec] is started.
*
@@ -45,6 +47,8 @@ interface TestListener {
*/
fun afterSpec(description: Description, spec: Spec): Unit = Unit

fun afterSpecCompleted(description: Description, spec: Spec): Unit = Unit

/**
* Is invoked as soon as the Test Engine is started.
*/
@@ -8,6 +8,10 @@ import io.kotlintest.TestCase
import io.kotlintest.TestContext
import kotlin.reflect.KClass

object InstancePerTestSpecRunnerFactory : SpecRunnerFactory {
override fun specRunner(listener: TestEngineListener) = InstancePerTestSpecRunner(listener)
}

class InstancePerTestSpecRunner(listener: TestEngineListener) : SpecRunner(listener) {

private val executed = HashSet<Description>()
@@ -0,0 +1,93 @@
package io.kotlintest.runner.jvm

import arrow.core.Failure
import arrow.core.Success
import io.kotlintest.Description
import io.kotlintest.Spec
import io.kotlintest.TestCase
import io.kotlintest.TestContext
import org.slf4j.LoggerFactory
import java.util.*

class InstancePerTestSpecRunner2(listener: TestEngineListener) : SpecRunner(listener) {

private val logger = LoggerFactory.getLogger(this.javaClass)

private val executed = HashSet<Description>()
private val discovered = HashSet<Description>()
private val queue = ArrayDeque<TestCase>()

/**
* When executing a [TestCase], any child test cases that are found, are placed onto
* a stack. When the test case has completed, we take the next test case from the
* stack, and begin executing that.
*/
override fun execute(spec: Spec) {
topLevelTests(spec).forEach { enqueue(it) }
logger.debug("queue=$queue")
while (queue.isNotEmpty()) {
val element = queue.removeFirst()
logger.debug("Retrieving element from queue: ${element.name}")
execute(element)
}
logger.debug("Final queue=$queue")
}

private fun enqueue(testCase: TestCase) {
if (!discovered.contains(testCase.description)) {
discovered.add(testCase.description)
queue.add(testCase)
}
logger.debug("new queue=$queue")
}

// todo add checks for duplicate test names at runtime
//if (executed.contains(testCase.description))
//throw IllegalStateException("Cannot add duplicate test name ${testCase.name}")

/**
* The intention of this runner is that each [TestCase] executes in it's own instance
* of the containing [Spec] class. Therefore, when we begin executing a test case from
* the queue, we must first instantiate a new spec, and begin execution on _that_ instance.
*
* As test lambdas are executed, nested test cases will be registered, these should be ignored
* if they are not an ancestor of the target. If they are then we can step into them, and
* continue recursively until we find the target.
*
* One the target is found it can be executed as normal, and any test lambdas it contains
* can be registered back with the stack for execution later.
*/
private fun execute(testCase: TestCase) {
logger.debug("Executing $testCase")
instantiateSpec(testCase.spec::class).let {
when (it) {
is Failure -> throw it.exception
is Success -> {
val spec = it.value
interceptSpec(spec) {
spec.testCases().forEach { locate(testCase.description, it) }
}
}
}
}
}

private fun locate(target: Description, current: TestCase) {
// if equals then we've found the test we want to invoke
if (target == current.description) {
val context = object : TestContext() {
override fun description(): Description = target
override fun registerTestCase(testCase: TestCase) {
enqueue(testCase)
}
}
io.kotlintest.runner.jvm.TestCaseExecutor(listener, current, context).execute()
// otherwise if it's an ancestor then we want to search it recursively
} else if (current.description.isAncestorOf(target)) {
current.test.invoke(object : TestContext() {
override fun description(): Description = current.description
override fun registerTestCase(testCase: TestCase) = locate(target, testCase)
})
}
}
}
@@ -8,6 +8,10 @@ import io.kotlintest.TestCaseOrder
import io.kotlintest.extensions.SpecExtension
import io.kotlintest.extensions.SpecInterceptContext

interface SpecRunnerFactory {
fun specRunner(listener: TestEngineListener): SpecRunner
}

abstract class SpecRunner(val listener: TestEngineListener) {

abstract fun execute(spec: Spec)
@@ -4,6 +4,7 @@ import arrow.core.Try
import io.kotlintest.Description
import io.kotlintest.Project
import io.kotlintest.Spec
import io.kotlintest.SpecIsolationMode
import org.slf4j.LoggerFactory
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
@@ -108,17 +109,25 @@ class TestEngine(val classes: List<KClass<out Spec>>,
private fun executeSpec(spec: Spec) = Try {
listener.prepareSpec(spec.description(), spec::class)
Try {
spec.beforeSpecStarted(spec.description(), spec)
runner(spec).execute(spec)
spec.afterSpecCompleted(spec.description(), spec)
}.fold(
{ listener.completeSpec(spec.description(), spec.javaClass.kotlin, it) },
{ listener.completeSpec(spec.description(), spec.javaClass.kotlin, null) }
)
spec.closeResources()
}

private fun runner(spec: Spec): SpecRunner =
when {
spec.isInstancePerTest() -> InstancePerTestSpecRunner(listener)
private fun runner(spec: Spec): SpecRunner {
return when (spec.specIsolationMode()) {
SpecIsolationMode.SharedInstance -> SharedInstanceSpecRunner(listener)
SpecIsolationMode.InstancePerLeaf -> InstancePerTestSpecRunner2(listener)
SpecIsolationMode.InstancePerNode -> InstancePerTestSpecRunner2(listener)
null -> when {
spec.isInstancePerTest() -> InstancePerTestSpecRunner2(listener)
else -> SharedInstanceSpecRunner(listener)
}
}
}
}
@@ -0,0 +1,42 @@
package com.sksamuel.kotlintest.specs

import io.kotlintest.Description
import io.kotlintest.Spec
import io.kotlintest.SpecIsolationMode
import io.kotlintest.shouldBe
import io.kotlintest.specs.FreeSpec

class OneInstancePerTestOrderingTest : FreeSpec() {

companion object {
var string = ""
var count = 0
}

override fun isInstancePerTest(): Boolean = true

override fun specIsolationMode(): SpecIsolationMode? = SpecIsolationMode.InstancePerNode

override fun afterSpecCompleted(description: Description, spec: Spec) {
string shouldBe "a_ab_ae_abc_abd_"
}

init {
"1" - {
string += "a"
"1.1" - {
string += "b"
"1.1.1" {
string += "c"
}
"1.1.2" {
string += "d"
}
}
"1.2" {
string += "e"
}
string += "_"
}
}
}

0 comments on commit 30e6f8e

Please sign in to comment.
You can’t perform that action at this time.