Skip to content
Permalink
Browse files

Timeout config fails #879 Global timeout #858 (#910)

* Timeout config fails #879 Global timeout #858

* Fixing tests

* Fixed test
  • Loading branch information...
sksamuel committed Jul 21, 2019
1 parent 5fd5496 commit d04f05b1c9f5bf757704052303ee35a3146a6420
@@ -4,6 +4,7 @@ import io.kotlintest.extensions.ProjectExtension
import io.kotlintest.extensions.ProjectLevelExtension
import io.kotlintest.extensions.ProjectListener
import io.kotlintest.extensions.TestListener
import java.time.Duration

/**
* Project-wide configuration. Extensions returned by an
@@ -55,7 +56,13 @@ abstract class AbstractProjectConfig {
/**
* The [IsolationMode] set here will be applied if the isolation mode in a spec is null.
*/
fun isolationMode(): IsolationMode? = null
open fun isolationMode(): IsolationMode? = null

/**
* A global timeout that is applied to all tests if not null.
* Tests which define their own timeout will override this.
*/
open val timeout: Duration? = null

/**
* Override this function and return a number greater than 1 if you wish to
@@ -4,6 +4,7 @@ package io.kotlintest

import io.kotlintest.extensions.*
import java.lang.StringBuilder
import java.time.Duration

/**
* Internal class used to hold project wide configuration.
@@ -30,14 +31,14 @@ object Project {
}

private const val defaultProjectConfigFullyQualifiedName = "io.kotlintest.provided.ProjectConfig"
private val timeoutDefault = Duration.ofSeconds(600)

private fun discoverProjectConfig(): AbstractProjectConfig? {
return try {
val projectConfigFullyQualifiedName = System.getProperty("kotlintest.project.config")
?: defaultProjectConfigFullyQualifiedName
val clas = Class.forName(projectConfigFullyQualifiedName)
val field = clas.declaredFields.find { it.name == "INSTANCE" }
when (field) {
when (val field = clas.declaredFields.find { it.name == "INSTANCE" }) {
// if the static field for an object cannot be found, then instantiate
null -> clas.newInstance() as AbstractProjectConfig
// if the static field can be found then use it
@@ -56,6 +57,7 @@ object Project {
private var writeSpecFailureFile: Boolean = true
private var _globalAssertSoftly: Boolean = false
private var parallelism: Int = 1
private var _timeout: Duration? = null

fun discoveryExtensions(): List<DiscoveryExtension> = _extensions.filterIsInstance<DiscoveryExtension>()
fun constructorExtensions(): List<ConstructorExtension> = _extensions.filterIsInstance<ConstructorExtension>()
@@ -71,6 +73,8 @@ object Project {
fun globalAssertSoftly(): Boolean = _globalAssertSoftly
fun parallelism() = parallelism

fun timeout(): Duration = _timeout ?: timeoutDefault

var failOnIgnoredTests: Boolean = false

fun tags(): Tags {
@@ -85,6 +89,7 @@ object Project {
_filters.addAll(it.filters())
_specExecutionOrder = it.specExecutionOrder()
_globalAssertSoftly = System.getProperty("kotlintest.assertions.global-assert-softly") == "true" || it.globalAssertSoftly
_timeout = it.timeout
parallelism = System.getProperty("kotlintest.parallelism")?.toInt() ?: it.parallelism()
writeSpecFailureFile = System.getProperty("kotlintest.write.specfailures") == "true" || it.writeSpecFailureFile()
failOnIgnoredTests = System.getProperty("kotlintest.build.fail-on-ignore") == "true" || it.failOnIgnoredTests
@@ -70,11 +70,6 @@ interface Spec : TestListener {
*/
fun testCases(): List<TestCase>

/**
* Returns the focused tests for this Spec. Can be empty if no test is marked as focused.
*/
fun focused(): List<TestCase> = testCases().filter { it.name.startsWith("f:") }

fun hasFocusedTest(): Boolean = focused().isNotEmpty()

fun closeResources()
@@ -125,3 +120,8 @@ private class LazyWithReceiver<This, Return>(val initializer: This.() -> Return)
return values.getOrPut(thisRef) { thisRef.initializer() }
}
}

/**
* Returns the focused tests for this Spec. Can be empty if no test is marked as focused.
*/
fun Spec.focused(): List<TestCase> = testCases().filter { it.name.startsWith("f:") }
@@ -4,18 +4,23 @@ import io.kotlintest.extensions.TestCaseExtension
import java.time.Duration

data class TestCaseConfig(
val enabled: Boolean = true,
val invocations: Int = 1,
val timeout: Duration = Duration.ofSeconds(600),
val enabled: Boolean = true,
val invocations: Int = 1,
val timeout: Duration? = null,
// provides for concurrent execution of the test case
// only has an effect if invocations > 1
val threads: Int = 1,
val tags: Set<Tag> = emptySet(),
val extensions: List<TestCaseExtension> = emptyList(),
val threads: Int = 1,
val tags: Set<Tag> = emptySet(),
val extensions: List<TestCaseExtension> = emptyList(),
// an issue number, or link to the issue, can be used by plugins
val issue: String? = null) {
val issue: String? = null) {
init {
require(threads > 0) { "Threads must be > 0" }
require(invocations > 0) { "Invocations must be > 0" }
}
}
}

/**
* Returns the timeout for a [TestCase] taking into account global settings.
*/
fun TestCaseConfig.timeout(): Duration = this.timeout ?: Project.timeout()
@@ -100,7 +100,7 @@ class TestCaseExecutor(private val listener: TestEngineListener,
return if (active) executeTest(testCase, context, start) else TestResult.Ignored
}

// exectues the test case or if the test is not active then returns a ignored test result
// exectues the test case or if the test is not active then returns an ignored test result
private suspend fun executeTest(testCase: TestCase, context: TestContext, start: Long): TestResult {
listener.beforeTestCaseExecution(testCase)

@@ -146,10 +146,10 @@ class TestCaseExecutor(private val listener: TestEngineListener,
}

// we schedule a timeout, (if timeout has been configured) which will fail the test with a timed-out status
if (testCase.config.timeout.nano > 0) {
if (testCase.config.timeout().toNanos() > 0) {
scheduler.schedule({
error.compareAndSet(null, TimeoutException("Execution of test took longer than ${testCase.config.timeout}"))
}, testCase.config.timeout.toMillis(), TimeUnit.MILLISECONDS)
error.compareAndSet(null, TimeoutException("Execution of test took longer than ${testCase.config.timeout().toMillis()}ms"))
}, testCase.config.timeout().toNanos(), TimeUnit.NANOSECONDS)
}

supervisorJob.invokeOnCompletion { e ->
@@ -8,7 +8,6 @@ import io.kotlintest.TestCase
import io.kotlintest.TestCaseConfig
import io.kotlintest.TestContext
import io.kotlintest.TestStatus
import io.kotlintest.TestType
import io.kotlintest.milliseconds
import io.kotlintest.runner.jvm.TestCaseExecutor
import io.kotlintest.runner.jvm.TestEngineListener
@@ -237,7 +236,7 @@ class TestCaseExecutorTest : FunSpec() {

then(listener).should().exitTestCase(
argThat { description == Description.spec("wibble") },
argThat { status == TestStatus.Error && this.error?.message == "Execution of test took longer than PT0.1S" }
argThat { status == TestStatus.Error && this.error?.message == "Execution of test took longer than 100ms" }
)
}

@@ -258,11 +257,12 @@ class TestCaseExecutorTest : FunSpec() {
override suspend fun registerTestCase(testCase: TestCase) {}
override fun description(): Description = Description.spec("wibble")
}

executor.execute(testCase, context)

then(listener).should().exitTestCase(
argThat { description == Description.spec("wibble") },
argThat { status == TestStatus.Error && this.error?.message == "Execution of test took longer than PT0.125S" }
argThat { status == TestStatus.Error && this.error?.message == "Execution of test took longer than 125ms" }
)
}

@@ -293,11 +293,11 @@ class TestCaseExecutorTest : FunSpec() {
val listener = mock<TestEngineListener> {}
val executor = TestCaseExecutor(listener, listenerExecutor, scheduler)

val testCase = TestCase.test(Description.spec("wibble"), this@TestCaseExecutorTest, {
val testCase = TestCase.test(Description.spec("wibble"), this@TestCaseExecutorTest) {
while (true) {
"this" shouldBe "that"
}
}).copy(config = TestCaseConfig(true, invocations = 2, threads = 1))
}.copy(config = TestCaseConfig(true, invocations = 2, threads = 1))

val context = object : TestContext(GlobalScope.coroutineContext) {
override suspend fun registerTestCase(testCase: TestCase) {}
@@ -5,29 +5,29 @@ import io.kotlintest.TestCase
import io.kotlintest.shouldBe
import io.kotlintest.specs.FunSpec

class TestCaseTest : FunSpec() {
class TestCasePrefixTest : FunSpec() {

init {

test("test case is focused should return true for top level f: test") {
val test = TestCase.test(Description.spec("f: my test"), this@TestCaseTest) {}
val test = TestCase.test(Description.spec("f: my test"), this@TestCasePrefixTest) {}
test.isFocused() shouldBe true
}

test("test case is focused should return false for top level test without f: prefix") {
val test = TestCase.test(Description.spec("my test"), this@TestCaseTest) {}
val test = TestCase.test(Description.spec("my test"), this@TestCasePrefixTest) {}
test.isFocused() shouldBe false
}

test("is bang should return true for tests with ! prefix") {
val test = TestCase.test(Description.spec("!my test"), this@TestCaseTest) {}
val test = TestCase.test(Description.spec("!my test"), this@TestCasePrefixTest) {}
test.isBang() shouldBe true
}

test("is bang should return false for tests without ! prefix") {
val test = TestCase.test(Description.spec("my test"), this@TestCaseTest) {}
val test = TestCase.test(Description.spec("my test"), this@TestCasePrefixTest) {}
test.isBang() shouldBe false
}
}

}
}
@@ -0,0 +1,45 @@
buildscript {
repositories {
mavenCentral()
mavenLocal()
}
}

plugins {
id 'java'
id 'org.jetbrains.kotlin.jvm'
id 'java-library'
id 'maven-publish'
id 'signing'
}

repositories {
mavenCentral()
mavenLocal()
}

dependencies {
testImplementation project(':kotlintest-core')
testImplementation project(':kotlintest-assertions')
testImplementation project(':kotlintest-runner:kotlintest-runner-junit5')
testImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.11.2'
testImplementation "com.nhaarman:mockito-kotlin:1.6.0"
testImplementation 'org.mockito:mockito-core:2.24.0'
}

test {
useJUnitPlatform()

// show standard out and standard error of the test JVM(s) on the console
testLogging.showStandardStreams = true

// Always run tests, even when nothing changed.
dependsOn 'cleanTest'

testLogging {
events "PASSED", "FAILED", "SKIPPED", "STANDARD_OUT", "STANDARD_ERROR"
exceptionFormat = 'full'
}
}

publish.enabled = false
@@ -0,0 +1,40 @@
package com.sksamuel.kotlintest.timeout

import io.kotlintest.TestCase
import io.kotlintest.TestResult
import io.kotlintest.TestStatus
import io.kotlintest.extensions.SpecLevelExtension
import io.kotlintest.extensions.TestCaseExtension
import io.kotlintest.specs.StringSpec
import kotlinx.coroutines.delay
import java.lang.RuntimeException
import java.time.Duration

class GlobalTimeoutTest : StringSpec() {

init {

"a blocked thread should timeout if global timeout is applied" {
Thread.sleep(2500)
}

"a suspended coroutine should timeout if a global timeout is applied" {
delay(2500)
}

}

override fun extensions(): List<SpecLevelExtension> = listOf(object : TestCaseExtension {
override suspend fun intercept(testCase: TestCase,
execute: suspend (TestCase, suspend (TestResult) -> Unit) -> Unit,
complete: suspend (TestResult) -> Unit) {
execute(testCase) {
when (it.status) {
TestStatus.Failure, TestStatus.Error -> complete(TestResult.success(Duration.ofSeconds(1)))
else -> throw RuntimeException("This should not occur")
}
}
}
})

}
@@ -1,27 +1,23 @@
package com.sksamuel.kotlintest
package com.sksamuel.kotlintest.timeout

import com.nhaarman.mockito_kotlin.argThat
import com.nhaarman.mockito_kotlin.mock
import com.nhaarman.mockito_kotlin.then
import io.kotlintest.Description
import io.kotlintest.Spec
import io.kotlintest.TestCase
import io.kotlintest.TestCaseConfig
import io.kotlintest.TestContext
import io.kotlintest.TestResult
import io.kotlintest.TestStatus
import io.kotlintest.milliseconds
import io.kotlintest.runner.jvm.TestCaseExecutor
import io.kotlintest.runner.jvm.TestEngineListener
import io.kotlintest.shouldBe
import io.kotlintest.specs.FunSpec
import kotlinx.coroutines.GlobalScope
import java.time.Duration
import java.util.concurrent.Executors

@Suppress("BlockingMethodInNonBlockingContext")
class TestCaseExecutorListenerAfterTimeoutTest : FunSpec() {

private val scheduler = Executors.newScheduledThreadPool(1)
class TestCaseTimeoutListenerTest : FunSpec() {

private var listenerRan = false

@@ -35,25 +31,22 @@ class TestCaseExecutorListenerAfterTimeoutTest : FunSpec() {

init {

test("tests which timeout should still run the 'after test' listeners").config {
test("tests which timeout should still run the 'after test' listeners").config(timeout = Duration.ofSeconds(10)) {

val listenerExecutor = Executors.newSingleThreadExecutor()
val listener = mock<TestEngineListener> {}
val scheduler = Executors.newScheduledThreadPool(1)
val executor = TestCaseExecutor(listener, listenerExecutor, scheduler)

val testCase = TestCase.test(Description.spec("wibble"), this@TestCaseExecutorListenerAfterTimeoutTest) {
val testCase = TestCase.test(Description.spec("wibble"), this@TestCaseTimeoutListenerTest) {
Thread.sleep(500)
}.copy(config = TestCaseConfig(true, invocations = 1, threads = 1, timeout = 100.milliseconds))
}.copy(config = TestCaseConfig(true, invocations = 1, threads = 1, timeout = 125.milliseconds))

val context = object : TestContext(GlobalScope.coroutineContext) {
override suspend fun registerTestCase(testCase: TestCase) {}
override fun description(): Description = Description.spec("wibble")
}
executor.execute(testCase, context)

then(listener).should().exitTestCase(
argThat { description == Description.spec("wibble") },
argThat { status == TestStatus.Error && this.error?.message == "Execution of test took longer than PT0.1S" }
)
}
}
}
}

0 comments on commit d04f05b

Please sign in to comment.
You can’t perform that action at this time.