forAll displays all errors at once, even when they are too many #929

kotest-assertions/build.gradle.kts

@@ -2,7 +2,6 @@ plugins {
    id("java")
    id("kotlin-multiplatform")
    id("java-library")
-   id("com.adarshr.test-logger")
 }
 
 repositories {

kotest-assertions/src/commonMain/kotlin/io/kotest/assertions/exceptionToMessage.kt

@@ -1,10 +1,15 @@
 package io.kotest.assertions
 
+/**
+ * Returns a string error message from the given throwable.
+ * If the type is an [AssertionError] then the message is taken from the exceptions own message,
+ * otherwise the exception is converted to a string.
+ */
 fun exceptionToMessage(t: Throwable): String =
-  when (t) {
-    is AssertionError -> when (t.message) {
-      null -> t.toString()
-      else -> t.message!!
-    }
-    else -> t.toString()
-  }
+   when (t) {
+      is AssertionError -> when (t.message) {
+         null -> t.toString()
+         else -> t.message!!
+      }
+      else -> t.toString()
+   }

kotest-assertions/src/commonMain/kotlin/io/kotest/assertions/sysprop.kt

@@ -0,0 +1,3 @@
+package io.kotest.assertions
+
+expect fun sysprop(name: String): String?

kotest-assertions/src/commonMain/kotlin/io/kotest/inspectors/error.kt

@@ -0,0 +1,48 @@
+package io.kotest.inspectors
+
+import io.kotest.assertions.Failures
+import io.kotest.assertions.exceptionToMessage
+import io.kotest.assertions.show.show
+import io.kotest.assertions.sysprop
+
+/**
+ * Build assertion error message.
+ *
+ * Show 10 passed and failed results by default. You can change the number of output results by setting the
+ * system property `kotlintest.assertions.output.max=20`.
+ *
+ * E.g.:
+ *
+ * ```
+ *     -Dkotlintest.assertions.output.max=20
+ * ```
+ */
+fun <T> buildAssertionError(msg: String, results: List<ElementResult<T>>): String {
+
+   val maxResults = sysprop("kotlintest.assertions.output.max")?.toInt() ?: 10
+
+   val passed = results.filterIsInstance<ElementPass<T>>()
+   val failed = results.filterIsInstance<ElementFail<T>>()
+
+   val builder = StringBuilder(msg)
+   builder.append("\n\nThe following elements passed:\n")
+   if (passed.isEmpty()) {
+      builder.append("--none--")
+   } else {
+      builder.append(passed.take(maxResults).map { it.t }.joinToString("\n"))
+      if (passed.size > maxResults) {
+         builder.append("\n... and ${passed.size - maxResults} more passed elements")
+      }
+   }
+   builder.append("\n\nThe following elements failed:\n")
+   if (failed.isEmpty()) {
+      builder.append("--none--")
+   } else {
+      builder.append(failed.take(maxResults).joinToString("\n") { it.t.show() + " => " + exceptionToMessage(it.throwable) })
+      if (failed.size > maxResults) {
+         builder.append("\n... and ${failed.size - maxResults} more failed elements")
+      }
+   }
+   throw Failures.failure(builder.toString())
+}
+

kotest-assertions/src/jsMain/kotlin/io/kotest/assertions/sysprop.kt

@@ -0,0 +1,3 @@
+package io.kotest.assertions
+
+actual fun sysprop(name: String): String? = null

kotest-assertions/src/jvmMain/kotlin/io/kotest/assertions/sysprop.kt

@@ -0,0 +1,3 @@
+package io.kotest.assertions
+
+actual fun sysprop(name: String): String? = System.getProperty(name)

kotest-assertions/src/jvmTest/kotlin/com/sksamuel/kotest/inspectors/sizes.kt

@@ -1,9 +1,12 @@
 package com.sksamuel.kotest.inspectors
 
 import io.kotest.assertions.shouldFail
+import io.kotest.assertions.throwables.shouldThrowAny
 import io.kotest.core.spec.CompositeSpec
 import io.kotest.core.spec.style.funSpec
+import io.kotest.extensions.system.withSystemProperty
 import io.kotest.inspectors.forAll
+import io.kotest.matchers.ints.shouldBeLessThanOrEqual
 import io.kotest.matchers.shouldBe
 
 val tests = funSpec {
@@ -14,6 +17,61 @@ val tests = funSpec {
          }
       }
    }
+   test("passed results are truncated when passed list length is over 10") {
+      shouldThrowAny {
+         (1..13).toList().forAll {
+            it.shouldBeLessThanOrEqual(12)
+         }
+      }.message shouldBe "12 elements passed but expected 13\n" +
+         "\n" +
+         "The following elements passed:\n" +
+         "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n" +
+         "... and 2 more passed elements\n" +
+         "\n" +
+         "The following elements failed:\n" +
+         "13 => 13 should be <= 12"
+   }
+   test("failed results are truncated when failed array size is over 10") {
+      shouldThrowAny {
+         arrayOf(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12).forAll {
+            if (System.currentTimeMillis() > 0) throw NullPointerException()
+         }
+      }.message shouldBe "0 elements passed but expected 12\n" +
+         "\n" +
+         "The following elements passed:\n" +
+         "--none--\n" +
+         "\n" +
+         "The following elements failed:\n" +
+         "1 => java.lang.NullPointerException\n" +
+         "2 => java.lang.NullPointerException\n" +
+         "3 => java.lang.NullPointerException\n" +
+         "4 => java.lang.NullPointerException\n" +
+         "5 => java.lang.NullPointerException\n" +
+         "6 => java.lang.NullPointerException\n" +
+         "7 => java.lang.NullPointerException\n" +
+         "8 => java.lang.NullPointerException\n" +
+         "9 => java.lang.NullPointerException\n" +
+         "10 => java.lang.NullPointerException\n" +
+         "... and 2 more failed elements"
+   }
+   test("max results is controllable by sys prop") {
+      withSystemProperty("kotlintest.assertions.output.max", "3") {
+         shouldThrowAny {
+            arrayOf(1, 2, 3, 4, 5).forAll {
+               if (System.currentTimeMillis() > 0) throw NullPointerException()
+            }
+         }.message shouldBe "0 elements passed but expected 5\n" +
+            "\n" +
+            "The following elements passed:\n" +
+            "--none--\n" +
+            "\n" +
+            "The following elements failed:\n" +
+            "1 => java.lang.NullPointerException\n" +
+            "2 => java.lang.NullPointerException\n" +
+            "3 => java.lang.NullPointerException\n" +
+            "... and 2 more failed elements"
+      }
+   }
 }
 
 class InspectorTests : CompositeSpec(tests)

kotest-core/src/commonMain/kotlin/io/kotest/core/extensions/SystemPropertyTagExtension.kt

@@ -1,10 +1,11 @@
 package io.kotest.core.extensions
 
+import io.kotest.assertions.sysprop
 import io.kotest.core.StringTag
 import io.kotest.core.Tag
 import io.kotest.core.Tags
-import io.kotest.core.sysprop
 import io.kotest.fp.getOrElse
+import io.kotest.fp.toOption
 
 /**
  * This [TagExtension] includes and excludes tags using the system properties
@@ -17,7 +18,7 @@ object SystemPropertyTagExtension : TagExtension {
    override fun tags(): Tags {
 
       fun readTagsProperty(name: String): List<Tag> =
-         sysprop(name).getOrElse("").split(',').filter { it.isNotBlank() }.map {
+         sysprop(name).toOption().getOrElse("").split(',').filter { it.isNotBlank() }.map {
             StringTag(
                it.trim()
             )

kotest-core/src/commonMain/kotlin/io/kotest/core/test/active.kt

@@ -1,9 +1,10 @@
 package io.kotest.core.test
 
+import io.kotest.assertions.sysprop
 import io.kotest.core.config.Project
 import io.kotest.core.filters.TestFilterResult
 import io.kotest.core.spec.focusTests
-import io.kotest.core.sysprop
+import io.kotest.fp.toOption
 
 /**
  * Returns true if the given [TestCase] is active.
@@ -18,7 +19,7 @@ import io.kotest.core.sysprop
  * - Include tag filters have been specified and this test either has no tags, or does not have a tag that is one of those included
  * - The test is filtered out via a [TestCaseFilter]
  *
- * Note: tags are defined either through [TestCaseConfig] or in the [SpecConfiguration] dsl.
+ * Note: tags are defined either through [TestCaseConfig] or in the [Spec] dsl.
  *
  * Note2: Focused tests will override any settings here.
  *
@@ -27,7 +28,7 @@ fun TestCase.isActive(): Boolean {
    val focused = isFocused() && isTopLevel()
    val hasFocused = spec.focusTests().isNotEmpty()
    val enabledInConfig = config.enabled && config.enabledIf()
-   val bangEnabled = sysprop("kotest.bang.disable").isEmpty()
+   val bangEnabled = sysprop("kotest.bang.disable").toOption().isEmpty()
    val disabledViaBang = isBang() && bangEnabled
    val activeViaTags = Project.tags().isActive(config.tags + spec.tags() + spec._tags)
    val filtered = Project.testCaseFilters()