Implemented warn on verification error

Added property to suppress task execution failure if there has been a violation of the verification rules

Resolves #339
PR #577

kover-features-jvm/build.gradle.kts

@@ -28,13 +28,10 @@ extensions.configure<Kover_publishing_conventions_gradle.KoverPublicationExtensi
     description.set("Implementation of calling the main features of Kover programmatically")
 }
 
-java {
-    sourceCompatibility = JavaVersion.VERSION_1_8
-    targetCompatibility = JavaVersion.VERSION_1_8
-}
-
-tasks.compileJava {
-    options.release.set(8)
+kotlin {
+    jvmToolchain {
+        languageVersion.set(JavaLanguageVersion.of(8))
+    }
 }
 
 // Workaround:
@@ -48,7 +45,7 @@ afterEvaluate {
             jvmTarget.set(JvmTarget.JVM_1_8)
             languageVersion.set(KotlinVersion.KOTLIN_1_5)
             apiVersion.set(KotlinVersion.KOTLIN_1_5)
-            freeCompilerArgs.addAll("-Xsuppress-version-warnings", "-Xjdk-release=1.8")
+            freeCompilerArgs.addAll("-Xsuppress-version-warnings")
         }
     }
 }

kover-gradle-plugin/api/kover-gradle-plugin.api

@@ -230,6 +230,7 @@ public abstract interface class kotlinx/kover/gradle/plugin/dsl/KoverVariantSour
 }
 
 public abstract interface class kotlinx/kover/gradle/plugin/dsl/KoverVerificationRulesConfig {
+	public abstract fun getWarningInsteadOfFailure ()Lorg/gradle/api/provider/Property;
 	public abstract fun rule (Ljava/lang/String;Lorg/gradle/api/Action;)V
 	public abstract fun rule (Lorg/gradle/api/Action;)V
 }

kover-gradle-plugin/docs/configuring.md

@@ -29,11 +29,14 @@ koverReport {
 
     // verification rules for verification task
     verify {
+        // fail on verification error
+        warningInsteadOfFailure = false
+
         // add common verification rule
         rule {
             // check this rule during verification 
             disabled = false
-
+          
             // specify the code unit for which coverage will be aggregated 
             groupBy = kotlinx.kover.gradle.plugin.dsl.GroupingEntityType.APPLICATION
 
@@ -162,6 +165,9 @@ koverReport {
         verify {
             //  verify coverage when running the `check` task
             onCheck = true
+
+            // fail on verification error
+            warningInsteadOfFailure = false
         }
 
         // configure coverage logging
@@ -222,6 +228,9 @@ koverReport {
 
     // verification rules for verification tasks in all variants
     verify {
+        // fail on verification error
+        warningInsteadOfFailure = false
+
         // add common verification rule
         rule {
             // check this rule during verification 
@@ -355,6 +364,10 @@ koverReport {
             //  verify coverage when running the `check` task
             onCheck = true
 
+
+            // fail on verification error
+            warningInsteadOfFailure = false
+
             // add verification rule
             rule {
                 // check this rule during verification 

kover-gradle-plugin/src/functionalTest/kotlin/kotlinx/kover/gradle/plugin/test/functional/cases/BuildCacheRelocationTests.kt

@@ -59,7 +59,9 @@ class BuildCacheRelocationTests {
             assertEquals("FROM-CACHE", result2.taskOutcome(":koverXmlReport"))
             assertEquals("FROM-CACHE", result2.taskOutcome(":koverHtmlReport"))
             assertEquals("FROM-CACHE", result2.taskOutcome(":koverBinaryReport"))
-            assertEquals("FROM-CACHE", result2.taskOutcome(":koverVerify"))
+            assertEquals("FROM-CACHE", result2.taskOutcome(":koverCachedVerify"))
+            // should always be executed
+            assertEquals("SUCCESS", result2.taskOutcome(":koverVerify"))
         } catch (e: Throwable) {
             throw AssertionError("Build log \n${result2.output}",e)
         }

kover-gradle-plugin/src/functionalTest/kotlin/kotlinx/kover/gradle/plugin/test/functional/cases/VerificationTests.kt

@@ -175,6 +175,71 @@ Rule violated: lines missed count for package 'org.jetbrains.kover.test.function
         }
     }
 
+    @SlicedGeneratedTest(allLanguages = true, allTools = true)
+    fun BuildConfigurator.testRootRulesWarnOnFail() {
+        addProjectWithKover {
+            sourcesFrom("simple")
+
+            kover {
+                reports {
+                    verify {
+                        warningInsteadOfFailure.set(true)
+                        rule("root rule") {
+                            bound {
+                                minValue.set(99)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        run("koverVerify", errorExpected = false) {
+            taskOutput(":koverVerify") {
+                match {
+                    assertContains("Kover Verification Error")
+                    assertKoverContains("Rule 'root rule' violated: lines covered percentage is *, but expected minimum is 99\n")
+                    assertJaCoCoContains("Rule violated: lines covered percentage is *, but expected minimum is 99.0000\n")
+                }
+            }
+        }
+    }
+
+    @SlicedGeneratedTest(allLanguages = true, allTools = true)
+    fun BuildConfigurator.testRootRulesOverrideWarnOnFail() {
+        addProjectWithKover {
+            sourcesFrom("simple")
+
+            kover {
+                reports {
+                    verify {
+                        warningInsteadOfFailure.set(false)
+                    }
+                    total {
+                        verify {
+                            warningInsteadOfFailure.set(true)
+                            rule("root rule") {
+                                bound {
+                                    minValue.set(99)
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        run("koverVerify", errorExpected = false) {
+            taskOutput(":koverVerify") {
+                match {
+                    assertContains("Kover Verification Error")
+                    assertKoverContains("Rule 'root rule' violated: lines covered percentage is *, but expected minimum is 99\n")
+                    assertJaCoCoContains("Rule violated: lines covered percentage is *, but expected minimum is 99.0000\n")
+                }
+            }
+        }
+    }
+
     @SlicedGeneratedTest(allLanguages = true, allTools = true)
     fun BuildConfigurator.testRootRulesOverride() {
         addProjectWithKover {

kover-gradle-plugin/src/functionalTest/kotlin/kotlinx/kover/gradle/plugin/test/functional/framework/checker/Checker.kt

@@ -117,6 +117,10 @@ private class CheckerContextImpl(
         VerifyReportCheckerImpl(this, verificationResultFile.readText()).checker()
     }
 
+    override fun String.match(matcher: TextMatcher.() -> Unit) {
+        TextMatcherImpl(this@CheckerContextImpl, this).matcher()
+    }
+
     override fun checkDefaultBinReport(mustExist: Boolean) {
         if (mustExist) {
             file(defaultBinReport) {
@@ -416,6 +420,31 @@ private class VerifyReportCheckerImpl(val context: CheckerContextImpl, val conte
     }
 }
 
+private class TextMatcherImpl(val context: CheckerContextImpl, val content: String) : TextMatcher {
+    override fun assertContains(expected: String) {
+        val regex = KoverFeatures.koverWildcardToRegex(expected).toRegex()
+        if (!content.contains(regex)) {
+            throw AssertionError("Unexpected text.\n\tActual\n[\n$content\n]\nExpected regex\n[\n$expected\n]")
+        }
+    }
+
+    override fun assertKoverContains(expected: String) {
+        if (context.project.toolVariant.vendor != CoverageToolVendor.KOVER) return
+        val regex = KoverFeatures.koverWildcardToRegex(expected).toRegex()
+        if (!content.contains(regex)) {
+            throw AssertionError("Unexpected text for Kover Tool.\n\tActual\n[\n$content\n]\nExpected regex\n[\n$expected\n]")
+        }
+    }
+
+    override fun assertJaCoCoContains(expected: String) {
+        if (context.project.toolVariant.vendor != CoverageToolVendor.JACOCO) return
+        val regex = KoverFeatures.koverWildcardToRegex(expected).toRegex()
+        if (!content.contains(regex)) {
+            throw AssertionError("Unexpected text for JaCoCo Tool.\n\tActual\n[\n$content\n]\nExpected regex\n[\n$expected\n]")
+        }
+    }
+}
+
 private fun Element.filter(tag: String, attributeName: String, attributeValue: String): Element? {
     val elements = getElementsByTagName(tag)
     for (i in 0 until elements.length) {

kover-gradle-plugin/src/functionalTest/kotlin/kotlinx/kover/gradle/plugin/test/functional/framework/checker/CheckerTypes.kt

@@ -33,6 +33,8 @@ internal interface CheckerContext {
 
     fun verification(checker: VerifyReportChecker.() -> Unit)
 
+    fun String.match(matcher: TextMatcher.() -> Unit)
+
     val defaultBinReport: String
 
     fun checkXmlReport(variantName: String = "", mustExist: Boolean = true)
@@ -89,6 +91,12 @@ internal interface VerifyReportChecker {
     fun assertJaCoCoResult(expected: String)
 }
 
+internal interface TextMatcher {
+    fun assertContains(expected: String)
+    fun assertKoverContains(expected: String)
+    fun assertJaCoCoContains(expected: String)
+}
+
 internal interface XmlReportChecker {
     fun classCounter(className: String, type: String = "INSTRUCTION"): Counter
     fun methodCounter(className: String, methodName: String, type: String = "INSTRUCTION"): Counter

kover-gradle-plugin/src/main/kotlin/kotlinx/kover/gradle/plugin/appliers/tasks/VariantReportsSet.kt

@@ -35,7 +35,7 @@ internal class VariantReportsSet(
     private val htmlTask: TaskProvider<KoverHtmlTask>
     private val xmlTask: TaskProvider<KoverXmlTask>
     private val binTask: TaskProvider<KoverBinaryTask>
-    private val verifyTask: TaskProvider<KoverVerifyTask>
+    private val doVerifyTask: TaskProvider<KoverDoVerifyTask>
     private val logTask: TaskProvider<KoverFormatCoverageTask>
 
     init {
@@ -52,10 +52,12 @@ internal class VariantReportsSet(
             "Task to generate binary coverage report in IntelliJ format for ${variantSuffix()}"
         )
 
-        verifyTask = project.tasks.createReportTask<KoverVerifyTask>(
-            verifyTaskName(variantName),
+        doVerifyTask = project.tasks.createReportTask<KoverDoVerifyTask>(
+            verifyCachedTaskName(variantName),
             "Task to validate coverage bounding rules for ${variantSuffix()}"
         )
+        val verifyTask = project.tasks.register<KoverVerifyTask>(verifyTaskName(variantName))
+
         logTask = project.tasks.createReportTask<KoverFormatCoverageTask>(
             logTaskName(variantName),
             "Task to print coverage to log for ${variantSuffix()}"
@@ -93,20 +95,37 @@ internal class VariantReportsSet(
             if (run) listOf(binTask) else emptyList()
         }
 
-        verifyTask.configure {
+
+        doVerifyTask.configure {
             val resultRules = config.verify.rules
             val converted = resultRules.map { rules -> rules.map { it.convert() } }
 
+            filters.set((config.filters).convert())
+            rules.addAll(converted)
+
             // path can't be changed
             resultFile.convention(project.layout.buildDirectory.file(verificationErrorsPath(variantName)))
 
-            filters.set((config.filters).convert())
-            rules.addAll(converted)
+            description = "Cacheable task for performing verification for ${variantSuffix()}"
+        }
+        verifyTask.configure {
+            warningInsteadOfFailure.convention(config.verify.warningInsteadOfFailure)
+            errorFile.convention(doVerifyTask.flatMap { it.resultFile })
 
             shouldRunAfter(htmlTask)
             shouldRunAfter(xmlTask)
             shouldRunAfter(binTask)
             shouldRunAfter(logTask)
+
+            dependsOn(doVerifyTask)
+
+            group = LifecycleBasePlugin.VERIFICATION_GROUP
+
+            // always execute
+            outputs.upToDateWhen { false }
+
+            val koverDisabledProvider = koverDisabled
+            onlyIf { !koverDisabledProvider.get() }
         }
         runOnCheck += config.verify.onCheck.map { run ->
             if (run) listOf(verifyTask) else emptyList()
@@ -117,6 +136,9 @@ internal class VariantReportsSet(
             onlyIf {
                 fileWithMessage.asFile.get().exists()
             }
+
+            // always execute
+            outputs.upToDateWhen { false }
         }
 
         logTask.configure {
@@ -146,7 +168,7 @@ internal class VariantReportsSet(
         htmlTask.assign(variant)
         xmlTask.assign(variant)
         binTask.assign(variant)
-        verifyTask.assign(variant)
+        doVerifyTask.assign(variant)
         logTask.assign(variant)
     }
 

kover-gradle-plugin/src/main/kotlin/kotlinx/kover/gradle/plugin/commons/Naming.kt

@@ -84,6 +84,10 @@ internal fun xmlReportTaskName(variant: String) = "$XML_REPORT_NAME${variant.cap
  */
 internal fun binaryReportTaskName(variant: String) = "$BINARY_REPORT_NAME${variant.capitalized()}"
 
+/**
+ * Name for cached verifying task for specified report namespace.
+ */
+internal fun verifyCachedTaskName(variant: String) = "koverCachedVerify${variant.capitalized()}"
 /**
  * Name for verifying task for specified report namespace.
  */

kover-gradle-plugin/src/main/kotlin/kotlinx/kover/gradle/plugin/dsl/KoverReportsConfig.kt

@@ -103,6 +103,9 @@ public interface KoverReportsConfig {
      *      rule("custom rule name") {
      *          // named verification rule
      *      }
+     *
+     *      // fail on verification error
+     *      warningInsteadOfFailure = false
      *  }
      * ```
      */
@@ -313,6 +316,9 @@ public interface KoverReportSetConfig {
      *      rule("Custom Name") {
      *          // ...
      *      }
+     *
+     *      // fail on verification error
+     *      warningInsteadOfFailure = false
      *  }
      * ```
      */
@@ -884,6 +890,9 @@ public interface KoverBinaryTaskConfig {
  *      rule("Custom Name") {
  *          // ...
  *      }
+ *
+ *      // fail on verification error
+ *      warningInsteadOfFailure = false
  *  }
  * ```
  */
@@ -910,6 +919,9 @@ public interface KoverVerifyTaskConfig: KoverVerificationRulesConfig {
  *      rule("custom rule name") {
  *          // named verification rule
  *      }
+ *
+ *      // fail on verification error
+ *      warningInsteadOfFailure = false
  *  }
  * ```
  */
@@ -926,6 +938,15 @@ public interface KoverVerificationRulesConfig {
      * The name will be displayed in case of a verification error if Kover Tool was used.
      */
     public fun rule(name: String, config: Action<KoverVerifyRule>)
+
+    /**
+     * In case of a verification error, print a message to the log with the warn level instead of the Gradle task execution error.
+     *
+     * Gradle task error if `false`, warn message if `true`.
+     *
+     * `false` by default.
+     */
+    public val warningInsteadOfFailure: Property<Boolean>
 }
 
 /**