New treating of Overflows

Author: EgorkaKulikov

utbot-framework/src/main/kotlin/org/utbot/engine/ArtificialErrors.kt

@@ -0,0 +1,18 @@
+package org.utbot.engine
+
+/**
+ * Represents an error that may be detected or not
+ * during analysis in accordance with custom settings.
+ *
+ * Usually execution may be continued somehow after such error,
+ * but the result may be different from basic expectations.
+ */
+sealed class ArtificialError(message: String): Error(message)
+
+/**
+ * Represents overflow detection errors in symbolic engine,
+ * if a mode to detect them is turned on.
+ *
+ * See [TraversalContext.intOverflowCheck] for more details.
+ */
+class OverflowDetectionError(message: String): ArtificialError(message)

utbot-framework/src/main/kotlin/org/utbot/engine/DataClasses.kt

@@ -63,11 +63,11 @@ fun explicitThrown(exception: Throwable, addr: UtAddrExpression, inNestedMethod:
 /**
  * Implicitly thrown exception. There are no difference if it happens in nested call or not.
  */
-fun implicitThrown(exception: Throwable, addr: UtAddrExpression, inNestedMethod: Boolean) =
-    SymbolicFailure(symbolic(exception, addr), exception, explicit = false, inNestedMethod = inNestedMethod)
+fun implicitThrown(throwable: Throwable, addr: UtAddrExpression, inNestedMethod: Boolean) =
+    SymbolicFailure(symbolic(throwable, addr), throwable, explicit = false, inNestedMethod = inNestedMethod)
 
-private fun symbolic(exception: Throwable, addr: UtAddrExpression) =
-    objectValue(Scene.v().getRefType(exception.javaClass.canonicalName), addr, ThrowableWrapper(exception))
+private fun symbolic(throwable: Throwable, addr: UtAddrExpression) =
+    objectValue(Scene.v().getRefType(throwable.javaClass.canonicalName), addr, ThrowableWrapper(throwable))
 
 private fun extractConcrete(symbolic: SymbolicValue) =
     (symbolic.asWrapperOrNull as? ThrowableWrapper)?.throwable

utbot-framework/src/main/kotlin/org/utbot/engine/Resolver.kt

@@ -390,10 +390,9 @@ class Resolver(
         return if (explicit) {
             UtExplicitlyThrownException(exception, inNestedMethod)
         } else {
-            when {
-                // TODO SAT-1561
-                exception is ArithmeticException && exception.message?.contains("overflow") == true -> UtOverflowFailure(exception)
-                exception is AccessControlException -> UtSandboxFailure(exception)
+            when (exception) {
+                is OverflowDetectionError -> UtOverflowFailure(exception)
+                is AccessControlException -> UtSandboxFailure(exception)
                 else -> UtImplicitlyThrownException(exception, inNestedMethod)
             }
         }

utbot-framework/src/main/kotlin/org/utbot/engine/Traverser.kt

@@ -2300,6 +2300,10 @@ class Traverser(
         exception: SymbolicFailure,
         conditions: Set<UtBoolExpression>
     ): Boolean {
+        if (exception.concrete is ArtificialError) {
+            return false
+        }
+
         val classId = exception.fold(
             { it.javaClass.id },
             { (exception.symbolic as ObjectValue).type.id }
@@ -3406,7 +3410,7 @@ class Traverser(
         }
 
         if (overflow != null) {
-            implicitlyThrowException(ArithmeticException("${left.type} ${op.symbol} overflow"), setOf(overflow))
+            implicitlyThrowException(OverflowDetectionError("${left.type} ${op.symbol} overflow"), setOf(overflow))
             queuedSymbolicStateUpdates += mkNot(overflow).asHardConstraint()
         }
     }
@@ -3482,13 +3486,13 @@ class Traverser(
     }
 
     private fun TraversalContext.implicitlyThrowException(
-        exception: Exception,
+        throwable: Throwable,
         conditions: Set<UtBoolExpression>,
         softConditions: Set<UtBoolExpression> = emptySet()
     ) {
         if (environment.state.executionStack.last().doesntThrow) return
 
-        val symException = implicitThrown(exception, findNewAddr(), environment.state.isInNestedMethod())
+        val symException = implicitThrown(throwable, findNewAddr(), environment.state.isInNestedMethod())
         if (!traverseCatchBlock(environment.state.stmt, symException, conditions)) {
             environment.state.expectUndefined()
             val nextState = createExceptionStateQueued(

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/Domain.kt

@@ -178,6 +178,7 @@ abstract class TestFramework(
     abstract val mainPackage: String
     abstract val assertionsClass: ClassId
     abstract val arraysAssertionsClass: ClassId
+    abstract val kotlinFailureAssertionsClass: ClassId
     abstract val testAnnotation: String
     abstract val testAnnotationId: ClassId
     abstract val testAnnotationFqn: String
@@ -226,11 +227,19 @@ abstract class TestFramework(
 
     val assertNotEquals by lazy { assertionId("assertNotEquals", objectClassId, objectClassId) }
 
+    val fail by lazy { assertionId("fail", objectClassId) }
+
+    val kotlinFail by lazy { kotlinFailAssertionId("fail", objectClassId) }
+
     protected open fun assertionId(name: String, vararg params: ClassId): MethodId =
         builtinStaticMethodId(assertionsClass, name, voidClassId, *params)
+
     private fun arrayAssertionId(name: String, vararg params: ClassId): MethodId =
         builtinStaticMethodId(arraysAssertionsClass, name, voidClassId, *params)
 
+    private fun kotlinFailAssertionId(name: String, vararg params: ClassId): MethodId =
+        builtinStaticMethodId(kotlinFailureAssertionsClass, name, voidClassId, *params)
+
     abstract fun getRunTestsCommand(
         executionInvoke: String,
         classPath: String,
@@ -272,6 +281,8 @@ object TestNg : TestFramework(id = "TestNG",displayName = "TestNG") {
         simpleName = "ArrayAsserts"
     )
 
+    override val kotlinFailureAssertionsClass = assertionsClass
+
     override val assertBooleanArrayEquals by lazy { assertionId("assertEquals", booleanArrayClassId, booleanArrayClassId) }
 
     val throwingRunnableClassId = BuiltinClassId(
@@ -389,6 +400,7 @@ object Junit4 : TestFramework(id = "JUnit4",displayName = "JUnit 4") {
         simpleName = "Assert"
     )
     override val arraysAssertionsClass = assertionsClass
+    override val kotlinFailureAssertionsClass = assertionsClass
 
     val ignoreAnnotationClassId = with("$JUNIT4_PACKAGE.Ignore") {
         BuiltinClassId(
@@ -485,6 +497,11 @@ object Junit5 : TestFramework(id = "JUnit5", displayName = "JUnit 5") {
 
     override val arraysAssertionsClass = assertionsClass
 
+    override val kotlinFailureAssertionsClass = BuiltinClassId(
+        canonicalName = "org.junit.jupiter.api",
+        simpleName = "Assertions"
+    )
+
     val assertThrows = builtinStaticMethodId(
         classId = assertionsClass,
         name = "assertThrows",

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/models/CgElement.kt

@@ -309,6 +309,7 @@ enum class CgTestMethodType(val displayName: String, val isThrowing: Boolean) {
     PASSED_EXCEPTION(displayName = "Thrown exceptions marked as passed", isThrowing = true),
     FAILING(displayName = "Failing tests (with exceptions)", isThrowing = true),
     TIMEOUT(displayName = "Failing tests (with timeout)", isThrowing = true),
+    ARTIFICIAL(displayName = "Failing tests (with custom exception)", isThrowing = true),
     CRASH(displayName = "Possibly crashing tests", isThrowing = true),
     PARAMETRIZED(displayName = "Parametrized tests", isThrowing = false);
 

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/reports/TestsGenerationReport.kt

@@ -16,6 +16,7 @@ data class TestsGenerationReport(
     var successfulExecutions: MethodGeneratedTests = mutableMapOf(),
     var timeoutExecutions: MethodGeneratedTests = mutableMapOf(),
     var failedExecutions: MethodGeneratedTests = mutableMapOf(),
+    var artificiallyFailedExecutions: MethodGeneratedTests = mutableMapOf(),
     var crashExecutions: MethodGeneratedTests = mutableMapOf(),
     var errors: MutableMap<ExecutableId, ErrorsCount> = mutableMapOf()
 ) {
@@ -59,6 +60,7 @@ data class TestsGenerationReport(
                 when (it.type) {
                     CgTestMethodType.SUCCESSFUL, CgTestMethodType.PASSED_EXCEPTION -> updateExecutions(it, successfulExecutions)
                     CgTestMethodType.FAILING -> updateExecutions(it, failedExecutions)
+                    CgTestMethodType.ARTIFICIAL -> updateExecutions(it, artificiallyFailedExecutions)
                     CgTestMethodType.TIMEOUT -> updateExecutions(it, timeoutExecutions)
                     CgTestMethodType.CRASH -> updateExecutions(it, crashExecutions)
                     CgTestMethodType.PARAMETRIZED -> {

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/services/framework/TestFrameworkManager.kt

@@ -66,6 +66,9 @@ abstract class TestFrameworkManager(val context: CgContext)
     val assertTrue = context.testFramework.assertTrue
     val assertFalse = context.testFramework.assertFalse
 
+    val fail = context.testFramework.fail
+    val kotlinFail = context.testFramework.kotlinFail
+
     val assertArrayEquals = context.testFramework.assertArrayEquals
     val assertBooleanArrayEquals = context.testFramework.assertBooleanArrayEquals
     val assertByteArrayEquals = context.testFramework.assertByteArrayEquals
@@ -173,6 +176,13 @@ abstract class TestFrameworkManager(val context: CgContext)
 
     fun assertBoolean(actual: CgExpression) = assertBoolean(expected = true, actual)
 
+    fun fail(actual: CgExpression) {
+        when (codegenLanguage) {
+            CodegenLanguage.JAVA -> +assertions[fail](actual)
+            CodegenLanguage.KOTLIN -> +assertions[kotlinFail](actual)
+        }
+    }
+
     // Exception expectation differs between test frameworks
     // JUnit4 requires to add a specific argument to the test method annotation
     // JUnit5 requires using method assertThrows()

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt

@@ -4,6 +4,8 @@ import org.utbot.common.PathUtil
 import org.utbot.common.WorkaroundReason
 import org.utbot.common.isStatic
 import org.utbot.common.workaround
+import org.utbot.engine.ArtificialError
+import org.utbot.engine.OverflowDetectionError
 import org.utbot.framework.assemble.assemble
 import org.utbot.framework.codegen.domain.ForceStaticMocking
 import org.utbot.framework.codegen.domain.ParametrizedTestSource
@@ -141,6 +143,7 @@ import org.utbot.framework.codegen.tree.CgTestClassConstructor.CgComponents.getS
 import org.utbot.framework.codegen.tree.CgTestClassConstructor.CgComponents.getTestFrameworkManagerBy
 import org.utbot.framework.codegen.tree.CgTestClassConstructor.CgComponents.getVariableConstructorBy
 import org.utbot.framework.plugin.api.UtExecutionResult
+import org.utbot.framework.plugin.api.UtOverflowFailure
 import org.utbot.framework.plugin.api.UtStreamConsumingFailure
 import org.utbot.framework.plugin.api.util.allSuperTypes
 import org.utbot.framework.plugin.api.util.baseStreamClassId
@@ -350,6 +353,12 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
                 }
                 else -> error("Unexpected crash suite for failing execution with $expectedException exception")
             }
+            ARTIFICIAL -> {
+                methodInvocationBlock()
+
+                val failureMessage = prepareArtificialFailureMessage(executionResult)
+                testFrameworkManager.fail(failureMessage)
+            }
             FAILING -> {
                 writeWarningAboutFailureTest(expectedException)
                 methodInvocationBlock()
@@ -358,6 +367,25 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
         }
     }
 
+    //TODO: ISSUE-1546 introduce some kind of language-specific string interpolation in Codegen
+    private fun prepareArtificialFailureMessage(executionResult: UtExecutionResult): CgVariable {
+        when (executionResult) {
+            is UtOverflowFailure -> {
+                val preparedMethodArguments = methodArguments.joinToString(separator = ", ") {
+                    when (it) {
+                        is CgVariable -> it.name
+                        is CgLiteral -> it.value.toString()
+                        else -> it.toString()
+                    }
+                }
+
+                val failureMessage = "\"Overflow detected in \'${currentExecutable!!.name}\' call with arguments $preparedMethodArguments \""
+                return CgVariable(failureMessage, stringClassId)
+            }
+            else -> error("$executionResult is not supported in artificial errors")
+        }
+    }
+
     private fun constructExceptionProducingBlock(
         exceptionFromAnalysis: Throwable,
         executionResult: UtExecutionResult
@@ -405,6 +433,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
         if (exception is AccessControlException) return false
         // tests with timeout or crash should be processed differently
         if (exception is TimeoutException || exception is ConcreteExecutionFailureException) return false
+        if (exception is OverflowDetectionError) return false
         if (UtSettings.treatAssertAsErrorSuite && exception is AssertionError) return false
 
         val exceptionRequiresAssert = exception !is RuntimeException || runtimeExceptionTestsBehaviour == PASS
@@ -1796,6 +1825,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
                     shouldTestPassWithException(currentExecution, exception) -> PASSED_EXCEPTION
                     shouldTestPassWithTimeoutException(currentExecution, exception) -> TIMEOUT
                     else -> when (exception) {
+                        is ArtificialError -> ARTIFICIAL
                         is ConcreteExecutionFailureException -> CRASH
                         is AccessControlException -> CRASH // exception from sandbox
                         else -> FAILING

utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt

@@ -29,6 +29,7 @@ import org.utbot.framework.plugin.api.UtVoidModel
  * * Regression suite
  * * Error suite (invocations in which implicitly thrown unchecked exceptions reached to the top)
  * * Crash suite (invocations in which the instrumented process crashed or unexpected exception in our code occurred)
+ * * Artificial error suite (invocations in which some custom exception like overflow detection occurred)
  * * Timeout suite
  *
  * We want to minimize tests independently in each of these suites.