diff --git a/utbot-framework-test/src/test/kotlin/org/utbot/examples/math/OverflowAsErrorTest.kt b/utbot-framework-test/src/test/kotlin/org/utbot/examples/math/OverflowAsErrorTest.kt index ad99f2e501..1f6b3f468b 100644 --- a/utbot-framework-test/src/test/kotlin/org/utbot/examples/math/OverflowAsErrorTest.kt +++ b/utbot-framework-test/src/test/kotlin/org/utbot/examples/math/OverflowAsErrorTest.kt @@ -2,6 +2,7 @@ package org.utbot.examples.math import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test +import org.utbot.engine.OverflowDetectionError import org.utbot.examples.algorithms.Sort import org.utbot.framework.plugin.api.CodegenLanguage import org.utbot.testcheckers.eq @@ -31,8 +32,8 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::intOverflow, eq(5), - { x, _, r -> x * x * x <= 0 && x > 0 && r.isException() }, // through overflow - { x, _, r -> x * x * x <= 0 && x > 0 && r.isException() }, // through overflow (2nd '*') + { x, _, r -> x * x * x <= 0 && x > 0 && r.isException() }, // through overflow + { x, _, r -> x * x * x <= 0 && x > 0 && r.isException() }, // through overflow (2nd '*') { x, _, r -> x * x * x >= 0 && x >= 0 && r.getOrNull() == 0 }, { x, y, r -> x * x * x > 0 && x > 0 && y == 10 && r.getOrNull() == 1 }, { x, y, r -> x * x * x > 0 && x > 0 && y != 10 && r.getOrNull() == 0 }, @@ -47,11 +48,11 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::byteAddOverflow, eq(2), - { _, _, r -> !r.isException() }, + { _, _, r -> !r.isException() }, { x, y, r -> val negOverflow = ((x + y).toByte() >= 0 && x < 0 && y < 0) val posOverflow = ((x + y).toByte() <= 0 && x > 0 && y > 0) - (negOverflow || posOverflow) && r.isException() + (negOverflow || posOverflow) && r.isException() }, // through overflow ) } @@ -63,11 +64,11 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::byteSubOverflow, eq(2), - { _, _, r -> !r.isException() }, + { _, _, r -> !r.isException() }, { x, y, r -> val negOverflow = ((x - y).toByte() >= 0 && x < 0 && y > 0) val posOverflow = ((x - y).toByte() <= 0 && x > 0 && y < 0) - (negOverflow || posOverflow) && r.isException() + (negOverflow || posOverflow) && r.isException() }, // through overflow ) } @@ -79,8 +80,8 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::byteMulOverflow, eq(2), - { _, _, r -> !r.isException() }, - { _, _, r -> r.isException() }, // through overflow + { _, _, r -> !r.isException() }, + { _, _, r -> r.isException() }, // through overflow ) } } @@ -91,11 +92,11 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::shortAddOverflow, eq(2), - { _, _, r -> !r.isException() }, + { _, _, r -> !r.isException() }, { x, y, r -> val negOverflow = ((x + y).toShort() >= 0 && x < 0 && y < 0) val posOverflow = ((x + y).toShort() <= 0 && x > 0 && y > 0) - (negOverflow || posOverflow) && r.isException() + (negOverflow || posOverflow) && r.isException() }, // through overflow ) } @@ -107,11 +108,11 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::shortSubOverflow, eq(2), - { _, _, r -> !r.isException() }, + { _, _, r -> !r.isException() }, { x, y, r -> val negOverflow = ((x - y).toShort() >= 0 && x < 0 && y > 0) val posOverflow = ((x - y).toShort() <= 0 && x > 0 && y < 0) - (negOverflow || posOverflow) && r.isException() + (negOverflow || posOverflow) && r.isException() }, // through overflow ) } @@ -123,8 +124,8 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::shortMulOverflow, eq(2), - { _, _, r -> !r.isException() }, - { _, _, r -> r.isException() }, // through overflow + { _, _, r -> !r.isException() }, + { _, _, r -> r.isException() }, // through overflow ) } } @@ -135,11 +136,11 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::intAddOverflow, eq(2), - { _, _, r -> !r.isException() }, + { _, _, r -> !r.isException() }, { x, y, r -> val negOverflow = ((x + y) >= 0 && x < 0 && y < 0) val posOverflow = ((x + y) <= 0 && x > 0 && y > 0) - (negOverflow || posOverflow) && r.isException() + (negOverflow || posOverflow) && r.isException() }, // through overflow ) } @@ -151,11 +152,11 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::intSubOverflow, eq(2), - { _, _, r -> !r.isException() }, + { _, _, r -> !r.isException() }, { x, y, r -> val negOverflow = ((x - y) >= 0 && x < 0 && y > 0) val posOverflow = ((x - y) <= 0 && x > 0 && y < 0) - (negOverflow || posOverflow) && r.isException() + (negOverflow || posOverflow) && r.isException() }, // through overflow ) } @@ -171,8 +172,8 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::intMulOverflow, eq(2), - { _, _, r -> !r.isException() }, - { _, _, r -> r.isException() }, // through overflow + { _, _, r -> !r.isException() }, + { _, _, r -> r.isException() }, // through overflow ) } } @@ -184,11 +185,11 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::longAddOverflow, eq(2), - { _, _, r -> !r.isException() }, + { _, _, r -> !r.isException() }, { x, y, r -> val negOverflow = ((x + y) >= 0 && x < 0 && y < 0) val posOverflow = ((x + y) <= 0 && x > 0 && y > 0) - (negOverflow || posOverflow) && r.isException() + (negOverflow || posOverflow) && r.isException() }, // through overflow ) } @@ -200,11 +201,11 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::longSubOverflow, eq(2), - { _, _, r -> !r.isException() }, + { _, _, r -> !r.isException() }, { x, y, r -> val negOverflow = ((x - y) >= 0 && x < 0 && y > 0) val posOverflow = ((x - y) <= 0 && x > 0 && y < 0) - (negOverflow || posOverflow) && r.isException() + (negOverflow || posOverflow) && r.isException() }, // through overflow ) } @@ -221,8 +222,8 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::longMulOverflow, eq(2), - { _, _, r -> !r.isException() }, - { _, _, r -> r.isException() }, // through overflow + { _, _, r -> !r.isException() }, + { _, _, r -> r.isException() }, // through overflow ) } } @@ -234,8 +235,8 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( OverflowExamples::incOverflow, eq(2), - { _, r -> !r.isException() }, - { _, r -> r.isException() }, // through overflow + { _, r -> !r.isException() }, + { _, r -> r.isException() }, // through overflow ) } } @@ -251,8 +252,8 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( // Can't use abs(x) below, because abs(Int.MIN_VALUE) == Int.MIN_VALUE. // (Int.MAX_VALUE shr 16) is the border of square overflow and cube overflow. // Int.MAX_VALUE.toDouble().pow(1/3.toDouble()) - { x, r -> (x > -sqrtIntMax && x < sqrtIntMax) && r.isException() }, // through overflow - { x, r -> (x <= -sqrtIntMax || x >= sqrtIntMax) && r.isException() }, // through overflow + { x, r -> (x > -sqrtIntMax && x < sqrtIntMax) && r.isException() }, // through overflow + { x, r -> (x <= -sqrtIntMax || x >= sqrtIntMax) && r.isException() }, // through overflow ) } } @@ -265,8 +266,8 @@ internal class OverflowAsErrorTest : UtValueTestCaseChecker( checkWithException( Sort::quickSort, ignoreExecutionsNumber, - { _, _, _, r -> !r.isException() }, - { _, _, _, r -> r.isException() }, // through overflow + { _, _, _, r -> !r.isException() }, + { _, _, _, r -> r.isException() }, // through overflow ) } } diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/ArtificialErrors.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/ArtificialErrors.kt new file mode 100644 index 0000000000..4ab9daa425 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/ArtificialErrors.kt @@ -0,0 +1,18 @@ +package org.utbot.engine + +/** + * Represents an error that may be detected or not + * during analysis in accordance with custom settings. + * + * Usually execution may be continued somehow after such error, + * but the result may be different from basic expectations. + */ +sealed class ArtificialError(message: String): Error(message) + +/** + * Represents overflow detection errors in symbolic engine, + * if a mode to detect them is turned on. + * + * See [TraversalContext.intOverflowCheck] for more details. + */ +class OverflowDetectionError(message: String): ArtificialError(message) \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/DataClasses.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/DataClasses.kt index 367386ee54..46e7a49792 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/DataClasses.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/DataClasses.kt @@ -63,11 +63,11 @@ fun explicitThrown(exception: Throwable, addr: UtAddrExpression, inNestedMethod: /** * Implicitly thrown exception. There are no difference if it happens in nested call or not. */ -fun implicitThrown(exception: Throwable, addr: UtAddrExpression, inNestedMethod: Boolean) = - SymbolicFailure(symbolic(exception, addr), exception, explicit = false, inNestedMethod = inNestedMethod) +fun implicitThrown(throwable: Throwable, addr: UtAddrExpression, inNestedMethod: Boolean) = + SymbolicFailure(symbolic(throwable, addr), throwable, explicit = false, inNestedMethod = inNestedMethod) -private fun symbolic(exception: Throwable, addr: UtAddrExpression) = - objectValue(Scene.v().getRefType(exception.javaClass.canonicalName), addr, ThrowableWrapper(exception)) +private fun symbolic(throwable: Throwable, addr: UtAddrExpression) = + objectValue(Scene.v().getRefType(throwable.javaClass.canonicalName), addr, ThrowableWrapper(throwable)) private fun extractConcrete(symbolic: SymbolicValue) = (symbolic.asWrapperOrNull as? ThrowableWrapper)?.throwable diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/Resolver.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/Resolver.kt index d06c9c78be..807ff3a0ca 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/Resolver.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/Resolver.kt @@ -390,10 +390,9 @@ class Resolver( return if (explicit) { UtExplicitlyThrownException(exception, inNestedMethod) } else { - when { - // TODO SAT-1561 - exception is ArithmeticException && exception.message?.contains("overflow") == true -> UtOverflowFailure(exception) - exception is AccessControlException -> UtSandboxFailure(exception) + when (exception) { + is OverflowDetectionError -> UtOverflowFailure(exception) + is AccessControlException -> UtSandboxFailure(exception) else -> UtImplicitlyThrownException(exception, inNestedMethod) } } diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/Traverser.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/Traverser.kt index 179230cc62..59da8bf29d 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/Traverser.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/Traverser.kt @@ -2300,6 +2300,10 @@ class Traverser( exception: SymbolicFailure, conditions: Set ): Boolean { + if (exception.concrete is ArtificialError) { + return false + } + val classId = exception.fold( { it.javaClass.id }, { (exception.symbolic as ObjectValue).type.id } @@ -3406,7 +3410,7 @@ class Traverser( } if (overflow != null) { - implicitlyThrowException(ArithmeticException("${left.type} ${op.symbol} overflow"), setOf(overflow)) + implicitlyThrowException(OverflowDetectionError("${left.type} ${op.symbol} overflow"), setOf(overflow)) queuedSymbolicStateUpdates += mkNot(overflow).asHardConstraint() } } @@ -3482,13 +3486,13 @@ class Traverser( } private fun TraversalContext.implicitlyThrowException( - exception: Exception, + throwable: Throwable, conditions: Set, softConditions: Set = emptySet() ) { if (environment.state.executionStack.last().doesntThrow) return - val symException = implicitThrown(exception, findNewAddr(), environment.state.isInNestedMethod()) + val symException = implicitThrown(throwable, findNewAddr(), environment.state.isInNestedMethod()) if (!traverseCatchBlock(environment.state.stmt, symException, conditions)) { environment.state.expectUndefined() val nextState = createExceptionStateQueued( diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/Domain.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/Domain.kt index 929b647e38..86c0768626 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/Domain.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/Domain.kt @@ -178,6 +178,7 @@ abstract class TestFramework( abstract val mainPackage: String abstract val assertionsClass: ClassId abstract val arraysAssertionsClass: ClassId + abstract val kotlinFailureAssertionsClass: ClassId abstract val testAnnotation: String abstract val testAnnotationId: ClassId abstract val testAnnotationFqn: String @@ -226,11 +227,19 @@ abstract class TestFramework( val assertNotEquals by lazy { assertionId("assertNotEquals", objectClassId, objectClassId) } + val fail by lazy { assertionId("fail", objectClassId) } + + val kotlinFail by lazy { kotlinFailAssertionId("fail", objectClassId) } + protected open fun assertionId(name: String, vararg params: ClassId): MethodId = builtinStaticMethodId(assertionsClass, name, voidClassId, *params) + private fun arrayAssertionId(name: String, vararg params: ClassId): MethodId = builtinStaticMethodId(arraysAssertionsClass, name, voidClassId, *params) + private fun kotlinFailAssertionId(name: String, vararg params: ClassId): MethodId = + builtinStaticMethodId(kotlinFailureAssertionsClass, name, voidClassId, *params) + abstract fun getRunTestsCommand( executionInvoke: String, classPath: String, @@ -272,6 +281,8 @@ object TestNg : TestFramework(id = "TestNG",displayName = "TestNG") { simpleName = "ArrayAsserts" ) + override val kotlinFailureAssertionsClass = assertionsClass + override val assertBooleanArrayEquals by lazy { assertionId("assertEquals", booleanArrayClassId, booleanArrayClassId) } val throwingRunnableClassId = BuiltinClassId( @@ -389,6 +400,7 @@ object Junit4 : TestFramework(id = "JUnit4",displayName = "JUnit 4") { simpleName = "Assert" ) override val arraysAssertionsClass = assertionsClass + override val kotlinFailureAssertionsClass = assertionsClass val ignoreAnnotationClassId = with("$JUNIT4_PACKAGE.Ignore") { BuiltinClassId( @@ -485,6 +497,11 @@ object Junit5 : TestFramework(id = "JUnit5", displayName = "JUnit 5") { override val arraysAssertionsClass = assertionsClass + override val kotlinFailureAssertionsClass = BuiltinClassId( + canonicalName = "org.junit.jupiter.api", + simpleName = "Assertions" + ) + val assertThrows = builtinStaticMethodId( classId = assertionsClass, name = "assertThrows", diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/models/CgElement.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/models/CgElement.kt index ebcda00076..d0e72fcea8 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/models/CgElement.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/models/CgElement.kt @@ -309,6 +309,7 @@ enum class CgTestMethodType(val displayName: String, val isThrowing: Boolean) { PASSED_EXCEPTION(displayName = "Thrown exceptions marked as passed", isThrowing = true), FAILING(displayName = "Failing tests (with exceptions)", isThrowing = true), TIMEOUT(displayName = "Failing tests (with timeout)", isThrowing = true), + ARTIFICIAL(displayName = "Failing tests (with custom exception)", isThrowing = true), CRASH(displayName = "Possibly crashing tests", isThrowing = true), PARAMETRIZED(displayName = "Parametrized tests", isThrowing = false); diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/reports/TestsGenerationReport.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/reports/TestsGenerationReport.kt index bf6f3891dd..1f222cf94b 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/reports/TestsGenerationReport.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/reports/TestsGenerationReport.kt @@ -16,6 +16,7 @@ data class TestsGenerationReport( var successfulExecutions: MethodGeneratedTests = mutableMapOf(), var timeoutExecutions: MethodGeneratedTests = mutableMapOf(), var failedExecutions: MethodGeneratedTests = mutableMapOf(), + var artificiallyFailedExecutions: MethodGeneratedTests = mutableMapOf(), var crashExecutions: MethodGeneratedTests = mutableMapOf(), var errors: MutableMap = mutableMapOf() ) { @@ -38,6 +39,10 @@ data class TestsGenerationReport( appendHtmlLine( "Failing because of unexpected exception test methods: ${testMethodsStatistic.sumOf { it.failing }}" ) + appendHtmlLine( + "Failing because of exception (according to custom settings) test methods: " + + "${testMethodsStatistic.sumOf { it.artificiallyFailing }}" + ) appendHtmlLine( "Failing because of exceeding timeout test methods: ${testMethodsStatistic.sumOf { it.timeout }}" ) @@ -59,6 +64,7 @@ data class TestsGenerationReport( when (it.type) { CgTestMethodType.SUCCESSFUL, CgTestMethodType.PASSED_EXCEPTION -> updateExecutions(it, successfulExecutions) CgTestMethodType.FAILING -> updateExecutions(it, failedExecutions) + CgTestMethodType.ARTIFICIAL -> updateExecutions(it, artificiallyFailedExecutions) CgTestMethodType.TIMEOUT -> updateExecutions(it, timeoutExecutions) CgTestMethodType.CRASH -> updateExecutions(it, crashExecutions) CgTestMethodType.PARAMETRIZED -> { @@ -91,8 +97,9 @@ data class TestsGenerationReport( private fun ExecutableId.countTestMethods(): TestMethodStatistic = TestMethodStatistic( testMethodsNumber(successfulExecutions), testMethodsNumber(failedExecutions), + testMethodsNumber(artificiallyFailedExecutions), testMethodsNumber(timeoutExecutions), - testMethodsNumber(crashExecutions) + testMethodsNumber(crashExecutions), ) private fun ExecutableId.countErrors(): Int = errors.getOrDefault(this, emptyMap()).values.sum() @@ -104,7 +111,13 @@ data class TestsGenerationReport( executions.getOrPut(this) { mutableSetOf() } += it } - private data class TestMethodStatistic(val successful: Int, val failing: Int, val timeout: Int, val crashes: Int) { - val count: Int = successful + failing + timeout + crashes + private data class TestMethodStatistic( + val successful: Int, + val failing: Int, + val artificiallyFailing: Int, + val timeout: Int, + val crashes: Int, + ) { + val count: Int = successful + failing + artificiallyFailing + timeout + crashes } } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/services/framework/TestFrameworkManager.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/services/framework/TestFrameworkManager.kt index 98d36b5fb0..33100a890d 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/services/framework/TestFrameworkManager.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/services/framework/TestFrameworkManager.kt @@ -66,6 +66,9 @@ abstract class TestFrameworkManager(val context: CgContext) val assertTrue = context.testFramework.assertTrue val assertFalse = context.testFramework.assertFalse + val fail = context.testFramework.fail + val kotlinFail = context.testFramework.kotlinFail + val assertArrayEquals = context.testFramework.assertArrayEquals val assertBooleanArrayEquals = context.testFramework.assertBooleanArrayEquals val assertByteArrayEquals = context.testFramework.assertByteArrayEquals @@ -173,6 +176,15 @@ abstract class TestFrameworkManager(val context: CgContext) fun assertBoolean(actual: CgExpression) = assertBoolean(expected = true, actual) + fun fail(actual: CgExpression) { + // failure assertion may be implemented in different packages in Java and Kotlin + // more details at https://stackoverflow.com/questions/52967039/junit-5-assertions-fail-can-not-infer-type-in-kotlin + when (codegenLanguage) { + CodegenLanguage.JAVA -> +assertions[fail](actual) + CodegenLanguage.KOTLIN -> +assertions[kotlinFail](actual) + } + } + // Exception expectation differs between test frameworks // JUnit4 requires to add a specific argument to the test method annotation // JUnit5 requires using method assertThrows() diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt index 4e8a7c8133..617b83b947 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt @@ -4,6 +4,8 @@ import org.utbot.common.PathUtil import org.utbot.common.WorkaroundReason import org.utbot.common.isStatic import org.utbot.common.workaround +import org.utbot.engine.ArtificialError +import org.utbot.engine.OverflowDetectionError import org.utbot.framework.assemble.assemble import org.utbot.framework.codegen.domain.ForceStaticMocking import org.utbot.framework.codegen.domain.ParametrizedTestSource @@ -141,6 +143,7 @@ import org.utbot.framework.codegen.tree.CgTestClassConstructor.CgComponents.getS import org.utbot.framework.codegen.tree.CgTestClassConstructor.CgComponents.getTestFrameworkManagerBy import org.utbot.framework.codegen.tree.CgTestClassConstructor.CgComponents.getVariableConstructorBy import org.utbot.framework.plugin.api.UtExecutionResult +import org.utbot.framework.plugin.api.UtOverflowFailure import org.utbot.framework.plugin.api.UtStreamConsumingFailure import org.utbot.framework.plugin.api.util.allSuperTypes import org.utbot.framework.plugin.api.util.baseStreamClassId @@ -350,6 +353,12 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte } else -> error("Unexpected crash suite for failing execution with $expectedException exception") } + ARTIFICIAL -> { + methodInvocationBlock() + + val failureMessage = prepareArtificialFailureMessage(executionResult) + testFrameworkManager.fail(failureMessage) + } FAILING -> { writeWarningAboutFailureTest(expectedException) methodInvocationBlock() @@ -358,6 +367,18 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte } } + // TODO: ISSUE-1546 introduce some kind of language-specific string interpolation in Codegen + // and render arguments that cause overflow (but it requires engine enhancements) + private fun prepareArtificialFailureMessage(executionResult: UtExecutionResult): CgLiteral { + when (executionResult) { + is UtOverflowFailure -> { + val failureMessage = "Overflow detected in \'${currentExecutable!!.name}\' call" + return CgLiteral(stringClassId, failureMessage) + } + else -> error("$executionResult is not supported in artificial errors") + } + } + private fun constructExceptionProducingBlock( exceptionFromAnalysis: Throwable, executionResult: UtExecutionResult @@ -405,6 +426,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte if (exception is AccessControlException) return false // tests with timeout or crash should be processed differently if (exception is TimeoutException || exception is ConcreteExecutionFailureException) return false + if (exception is ArtificialError) return false if (UtSettings.treatAssertAsErrorSuite && exception is AssertionError) return false val exceptionRequiresAssert = exception !is RuntimeException || runtimeExceptionTestsBehaviour == PASS @@ -1796,6 +1818,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte shouldTestPassWithException(currentExecution, exception) -> PASSED_EXCEPTION shouldTestPassWithTimeoutException(currentExecution, exception) -> TIMEOUT else -> when (exception) { + is ArtificialError -> ARTIFICIAL is ConcreteExecutionFailureException -> CRASH is AccessControlException -> CRASH // exception from sandbox else -> FAILING diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt index a34af60f79..4c2a46d1ca 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt @@ -29,6 +29,7 @@ import org.utbot.framework.plugin.api.UtVoidModel * * Regression suite * * Error suite (invocations in which implicitly thrown unchecked exceptions reached to the top) * * Crash suite (invocations in which the instrumented process crashed or unexpected exception in our code occurred) + * * Artificial error suite (invocations in which some custom exception like overflow detection occurred) * * Timeout suite * * We want to minimize tests independently in each of these suites. diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/util/SootUtils.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/util/SootUtils.kt index 1a2c3da9ee..0fc195d704 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/util/SootUtils.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/util/SootUtils.kt @@ -197,6 +197,7 @@ private val classesToLoad = arrayOf( org.utbot.engine.overrides.stream.IntStream::class, org.utbot.engine.overrides.stream.LongStream::class, org.utbot.engine.overrides.stream.DoubleStream::class, + org.utbot.engine.OverflowDetectionError::class, ).map { it.java }.toTypedArray() private const val UTBOT_PACKAGE_PREFIX = "org.utbot" diff --git a/utbot-js/src/main/kotlin/framework/codegen/JsDomain.kt b/utbot-js/src/main/kotlin/framework/codegen/JsDomain.kt index e07b86bdbb..6fe2e76c60 100644 --- a/utbot-js/src/main/kotlin/framework/codegen/JsDomain.kt +++ b/utbot-js/src/main/kotlin/framework/codegen/JsDomain.kt @@ -13,6 +13,7 @@ object Mocha : TestFramework(id = "Mocha", displayName = "Mocha") { override val mainPackage = "" override val assertionsClass = jsUndefinedClassId override val arraysAssertionsClass = jsUndefinedClassId + override val kotlinFailureAssertionsClass: ClassId = jsUndefinedClassId override val testAnnotation = "" override val testAnnotationFqn = "" diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonDomain.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonDomain.kt index ea350f069f..0cf7a5fb74 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonDomain.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonDomain.kt @@ -15,6 +15,7 @@ object Pytest : TestFramework(displayName = "pytest", id = "pytest") { override val mainPackage: String = "pytest" override val assertionsClass: ClassId = pythonNoneClassId override val arraysAssertionsClass: ClassId = assertionsClass + override val kotlinFailureAssertionsClass: ClassId = assertionsClass override val testAnnotation: String get() = TODO("Not yet implemented") override val testAnnotationId: ClassId = BuiltinClassId( @@ -51,6 +52,7 @@ object Unittest : TestFramework(displayName = "Unittest", id = "Unittest") { override val mainPackage: String = "unittest" override val assertionsClass: ClassId = PythonClassId("self") override val arraysAssertionsClass: ClassId = assertionsClass + override val kotlinFailureAssertionsClass: ClassId = assertionsClass override val testAnnotation: String = "" override val testAnnotationId: ClassId = BuiltinClassId( canonicalName = "Unittest",