First autotests for tests generation on Spring projects

settings.gradle.kts

@@ -73,3 +73,7 @@ if (goIde.split(",").contains(ideType)) {
 include("utbot-spring-analyzer")
 include("utbot-spring-commons")
 include("utbot-spring-commons-api")
+
+include("utbot-spring-sample")
+include("utbot-spring-test")
+

utbot-framework-test/build.gradle

@@ -46,6 +46,8 @@ dependencies {
     implementation group: 'com.github.UnitTestBot.ksmt', name: 'ksmt-z3', version: ksmtVersion
 }
 
+// This is required to avoid conflict between SpringBoot standard logger and the logger of our project.
+// See https://stackoverflow.com/a/28735604 for more details.
 test {
     if (System.getProperty('DEBUG', 'false') == 'true') {
         jvmArgs '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=9009'

utbot-spring-sample/build.gradle

@@ -0,0 +1,21 @@
+plugins {
+    id 'java-library'
+}
+
+dependencies {
+    implementation(project(":utbot-api"))
+
+    implementation 'org.projectlombok:lombok:1.18.20'
+    annotationProcessor 'org.projectlombok:lombok:1.18.20'
+
+    implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: springBootVersion
+    implementation group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa', version: springBootVersion
+}
+
+// This is required to avoid conflict between SpringBoot standard logger and the logger of our project.
+// See https://stackoverflow.com/a/28735604 for more details.
+configurations {
+    all {
+        exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging'
+    }
+}

utbot-spring-sample/src/main/java/org/utbot/examples/spring/autowiring/Order.java

@@ -0,0 +1,24 @@
+package org.utbot.examples.spring.autowiring;
+
+import lombok.*;
+import lombok.extern.jackson.Jacksonized;
+
+import javax.persistence.*;
+
+@Getter
+@Setter
+@Builder
+@ToString
+@Jacksonized
+@NoArgsConstructor
+@AllArgsConstructor
+@Entity
+@Table(name = "orders")
+public class Order {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    Long id;
+    String buyer;
+    Double price;
+    int qty;
+}

utbot-spring-sample/src/main/java/org/utbot/examples/spring/autowiring/OrderRepository.java

@@ -0,0 +1,6 @@
+package org.utbot.examples.spring.autowiring;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface OrderRepository extends JpaRepository<Order, Long> {
+}

utbot-spring-sample/src/main/java/org/utbot/examples/spring/autowiring/OrderService.java

@@ -0,0 +1,21 @@
+package org.utbot.examples.spring.autowiring;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service
+public class OrderService {
+
+    @Autowired
+    private OrderRepository orderRepository;
+
+    public List<Order> getOrders() {
+        return orderRepository.findAll();
+    }
+
+    public Order createOrder(Order order) {
+        return orderRepository.save(order);
+    }
+}

utbot-spring-test/build.gradle

@@ -0,0 +1,31 @@
+dependencies {
+    testImplementation(project(":utbot-framework"))
+    testImplementation project(':utbot-testing')
+    testImplementation project(':utbot-spring-sample')
+
+    // To use JUnit4, comment out JUnit5 and uncomment JUnit4 dependencies here. Please also check "test" section
+    // testImplementation group: 'junit', name: 'junit', version: '4.13.1'
+    testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.1'
+    testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.1'
+
+    // used for testing code generation
+    testImplementation group: 'junit', name: 'junit', version: junit4Version
+    testImplementation group: 'org.junit.platform', name: 'junit-platform-console-standalone', version: junit4PlatformVersion
+    testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion
+    testImplementation group: 'org.mockito', name: 'mockito-inline', version: mockitoInlineVersion
+    testImplementation group: 'org.jacoco', name: 'org.jacoco.report', version: jacocoVersion
+
+    testImplementation group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa', version: springBootVersion
+}
+
+configurations {
+    all {
+        exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging'
+    }
+}
+
+test {
+    if (System.getProperty('DEBUG', 'false') == 'true') {
+        jvmArgs '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=9009'
+    }
+}

utbot-spring-test/src/test/kotlin/org/utbot/examples/spring/autowiring/OrderServiceTests.kt

@@ -0,0 +1,62 @@
+package org.utbot.examples.spring.autowiring
+
+import org.junit.jupiter.api.Test
+import org.utbot.examples.spring.utils.springAdditionalDependencies
+import org.utbot.examples.spring.utils.standardSpringTestingConfigurations
+import org.utbot.framework.plugin.api.MockStrategyApi
+import org.utbot.testcheckers.eq
+import org.utbot.testing.*
+import kotlin.reflect.full.functions
+import kotlin.reflect.KFunction1
+import kotlin.reflect.KFunction2
+
+internal class OrderServiceTests : UtValueTestCaseChecker(
+    testClass = OrderService::class,
+    configurations = standardSpringTestingConfigurations
+) {
+    @Test
+    fun testGetOrders() {
+        checkMocks(
+            method = OrderService::getOrders,
+            branches = eq(1),
+            { mocks, r ->
+                val orderRepository = mocks.singleMock("orderRepository", findAllRepositoryCall)
+                orderRepository.value<List<Order>?>() == r
+            },
+            coverage = DoNotCalculate,
+            mockStrategy = MockStrategyApi.OTHER_CLASSES,
+            additionalDependencies = springAdditionalDependencies,
+        )
+    }
+
+    @Test
+    fun testCreateOrder() {
+        checkMocksWithExceptions(
+            method = OrderService::createOrder,
+            // TODO: replace with `branches = eq(1)` after fix of https://github.com/UnitTestBot/UTBotJava/issues/2367
+            branches = ignoreExecutionsNumber,
+            { _: Order?, mocks, r: Result<Order?> ->
+                val orderRepository = mocks.singleMock("orderRepository", saveRepositoryCall)
+                orderRepository.value<Order?>() == r.getOrNull()
+            },
+            coverage = DoNotCalculate,
+            mockStrategy = MockStrategyApi.OTHER_CLASSES,
+            additionalDependencies = springAdditionalDependencies,
+        )
+    }
+
+    @Suppress("UNCHECKED_CAST")
+    private val findAllRepositoryCall: KFunction1<OrderRepository, List<Order>?> =
+        OrderRepository::class
+            .functions
+            .single { it.name == "findAll" && it.parameters.size == 1 }
+                as KFunction1<OrderRepository, List<Order>?>
+
+
+    @Suppress("UNCHECKED_CAST")
+    private val saveRepositoryCall: KFunction2<OrderRepository, Order?, Order?> =
+        OrderRepository::class
+            .functions
+            .single { it.name == "save" && it.parameters.size == 2 }
+                as KFunction2<OrderRepository, Order?, Order?>
+}

utbot-spring-test/src/test/kotlin/org/utbot/examples/spring/utils/SpringTestingConfiguration.kt

@@ -0,0 +1,17 @@
+package org.utbot.examples.spring.utils
+
+import org.springframework.data.jpa.repository.JpaRepository
+import org.springframework.data.repository.PagingAndSortingRepository
+import org.utbot.framework.codegen.domain.ParametrizedTestSource
+import org.utbot.framework.plugin.api.CodegenLanguage
+import org.utbot.testing.SpringConfiguration
+import org.utbot.testing.TestExecution
+
+val standardSpringTestingConfigurations: List<SpringConfiguration> = listOf(
+    SpringConfiguration(CodegenLanguage.JAVA, ParametrizedTestSource.DO_NOT_PARAMETRIZE, TestExecution)
+)
+
+val springAdditionalDependencies: Array<Class<*>> = arrayOf(
+    JpaRepository::class.java,
+    PagingAndSortingRepository::class.java,
+)

utbot-spring-test/src/test/resources/log4j2.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration>
+    <Appenders>
+        <File name="FrameworkAppender"
+                      append="false"
+                     fileName="logs/framework.log"
+                     filePattern="logs/framework-%d{MM-dd-yyyy}.log.gz"
+                     ignoreExceptions="false">
+
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} | %-5level | %-25c{1} | %msg%n"/>
+        </File>
+
+        <Console name="Console" target="SYSTEM_OUT">
+            <ThresholdFilter level="DEBUG"  onMatch="NEUTRAL"   onMismatch="DENY"/>
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} | %-5level | %-25c{1} | %msg%n"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <!--        Uncomment this logger to see path -->
+        <Logger name="org.utbot.engine.UtBotSymbolicEngine.path" level="debug"/>
+
+
+        <!-- Set this logger level to TRACE to see SMT requests, and SAT/UNSAT/UNKNOWN responses -->
+        <Logger name="org.utbot.engine.pc" level="debug"/>
+
+        <!-- Not interested in summarization logs now -->
+        <Logger name="org.utbot.summary" level="info"/>
+
+        <Logger name="soot.PackManager" level="INFO"/>
+
+        <Root level="debug">
+            <AppenderRef ref="Console"/>
+            <AppenderRef ref="FrameworkAppender"/>
+        </Root>
+    </Loggers>
+</Configuration>

utbot-testing/src/main/kotlin/org/utbot/testing/TestCodeGeneratorPipeline.kt

@@ -40,14 +40,14 @@ class TestCodeGeneratorPipeline(private val testInfrastructureConfiguration: Tes
 
     private fun runPipelinesStages(classPipeline: ClassPipeline): CodeGenerationResult {
         val classUnderTest = classPipeline.stageContext.classUnderTest
-        val classPipelineName =
-            classUnderTest.qualifiedName ?: error("${classUnderTest.simpleName} doesn't have a fqn name")
+        val classPipelineName = classUnderTest.qualifiedName
+            ?: error("${classUnderTest.simpleName} doesn't have a fqn name")
 
         logger
             .info()
             .measureTime({ "Executing code generation tests for [$classPipelineName]" }) {
                 CodeGeneration.verifyPipeline(classPipeline)?.let {
-                    withUtContext(UtContext(it.stageContext.classUnderTest.java.classLoader)) {
+                    withUtContext(UtContext(classUnderTest.java.classLoader)) {
                         processCodeGenerationStage(it)
                     }
                 }
@@ -86,13 +86,13 @@ class TestCodeGeneratorPipeline(private val testInfrastructureConfiguration: Tes
                     val prefix = when (codegenLanguage) {
                         CodegenLanguage.JAVA ->
                             when (parametrizedTestSource) {
-                                ParametrizedTestSource.DO_NOT_PARAMETRIZE -> "public void "
+                                ParametrizedTestSource.DO_NOT_PARAMETRIZE -> "@Test"
                                 ParametrizedTestSource.PARAMETRIZE -> "public void parameterizedTestsFor"
                             }
 
                         CodegenLanguage.KOTLIN ->
                             when (parametrizedTestSource) {
-                                ParametrizedTestSource.DO_NOT_PARAMETRIZE -> "fun "
+                                ParametrizedTestSource.DO_NOT_PARAMETRIZE -> "@Test"
                                 ParametrizedTestSource.PARAMETRIZE -> "fun parameterizedTestsFor"
                             }
                     }

utbot-testing/src/main/kotlin/org/utbot/testing/UtModelTestCaseChecker.kt

@@ -42,7 +42,7 @@ import kotlin.reflect.KFunction3
 abstract class UtModelTestCaseChecker(
     testClass: KClass<*>,
     testCodeGeneration: Boolean = true,
-    configurations: List<Configuration> = standardTestingConfigurations,
+    configurations: List<AbstractConfiguration> = standardTestingConfigurations,
 ) : CodeGenerationIntegrationTest(testClass, testCodeGeneration, configurations) {
     protected fun check(
         method: KFunction2<*, *, *>,

utbot-testing/src/main/kotlin/org/utbot/testing/UtValueTestCaseChecker.kt

@@ -61,7 +61,7 @@ import kotlin.reflect.KFunction5
 abstract class UtValueTestCaseChecker(
     testClass: KClass<*>,
     testCodeGeneration: Boolean = true,
-    configurations: List<Configuration> = standardTestingConfigurations,
+    configurations: List<AbstractConfiguration> = standardTestingConfigurations,
 ) : CodeGenerationIntegrationTest(testClass, testCodeGeneration, configurations) {
     // contains already analyzed by the engine methods
     private val analyzedMethods: MutableMap<MethodWithMockStrategy, MethodResult> = mutableMapOf()
@@ -530,6 +530,81 @@ abstract class UtValueTestCaseChecker(
         additionalMockAlwaysClasses = additionalMockAlwaysClasses
     )
 
+    protected inline fun <reified R> checkMocksWithExceptions(
+        method: KFunction1<*, R>,
+        branches: ExecutionsNumberMatcher,
+        vararg matchers: (Mocks, Result<R>) -> Boolean,
+        coverage: CoverageMatcher = Full,
+        mockStrategy: MockStrategyApi = NO_MOCKS,
+        additionalDependencies: Array<Class<*>> = emptyArray(),
+        additionalMockAlwaysClasses: Set<ClassId> = emptySet()
+    ) = internalCheck(
+        method, mockStrategy, branches, matchers, coverage,
+        arguments = ::withMocksAndExceptions,
+        additionalDependencies = additionalDependencies,
+        additionalMockAlwaysClasses = additionalMockAlwaysClasses
+    )
+
+    protected inline fun <reified T, reified R> checkMocksWithExceptions(
+        method: KFunction2<*, T, R>,
+        branches: ExecutionsNumberMatcher,
+        vararg matchers: (T, Mocks, Result<R>) -> Boolean,
+        coverage: CoverageMatcher = Full,
+        mockStrategy: MockStrategyApi = NO_MOCKS,
+        additionalDependencies: Array<Class<*>> = emptyArray(),
+        additionalMockAlwaysClasses: Set<ClassId> = emptySet()
+    ) = internalCheck(
+        method, mockStrategy, branches, matchers, coverage, T::class,
+        arguments = ::withMocksAndExceptions,
+        additionalDependencies = additionalDependencies,
+        additionalMockAlwaysClasses = additionalMockAlwaysClasses
+    )
+
+    protected inline fun <reified T1, reified T2, reified R> checkMocksWithExceptions(
+        method: KFunction3<*, T1, T2, R>,
+        branches: ExecutionsNumberMatcher,
+        vararg matchers: (T1, T2, Mocks, Result<R>) -> Boolean,
+        coverage: CoverageMatcher = Full,
+        mockStrategy: MockStrategyApi = NO_MOCKS,
+        additionalDependencies: Array<Class<*>> = emptyArray(),
+        additionalMockAlwaysClasses: Set<ClassId> = emptySet()
+    ) = internalCheck(
+        method, mockStrategy, branches, matchers, coverage, T1::class, T2::class,
+        arguments = ::withMocksAndExceptions,
+        additionalDependencies = additionalDependencies,
+        additionalMockAlwaysClasses = additionalMockAlwaysClasses
+    )
+
+    protected inline fun <reified T1, reified T2, reified T3, reified R> checkMocksWithExceptions(
+        method: KFunction4<*, T1, T2, T3, R>,
+        branches: ExecutionsNumberMatcher,
+        vararg matchers: (T1, T2, T3, Mocks, Result<R>) -> Boolean,
+        coverage: CoverageMatcher = Full,
+        mockStrategy: MockStrategyApi = NO_MOCKS,
+        additionalDependencies: Array<Class<*>> = emptyArray(),
+        additionalMockAlwaysClasses: Set<ClassId> = emptySet()
+    ) = internalCheck(
+        method, mockStrategy, branches, matchers, coverage, T1::class, T2::class, T3::class,
+        arguments = ::withMocksAndExceptions,
+        additionalDependencies = additionalDependencies,
+        additionalMockAlwaysClasses = additionalMockAlwaysClasses
+    )
+
+    protected inline fun <reified T1, reified T2, reified T3, reified T4, reified R> checkMocksWithExceptions(
+        method: KFunction5<*, T1, T2, T3, T4, R>,
+        branches: ExecutionsNumberMatcher,
+        vararg matchers: (T1, T2, T3, T4, Mocks, Result<R>) -> Boolean,
+        coverage: CoverageMatcher = Full,
+        mockStrategy: MockStrategyApi = NO_MOCKS,
+        additionalDependencies: Array<Class<*>> = emptyArray(),
+        additionalMockAlwaysClasses: Set<ClassId> = emptySet()
+    ) = internalCheck(
+        method, mockStrategy, branches, matchers, coverage, T1::class, T2::class, T3::class, T4::class,
+        arguments = ::withMocksAndExceptions,
+        additionalDependencies = additionalDependencies,
+        additionalMockAlwaysClasses = additionalMockAlwaysClasses
+    )
+
     // check paramsBefore, mocks and instrumentation and result value
     protected inline fun <reified R> checkMocksAndInstrumentation(
         method: KFunction1<*, R>,
@@ -2232,6 +2307,9 @@ fun withMocks(ex: UtValueExecution<*>) = ex.paramsBefore + listOf(ex.mocks) + ex
 fun withMocksAndInstrumentation(ex: UtValueExecution<*>) =
     ex.paramsBefore + listOf(ex.mocks) + listOf(ex.instrumentation) + ex.evaluatedResult
 
+fun withMocksAndExceptions(ex: UtValueExecution<*>) =
+    ex.paramsBefore + listOf(ex.mocks) + ex.returnValue
+
 fun withMocksInstrumentationAndThis(ex: UtValueExecution<*>) =
     listOf(ex.callerBefore) + ex.paramsBefore + listOf(ex.mocks) + listOf(ex.instrumentation) + ex.evaluatedResult