From e5d64effb71cb7bae2494ebc775b37fe16a860bb Mon Sep 17 00:00:00 2001 From: IlyaMuravjov Date: Thu, 6 Jul 2023 13:03:31 +0300 Subject: [PATCH 1/2] Prioritize executions with identical trace to minimize `stateBefore` --- .../org/utbot/engine/UtBotSymbolicEngine.kt | 19 +++- .../framework/minimization/Minimization.kt | 96 +++++++------------ .../org/utbot/framework/util/SizeUtils.kt | 55 +++++++++++ 3 files changed, 101 insertions(+), 69 deletions(-) create mode 100644 utbot-framework/src/main/kotlin/org/utbot/framework/util/SizeUtils.kt diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 405e71996c..e192c3d7db 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -36,6 +36,7 @@ import org.utbot.framework.UtSettings.useDebugVisualization import org.utbot.framework.plugin.api.* import org.utbot.framework.plugin.api.Step import org.utbot.framework.plugin.api.util.* +import org.utbot.framework.util.calculateSize import org.utbot.framework.util.convertToAssemble import org.utbot.framework.util.graph import org.utbot.framework.util.sootMethod @@ -423,6 +424,7 @@ class UtBotSymbolicEngine( .with(ValueProvider.of(relevantRepositories.map { SavedEntityValueProvider(defaultIdGenerator, it) })) .with(ValueProvider.of(generatedValueFieldIds.map { FieldValueProvider(defaultIdGenerator, it) })) }.let(transform) + val coverageToMinStateBeforeSize = mutableMapOf() runJavaFuzzing( defaultIdGenerator, methodUnderTest, @@ -449,15 +451,15 @@ class UtBotSymbolicEngine( return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP) } - val initialEnvironmentModels = EnvironmentModels(thisInstance?.model, values.map { it.model }, mapOf()) + val stateBefore = EnvironmentModels(thisInstance?.model, values.map { it.model }, mapOf()) val concreteExecutionResult: UtConcreteExecutionResult? = try { val timeoutMillis = min(UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis, diff) - concreteExecutor.executeConcretely(methodUnderTest, initialEnvironmentModels, listOf(), timeoutMillis) + concreteExecutor.executeConcretely(methodUnderTest, stateBefore, listOf(), timeoutMillis) } catch (e: CancellationException) { logger.debug { "Cancelled by timeout" }; null } catch (e: InstrumentedProcessDeathException) { - emitFailedConcreteExecutionResult(initialEnvironmentModels, e); null + emitFailedConcreteExecutionResult(stateBefore, e); null } catch (e: Throwable) { emit(UtError("Default concrete execution failed", e)); null } @@ -479,9 +481,16 @@ class UtBotSymbolicEngine( val result = concreteExecutionResult.result val coveredInstructions = concreteExecutionResult.coverage.coveredInstructions var trieNode: Trie.Node? = null + if (coveredInstructions.isNotEmpty()) { trieNode = descr.tracer.add(coveredInstructions) - if (trieNode.count > 1) { + + val earlierStateBeforeSize = coverageToMinStateBeforeSize[concreteExecutionResult.coverage] + val curStateBeforeSize = stateBefore.calculateSize() + + if (earlierStateBeforeSize == null || curStateBeforeSize < earlierStateBeforeSize) + coverageToMinStateBeforeSize[concreteExecutionResult.coverage] = curStateBeforeSize + else { if (++attempts >= attemptsLimit) { return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP) } @@ -499,7 +508,7 @@ class UtBotSymbolicEngine( emit( UtFuzzedExecution( - stateBefore = initialEnvironmentModels, + stateBefore = stateBefore, stateAfter = concreteExecutionResult.stateAfter, result = concreteExecutionResult.result, coverage = concreteExecutionResult.coverage, diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt index cb02d5364e..4da97d42db 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt @@ -1,26 +1,14 @@ package org.utbot.framework.minimization import org.utbot.framework.UtSettings -import org.utbot.framework.plugin.api.EnvironmentModels -import org.utbot.framework.plugin.api.UtArrayModel -import org.utbot.framework.plugin.api.UtAssembleModel -import org.utbot.framework.plugin.api.UtClassRefModel -import org.utbot.framework.plugin.api.UtCompositeModel import org.utbot.framework.plugin.api.UtConcreteExecutionFailure -import org.utbot.framework.plugin.api.UtDirectSetFieldModel -import org.utbot.framework.plugin.api.UtEnumConstantModel -import org.utbot.framework.plugin.api.UtExecutableCallModel import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionFailure import org.utbot.framework.plugin.api.UtExecutionResult -import org.utbot.framework.plugin.api.UtLambdaModel -import org.utbot.framework.plugin.api.UtModel -import org.utbot.framework.plugin.api.UtNullModel -import org.utbot.framework.plugin.api.UtPrimitiveModel -import org.utbot.framework.plugin.api.UtStatementCallModel -import org.utbot.framework.plugin.api.UtStatementModel import org.utbot.framework.plugin.api.UtSymbolicExecution -import org.utbot.framework.plugin.api.UtVoidModel +import org.utbot.framework.util.calculateSize +import org.utbot.fuzzer.UtFuzzedExecution +import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor /** @@ -53,16 +41,17 @@ fun minimizeTestCase( fun minimizeExecutions(executions: List): List { val unknownCoverageExecutions = - executions.indices.filter { executions[it].coverage?.coveredInstructions?.isEmpty() ?: true }.toSet() + executions.filter { it.coverage?.coveredInstructions.isNullOrEmpty() }.toSet() // ^^^ here we add executions with empty or null coverage, because it happens only if a concrete execution failed, // so we don't know the actual coverage for such executions - val filteredExecutions = executions.filterIndexed { idx, _ -> idx !in unknownCoverageExecutions } + val filteredExecutions = filterOutDuplicateCoverages(executions - unknownCoverageExecutions) val (mapping, executionToPriorityMapping) = buildMapping(filteredExecutions) - val usedExecutionIndexes = (GreedyEssential.minimize(mapping, executionToPriorityMapping) + unknownCoverageExecutions).toSet() + val usedFilteredExecutionIndexes = GreedyEssential.minimize(mapping, executionToPriorityMapping).toSet() + val usedFilteredExecutions = filteredExecutions.filterIndexed { idx, _ -> idx in usedFilteredExecutionIndexes } - val usedMinimizedExecutions = executions.filterIndexed { idx, _ -> idx in usedExecutionIndexes } + val usedMinimizedExecutions = usedFilteredExecutions + unknownCoverageExecutions return if (UtSettings.minimizeCrashExecutions) { usedMinimizedExecutions.filteredCrashExecutions() @@ -71,6 +60,18 @@ fun minimizeExecutions(executions: List): List { } } +private fun filterOutDuplicateCoverages(executions: List): List { + val (executionIdxToCoveredEdgesMap, _) = buildMapping(executions) + return executions + .withIndex() + // we need to group by coveredEdges and not just Coverage to not miss exceptional edges that buildMapping() function adds + .groupBy( + keySelector = { indexedExecution -> executionIdxToCoveredEdgesMap[indexedExecution.index] }, + valueTransform = { indexedExecution -> indexedExecution.value } + ).values + .map { executionsWithEqualCoverage -> executionsWithEqualCoverage.chooseOneExecution() } +} + /** * Groups the [executions] by their `paths` on `first` [branchInstructionsNumber] `branch` instructions. * @@ -192,55 +193,20 @@ private fun List.filteredCrashExecutions(): List { val notCrashExecutions = filterNot { it.result is UtConcreteExecutionFailure } - return notCrashExecutions + crashExecutions.chooseMinimalCrashExecution() + return notCrashExecutions + crashExecutions.chooseOneExecution() } /** - * As for now crash execution can only be produced by Concrete Executor, it does not have [UtExecution.stateAfter] and - * [UtExecution.result] is [UtExecutionFailure], so we check only [UtExecution.stateBefore]. + * Chooses one execution with the highest [execution priority][getExecutionPriority]. If multiple executions + * have the same priority, then the one with the [smallest][calculateSize] [UtExecution.stateBefore] is chosen. + * + * Only [UtExecution.stateBefore] is considered, because [UtExecution.result] and [UtExecution.stateAfter] + * don't represent true picture as they are limited by [construction depth][UtModelConstructor.maxDepth] and their + * sizes can't be calculated for crushed executions. */ -private fun List.chooseMinimalCrashExecution(): UtExecution = minByOrNull { - it.stateBefore.calculateSize() -} ?: error("Cannot find minimal crash execution within empty executions") - -private fun EnvironmentModels.calculateSize(): Int { - val thisInstanceSize = thisInstance?.calculateSize() ?: 0 - val parametersSize = parameters.sumOf { it.calculateSize() } - val staticsSize = statics.values.sumOf { it.calculateSize() } - - return thisInstanceSize + parametersSize + staticsSize -} - -/** - * We assume that "size" for "common" models is 1, 0 for [UtVoidModel] (as they do not return anything) and - * [UtPrimitiveModel] and [UtNullModel] (we use them as literals in codegen), summarising for all statements for [UtAssembleModel] and - * summarising for all fields and mocks for [UtCompositeModel]. As [UtCompositeModel] could be recursive, we need to - * store it in [used]. Moreover, if we already calculate size for [this], it means that we will use already created - * variable by this model and do not need to create it again, so size should be equal to 0. - */ -private fun UtModel.calculateSize(used: MutableSet = mutableSetOf()): Int { - if (this in used) return 0 - - used += this - - return when (this) { - is UtNullModel, is UtPrimitiveModel, UtVoidModel -> 0 - is UtClassRefModel, is UtEnumConstantModel, is UtArrayModel -> 1 - is UtAssembleModel -> { - 1 + instantiationCall.calculateSize(used) + modificationsChain.sumOf { it.calculateSize(used) } - } - is UtCompositeModel -> 1 + fields.values.sumOf { it.calculateSize(used) } - is UtLambdaModel -> 1 + capturedValues.sumOf { it.calculateSize(used) } - // PythonModel, JsUtModel, UtSpringContextModel may be here - else -> 0 - } -} - -private fun UtStatementModel.calculateSize(used: MutableSet = mutableSetOf()): Int = - when (this) { - is UtDirectSetFieldModel -> 1 + fieldModel.calculateSize(used) + instance.calculateSize(used) - is UtStatementCallModel -> 1 + params.sumOf { it.calculateSize(used) } + (instance?.calculateSize(used) ?: 0) - } +private fun List.chooseOneExecution(): UtExecution = minWithOrNull( + compareBy({ it.getExecutionPriority() }, { it.stateBefore.calculateSize() }) +) ?: error("Cannot find minimal execution within empty executions") /** * Extends the [instructionsWithoutExtra] with one extra instruction if the [result] is @@ -274,6 +240,8 @@ private fun Throwable.exceptionToInfo(): String = * Returns an execution priority. [UtSymbolicExecution] has the highest priority * over other executions like [UtFuzzedExecution], [UtFailedExecution], etc. * + * NOTE! Smaller number represents higher priority. + * * See [https://github.com/UnitTestBot/UTBotJava/issues/1504] for more details. */ private fun UtExecution.getExecutionPriority(): Int = when (this) { diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/util/SizeUtils.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/util/SizeUtils.kt new file mode 100644 index 0000000000..2e3d238121 --- /dev/null +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/util/SizeUtils.kt @@ -0,0 +1,55 @@ +package org.utbot.framework.util + +import org.utbot.framework.plugin.api.EnvironmentModels +import org.utbot.framework.plugin.api.UtArrayModel +import org.utbot.framework.plugin.api.UtAssembleModel +import org.utbot.framework.plugin.api.UtClassRefModel +import org.utbot.framework.plugin.api.UtCompositeModel +import org.utbot.framework.plugin.api.UtDirectSetFieldModel +import org.utbot.framework.plugin.api.UtEnumConstantModel +import org.utbot.framework.plugin.api.UtLambdaModel +import org.utbot.framework.plugin.api.UtModel +import org.utbot.framework.plugin.api.UtNullModel +import org.utbot.framework.plugin.api.UtPrimitiveModel +import org.utbot.framework.plugin.api.UtStatementCallModel +import org.utbot.framework.plugin.api.UtStatementModel +import org.utbot.framework.plugin.api.UtVoidModel + +fun EnvironmentModels.calculateSize(): Int { + val thisInstanceSize = thisInstance?.calculateSize() ?: 0 + val parametersSize = parameters.sumOf { it.calculateSize() } + val staticsSize = statics.values.sumOf { it.calculateSize() } + + return thisInstanceSize + parametersSize + staticsSize +} + +/** + * We assume that "size" for "common" models is 1, 0 for [UtVoidModel] (as they do not return anything) and + * [UtPrimitiveModel] and [UtNullModel] (we use them as literals in codegen), summarising for all statements for [UtAssembleModel] and + * summarising for all fields and mocks for [UtCompositeModel]. As [UtCompositeModel] could be recursive, we need to + * store it in [used]. Moreover, if we already calculate size for [this], it means that we will use already created + * variable by this model and do not need to create it again, so size should be equal to 0. + */ +private fun UtModel.calculateSize(used: MutableSet = mutableSetOf()): Int { + if (this in used) return 0 + + used += this + + return when (this) { + is UtNullModel, is UtPrimitiveModel, UtVoidModel -> 0 + is UtClassRefModel, is UtEnumConstantModel, is UtArrayModel -> 1 + is UtAssembleModel -> { + 1 + instantiationCall.calculateSize(used) + modificationsChain.sumOf { it.calculateSize(used) } + } + is UtCompositeModel -> 1 + fields.values.sumOf { it.calculateSize(used) } + is UtLambdaModel -> 1 + capturedValues.sumOf { it.calculateSize(used) } + // PythonModel, JsUtModel, UtSpringContextModel may be here + else -> 0 + } +} + +private fun UtStatementModel.calculateSize(used: MutableSet = mutableSetOf()): Int = + when (this) { + is UtDirectSetFieldModel -> 1 + fieldModel.calculateSize(used) + instance.calculateSize(used) + is UtStatementCallModel -> 1 + params.sumOf { it.calculateSize(used) } + (instance?.calculateSize(used) ?: 0) + } From 59aa0a1f6fb27b982db9dbc78f0d51e3c45d7bbc Mon Sep 17 00:00:00 2001 From: IlyaMuravjov Date: Fri, 7 Jul 2023 15:53:43 +0300 Subject: [PATCH 2/2] Use `Trie.Node` as key instead of `Coverage` --- .../src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index e192c3d7db..1e83da4de5 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -424,7 +424,7 @@ class UtBotSymbolicEngine( .with(ValueProvider.of(relevantRepositories.map { SavedEntityValueProvider(defaultIdGenerator, it) })) .with(ValueProvider.of(generatedValueFieldIds.map { FieldValueProvider(defaultIdGenerator, it) })) }.let(transform) - val coverageToMinStateBeforeSize = mutableMapOf() + val coverageToMinStateBeforeSize = mutableMapOf, Int>() runJavaFuzzing( defaultIdGenerator, methodUnderTest, @@ -485,11 +485,11 @@ class UtBotSymbolicEngine( if (coveredInstructions.isNotEmpty()) { trieNode = descr.tracer.add(coveredInstructions) - val earlierStateBeforeSize = coverageToMinStateBeforeSize[concreteExecutionResult.coverage] + val earlierStateBeforeSize = coverageToMinStateBeforeSize[trieNode] val curStateBeforeSize = stateBefore.calculateSize() if (earlierStateBeforeSize == null || curStateBeforeSize < earlierStateBeforeSize) - coverageToMinStateBeforeSize[concreteExecutionResult.coverage] = curStateBeforeSize + coverageToMinStateBeforeSize[trieNode] = curStateBeforeSize else { if (++attempts >= attemptsLimit) { return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP)