Skip to content

Prioritize executions with identical trace to minimize stateBefore #2371

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 10, 2023
Merged

Prioritize executions with identical trace to minimize stateBefore #2371

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e5d64ef
Prioritize executions with identical trace to minimize `stateBefore`
IlyaMuravjov Jul 6, 2023
59aa0a1
Use `Trie.Node<Instruction>` as key instead of `Coverage`
IlyaMuravjov Jul 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import org.utbot.framework.UtSettings.useDebugVisualization
import org.utbot.framework.plugin.api.*
import org.utbot.framework.plugin.api.Step
import org.utbot.framework.plugin.api.util.*
import org.utbot.framework.util.calculateSize
import org.utbot.framework.util.convertToAssemble
import org.utbot.framework.util.graph
import org.utbot.framework.util.sootMethod
Expand Down Expand Up @@ -423,6 +424,7 @@ class UtBotSymbolicEngine(
.with(ValueProvider.of(relevantRepositories.map { SavedEntityValueProvider(defaultIdGenerator, it) }))
.with(ValueProvider.of(generatedValueFieldIds.map { FieldValueProvider(defaultIdGenerator, it) }))
}.let(transform)
val coverageToMinStateBeforeSize = mutableMapOf<Trie.Node<Instruction>, Int>()
runJavaFuzzing(
defaultIdGenerator,
methodUnderTest,
Expand All @@ -449,15 +451,15 @@ class UtBotSymbolicEngine(
return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP)
}

val initialEnvironmentModels = EnvironmentModels(thisInstance?.model, values.map { it.model }, mapOf())
val stateBefore = EnvironmentModels(thisInstance?.model, values.map { it.model }, mapOf())

val concreteExecutionResult: UtConcreteExecutionResult? = try {
val timeoutMillis = min(UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis, diff)
concreteExecutor.executeConcretely(methodUnderTest, initialEnvironmentModels, listOf(), timeoutMillis)
concreteExecutor.executeConcretely(methodUnderTest, stateBefore, listOf(), timeoutMillis)
} catch (e: CancellationException) {
logger.debug { "Cancelled by timeout" }; null
} catch (e: InstrumentedProcessDeathException) {
emitFailedConcreteExecutionResult(initialEnvironmentModels, e); null
emitFailedConcreteExecutionResult(stateBefore, e); null
} catch (e: Throwable) {
emit(UtError("Default concrete execution failed", e)); null
}
Expand All @@ -479,9 +481,16 @@ class UtBotSymbolicEngine(
val result = concreteExecutionResult.result
val coveredInstructions = concreteExecutionResult.coverage.coveredInstructions
var trieNode: Trie.Node<Instruction>? = null

if (coveredInstructions.isNotEmpty()) {
trieNode = descr.tracer.add(coveredInstructions)
if (trieNode.count > 1) {

val earlierStateBeforeSize = coverageToMinStateBeforeSize[trieNode]
val curStateBeforeSize = stateBefore.calculateSize()

if (earlierStateBeforeSize == null || curStateBeforeSize < earlierStateBeforeSize)
coverageToMinStateBeforeSize[trieNode] = curStateBeforeSize
else {
if (++attempts >= attemptsLimit) {
return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP)
}
Expand All @@ -499,7 +508,7 @@ class UtBotSymbolicEngine(

emit(
UtFuzzedExecution(
stateBefore = initialEnvironmentModels,
stateBefore = stateBefore,
stateAfter = concreteExecutionResult.stateAfter,
result = concreteExecutionResult.result,
coverage = concreteExecutionResult.coverage,
Expand Down
96 changes: 32 additions & 64 deletions utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,14 @@
package org.utbot.framework.minimization

import org.utbot.framework.UtSettings
import org.utbot.framework.plugin.api.EnvironmentModels
import org.utbot.framework.plugin.api.UtArrayModel
import org.utbot.framework.plugin.api.UtAssembleModel
import org.utbot.framework.plugin.api.UtClassRefModel
import org.utbot.framework.plugin.api.UtCompositeModel
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
import org.utbot.framework.plugin.api.UtDirectSetFieldModel
import org.utbot.framework.plugin.api.UtEnumConstantModel
import org.utbot.framework.plugin.api.UtExecutableCallModel
import org.utbot.framework.plugin.api.UtExecution
import org.utbot.framework.plugin.api.UtExecutionFailure
import org.utbot.framework.plugin.api.UtExecutionResult
import org.utbot.framework.plugin.api.UtLambdaModel
import org.utbot.framework.plugin.api.UtModel
import org.utbot.framework.plugin.api.UtNullModel
import org.utbot.framework.plugin.api.UtPrimitiveModel
import org.utbot.framework.plugin.api.UtStatementCallModel
import org.utbot.framework.plugin.api.UtStatementModel
import org.utbot.framework.plugin.api.UtSymbolicExecution
import org.utbot.framework.plugin.api.UtVoidModel
import org.utbot.framework.util.calculateSize
import org.utbot.fuzzer.UtFuzzedExecution
import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor


/**
Expand Down Expand Up @@ -53,16 +41,17 @@ fun <T : Any> minimizeTestCase(

fun minimizeExecutions(executions: List<UtExecution>): List<UtExecution> {
val unknownCoverageExecutions =
executions.indices.filter { executions[it].coverage?.coveredInstructions?.isEmpty() ?: true }.toSet()
executions.filter { it.coverage?.coveredInstructions.isNullOrEmpty() }.toSet()
// ^^^ here we add executions with empty or null coverage, because it happens only if a concrete execution failed,
// so we don't know the actual coverage for such executions

val filteredExecutions = executions.filterIndexed { idx, _ -> idx !in unknownCoverageExecutions }
val filteredExecutions = filterOutDuplicateCoverages(executions - unknownCoverageExecutions)
Copy link
Collaborator

@Markoutte Markoutte Jul 7, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure that this works as expected, because it compared indices (which are integers) then but now it compares UtExecutions for subtracting. But there's no hashCode and equals for UtExecution. Also, this implementation is less efficient when comparing values.

Copy link
Collaborator Author

@IlyaMuravjov IlyaMuravjov Jul 7, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should work as expected, because reference equality is used when hashCode and equals are not overridden.

For some context, I changed this part, because there was a bug in minimization. To be more exact, in the following code fragment indices referring to positions of UtExecutions in filteredExecutions list were saved into usedExecutionIndexes:

val (mapping, executionToPriorityMapping) = buildMapping(filteredExecutions)
val usedExecutionIndexes = (GreedyEssential.minimize(mapping, executionToPriorityMapping) +  /* ... */).toSet()

And later on these usedExecutionIndexes were used as if they were indices that refer to positions of UtExecutions in the whole executions list:

val usedMinimizedExecutions = executions.filterIndexed { idx, _ -> idx in usedExecutionIndexes }

val (mapping, executionToPriorityMapping) = buildMapping(filteredExecutions)

val usedExecutionIndexes = (GreedyEssential.minimize(mapping, executionToPriorityMapping) + unknownCoverageExecutions).toSet()
val usedFilteredExecutionIndexes = GreedyEssential.minimize(mapping, executionToPriorityMapping).toSet()
val usedFilteredExecutions = filteredExecutions.filterIndexed { idx, _ -> idx in usedFilteredExecutionIndexes }

val usedMinimizedExecutions = executions.filterIndexed { idx, _ -> idx in usedExecutionIndexes }
val usedMinimizedExecutions = usedFilteredExecutions + unknownCoverageExecutions

return if (UtSettings.minimizeCrashExecutions) {
usedMinimizedExecutions.filteredCrashExecutions()
Expand All @@ -71,6 +60,18 @@ fun minimizeExecutions(executions: List<UtExecution>): List<UtExecution> {
}
}

private fun filterOutDuplicateCoverages(executions: List<UtExecution>): List<UtExecution> {
val (executionIdxToCoveredEdgesMap, _) = buildMapping(executions)
Copy link
Collaborator

@Markoutte Markoutte Jul 7, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This operation can be heavy when number of executions is huge. Could it be verified that it works OK for a huge number of executions?

Copy link
Collaborator Author

@IlyaMuravjov IlyaMuravjov Jul 7, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it doesn't work OK for a huge number of executions, then it should have already been noticed on functions with loops and recursions, beside that inspecting buildMapping function body we can see that it only iterates over instructions once.

Although, it worth noting that buildMapping may have some performance related issues, since it uses allCoveredEdges, that can get quite large and cause a lot of cache misses, but I still think it's premature to optimize it now without any evidence (nor even known incidents) of it being a bottleneck.

// (inst1, instr2) -> edge id --- edge represents as a pair of instructions, which are connected by this edge
val allCoveredEdges = mutableMapOf<Pair<Long, Long?>, Int>()

By the way, on the topic of filtering out duplicate coverages, is that intended that fuzzer and minimization do it differently? Right now, when engine produces multiple executions with identical coverage, but throwing different exceptions, all these executions are preserved by minimization, however fuzzer itself only keeps one execuction.

Executions can throw different exceptions, while having identical coverage, if exceptions are thrown from within the JRE itself or from some large libraries (e.g. Spring) that are not transformed for trace collection.

return executions
.withIndex()
// we need to group by coveredEdges and not just Coverage to not miss exceptional edges that buildMapping() function adds
.groupBy(
keySelector = { indexedExecution -> executionIdxToCoveredEdgesMap[indexedExecution.index] },
valueTransform = { indexedExecution -> indexedExecution.value }
).values
.map { executionsWithEqualCoverage -> executionsWithEqualCoverage.chooseOneExecution() }
}

/**
* Groups the [executions] by their `paths` on `first` [branchInstructionsNumber] `branch` instructions.
*
Expand Down Expand Up @@ -192,55 +193,20 @@ private fun List<UtExecution>.filteredCrashExecutions(): List<UtExecution> {

val notCrashExecutions = filterNot { it.result is UtConcreteExecutionFailure }

return notCrashExecutions + crashExecutions.chooseMinimalCrashExecution()
return notCrashExecutions + crashExecutions.chooseOneExecution()
}

/**
* As for now crash execution can only be produced by Concrete Executor, it does not have [UtExecution.stateAfter] and
* [UtExecution.result] is [UtExecutionFailure], so we check only [UtExecution.stateBefore].
* Chooses one execution with the highest [execution priority][getExecutionPriority]. If multiple executions
* have the same priority, then the one with the [smallest][calculateSize] [UtExecution.stateBefore] is chosen.
*
* Only [UtExecution.stateBefore] is considered, because [UtExecution.result] and [UtExecution.stateAfter]
* don't represent true picture as they are limited by [construction depth][UtModelConstructor.maxDepth] and their
* sizes can't be calculated for crushed executions.
*/
private fun List<UtExecution>.chooseMinimalCrashExecution(): UtExecution = minByOrNull {
it.stateBefore.calculateSize()
} ?: error("Cannot find minimal crash execution within empty executions")

private fun EnvironmentModels.calculateSize(): Int {
val thisInstanceSize = thisInstance?.calculateSize() ?: 0
val parametersSize = parameters.sumOf { it.calculateSize() }
val staticsSize = statics.values.sumOf { it.calculateSize() }

return thisInstanceSize + parametersSize + staticsSize
}

/**
* We assume that "size" for "common" models is 1, 0 for [UtVoidModel] (as they do not return anything) and
* [UtPrimitiveModel] and [UtNullModel] (we use them as literals in codegen), summarising for all statements for [UtAssembleModel] and
* summarising for all fields and mocks for [UtCompositeModel]. As [UtCompositeModel] could be recursive, we need to
* store it in [used]. Moreover, if we already calculate size for [this], it means that we will use already created
* variable by this model and do not need to create it again, so size should be equal to 0.
*/
private fun UtModel.calculateSize(used: MutableSet<UtModel> = mutableSetOf()): Int {
if (this in used) return 0

used += this

return when (this) {
is UtNullModel, is UtPrimitiveModel, UtVoidModel -> 0
is UtClassRefModel, is UtEnumConstantModel, is UtArrayModel -> 1
is UtAssembleModel -> {
1 + instantiationCall.calculateSize(used) + modificationsChain.sumOf { it.calculateSize(used) }
}
is UtCompositeModel -> 1 + fields.values.sumOf { it.calculateSize(used) }
is UtLambdaModel -> 1 + capturedValues.sumOf { it.calculateSize(used) }
// PythonModel, JsUtModel, UtSpringContextModel may be here
else -> 0
}
}

private fun UtStatementModel.calculateSize(used: MutableSet<UtModel> = mutableSetOf()): Int =
when (this) {
is UtDirectSetFieldModel -> 1 + fieldModel.calculateSize(used) + instance.calculateSize(used)
is UtStatementCallModel -> 1 + params.sumOf { it.calculateSize(used) } + (instance?.calculateSize(used) ?: 0)
}
private fun List<UtExecution>.chooseOneExecution(): UtExecution = minWithOrNull(
compareBy({ it.getExecutionPriority() }, { it.stateBefore.calculateSize() })
) ?: error("Cannot find minimal execution within empty executions")

/**
* Extends the [instructionsWithoutExtra] with one extra instruction if the [result] is
Expand Down Expand Up @@ -274,6 +240,8 @@ private fun Throwable.exceptionToInfo(): String =
* Returns an execution priority. [UtSymbolicExecution] has the highest priority
* over other executions like [UtFuzzedExecution], [UtFailedExecution], etc.
*
* NOTE! Smaller number represents higher priority.
*
* See [https://github.com/UnitTestBot/UTBotJava/issues/1504] for more details.
*/
private fun UtExecution.getExecutionPriority(): Int = when (this) {
Expand Down
55 changes: 55 additions & 0 deletions utbot-framework/src/main/kotlin/org/utbot/framework/util/SizeUtils.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package org.utbot.framework.util

import org.utbot.framework.plugin.api.EnvironmentModels
import org.utbot.framework.plugin.api.UtArrayModel
import org.utbot.framework.plugin.api.UtAssembleModel
import org.utbot.framework.plugin.api.UtClassRefModel
import org.utbot.framework.plugin.api.UtCompositeModel
import org.utbot.framework.plugin.api.UtDirectSetFieldModel
import org.utbot.framework.plugin.api.UtEnumConstantModel
import org.utbot.framework.plugin.api.UtLambdaModel
import org.utbot.framework.plugin.api.UtModel
import org.utbot.framework.plugin.api.UtNullModel
import org.utbot.framework.plugin.api.UtPrimitiveModel
import org.utbot.framework.plugin.api.UtStatementCallModel
import org.utbot.framework.plugin.api.UtStatementModel
import org.utbot.framework.plugin.api.UtVoidModel

fun EnvironmentModels.calculateSize(): Int {
val thisInstanceSize = thisInstance?.calculateSize() ?: 0
val parametersSize = parameters.sumOf { it.calculateSize() }
val staticsSize = statics.values.sumOf { it.calculateSize() }

return thisInstanceSize + parametersSize + staticsSize
}

/**
* We assume that "size" for "common" models is 1, 0 for [UtVoidModel] (as they do not return anything) and
* [UtPrimitiveModel] and [UtNullModel] (we use them as literals in codegen), summarising for all statements for [UtAssembleModel] and
* summarising for all fields and mocks for [UtCompositeModel]. As [UtCompositeModel] could be recursive, we need to
* store it in [used]. Moreover, if we already calculate size for [this], it means that we will use already created
* variable by this model and do not need to create it again, so size should be equal to 0.
*/
private fun UtModel.calculateSize(used: MutableSet<UtModel> = mutableSetOf()): Int {
if (this in used) return 0

used += this

return when (this) {
is UtNullModel, is UtPrimitiveModel, UtVoidModel -> 0
is UtClassRefModel, is UtEnumConstantModel, is UtArrayModel -> 1
is UtAssembleModel -> {
1 + instantiationCall.calculateSize(used) + modificationsChain.sumOf { it.calculateSize(used) }
}
is UtCompositeModel -> 1 + fields.values.sumOf { it.calculateSize(used) }
is UtLambdaModel -> 1 + capturedValues.sumOf { it.calculateSize(used) }
// PythonModel, JsUtModel, UtSpringContextModel may be here
else -> 0
}
}

private fun UtStatementModel.calculateSize(used: MutableSet<UtModel> = mutableSetOf()): Int =
when (this) {
is UtDirectSetFieldModel -> 1 + fieldModel.calculateSize(used) + instance.calculateSize(used)
is UtStatementCallModel -> 1 + params.sumOf { it.calculateSize(used) } + (instance?.calculateSize(used) ?: 0)
}