Skip to content
Permalink
Browse files

different strategies for handling extra heuristics

  • Loading branch information...
arcuri82 committed Jul 22, 2019
1 parent b5b014d commit 9fbe0590836262604fab6b267b3da8acf6077dc8
@@ -60,4 +60,8 @@
*/
public Map<String, Set<String>> failedWhere = new HashMap<>();

/**
* The total Number of SQL commands (e.g., SELECT and UPDATE) executed
*/
public int numberOfSqlCommands = 0;
}
@@ -44,6 +44,7 @@
private final Map<String, Set<String>> failedWhere;
private final List<String> deletedData;

private int numberOfSqlCommands;

private volatile Connection connection;

@@ -59,6 +60,7 @@ public SqlHandler() {
deletedData = new CopyOnWriteArrayList<>();

calculateHeuristics = true;
numberOfSqlCommands = 0;
}

public void reset() {
@@ -69,6 +71,7 @@ public void reset() {
insertedData.clear();
failedWhere.clear();
deletedData.clear();
numberOfSqlCommands = 0;
}

public void setConnection(Connection connection) {
@@ -93,6 +96,8 @@ public void handle(String sql) {
} else if(isUpdate(sql)){
mergeNewData(updatedData, ColumnTableAnalyzer.getUpdatedDataFields(sql));
}

numberOfSqlCommands++;
}

public ExecutionDto getExecutionDto() {
@@ -107,6 +112,7 @@ public ExecutionDto getExecutionDto() {
executionDto.insertedData.putAll(insertedData);
executionDto.updatedData.putAll(updatedData);
executionDto.deletedData.addAll(deletedData);
executionDto.numberOfSqlCommands = this.numberOfSqlCommands;

return executionDto;
}
@@ -441,6 +441,17 @@ class EMConfig {
@Cfg("Where the extra heuristics file (if any) is going to be written (in CSV format)")
var extraHeuristicsFile = "extra_heuristics.csv"


enum class SecondaryObjectiveStrategy{
AVG_DISTANCE,
AVG_DISTANCE_SAME_N_ACTIONS,
BEST_MIN
}

@Cfg("Strategy used to handle the extra heuristics in the secondary objectives")
var secondaryObjectiveStrategy = SecondaryObjectiveStrategy.AVG_DISTANCE


@Cfg("Probability of applying a mutation that can change the structure of a test")
@Min(0.0) @Max(1.0)
var structureMutationProbability = 0.5
@@ -501,6 +512,7 @@ class EMConfig {
@Min(1.0)
var maxSqlInitActionsPerMissingData = 5


@Cfg("Maximum size (in bytes) that EM handles response payloads in the HTTP responses. " +
"If larger than that, a response will not be stored internally in EM during the test generation. "+
"This is needed to avoid running out of memory.")
@@ -20,7 +20,8 @@ class DatabaseExecution(
val updatedData: Map<String, Set<String>>,
val insertedData: Map<String, Set<String>>,
val failedWhere: Map<String, Set<String>>,
val deletedData: List<String>
val deletedData: List<String>,
val numberOfSqlCommands: Int
) {

companion object {
@@ -32,7 +33,8 @@ class DatabaseExecution(
cloneData(dto?.updatedData),
cloneData(dto?.insertedData),
cloneData(dto?.failedWhere),
dto?.deletedData?.toList() ?: listOf()
dto?.deletedData?.toList() ?: listOf(),
dto?.numberOfSqlCommands ?: 0
)
}

@@ -1,6 +1,9 @@
package org.evomaster.core.search

import org.evomaster.core.EMConfig
import org.evomaster.core.database.DatabaseExecution
import org.evomaster.core.EMConfig.SecondaryObjectiveStrategy.*
import kotlin.math.min

/**
As the number of targets is unknown, we cannot have
@@ -72,7 +75,7 @@ class FitnessValue(

/**
* We keep track of DB interactions per action.
* However, there are are cases in which we only care of aggregated data for all actions.
* However, there are cases in which we only care of aggregated data for all actions.
* Instead of re-computing them each time, we just do it once and save the results
*/
fun aggregateDatabaseData(){
@@ -160,7 +163,11 @@ class FitnessValue(
* @param other, the one we compare to
* @param targetSubset, only calculate subsumption on these testing targets
*/
fun subsumes(other: FitnessValue, targetSubset: Set<Int>): Boolean {
fun subsumes(
other: FitnessValue,
targetSubset: Set<Int>,
strategy: EMConfig.SecondaryObjectiveStrategy)
: Boolean {

var atLeastOneBetter = false

@@ -172,7 +179,7 @@ class FitnessValue(
return false
}

val extra = compareExtraToMinimize(k, other)
val extra = compareExtraToMinimize(k, other, strategy)

if (v > z ||
(v == z && extra > 0) ||
@@ -184,39 +191,58 @@ class FitnessValue(
return atLeastOneBetter
}

fun averageExtraDistancesToMinimize(actionIndex: Int): Double{
return averageDistance(extraToMinimize[actionIndex])
}

/**
* Compare the extra heuristics between this and [other].
*
* @return 0 if equivalent, 1 if this is better, and -1 otherwise
*/
fun compareExtraToMinimize(target: Int, other: FitnessValue): Int {

//TODO parameter to experiment with
//return compareByRewardMore(other)
return compareByReduce(target, other)
fun compareExtraToMinimize(
target: Int,
other: FitnessValue,
strategy: EMConfig.SecondaryObjectiveStrategy)
: Int {

return when(strategy){
AVG_DISTANCE -> compareAverage(target, other)
AVG_DISTANCE_SAME_N_ACTIONS -> compareAverageSameNActions(target, other)
BEST_MIN -> compareByBestMin(target, other)
}
}

fun averageExtraDistancesToMinimize(actionIndex: Int): Double{
return aggregateDistances(extraToMinimize[actionIndex])
}

private fun aggregateDistances(distances: List<Double>?): Double {
private fun averageDistance(distances: List<Double>?): Double {
if (distances == null || distances.isEmpty()) {
return Double.MAX_VALUE
return 0.0
}

val sum = distances.map { v -> v / distances.size }.sum()

return sum
}

private fun compareByReduce(target: Int, other: FitnessValue): Int {
private fun compareAverageSameNActions(target: Int, other: FitnessValue): Int {

val thisN = databaseExecutions[target]?.numberOfSqlCommands ?: 0
val otherN = other.databaseExecutions[target]?.numberOfSqlCommands ?: 0

return when {
thisN > otherN -> 1
thisN < otherN -> -1
else -> compareAverage(target, other)
}
}

private fun compareAverage(target: Int, other: FitnessValue): Int {

val thisAction = targets[target]?.actionIndex
val otherAction = other.targets[target]?.actionIndex

val ts = aggregateDistances(this.extraToMinimize[thisAction])
val os = aggregateDistances(other.extraToMinimize[otherAction])
val ts = averageDistance(this.extraToMinimize[thisAction])
val os = averageDistance(other.extraToMinimize[otherAction])

return when {
ts < os -> +1
@@ -225,54 +251,48 @@ class FitnessValue(
}
}

/*
TODO: add back and fix. Use for experiments.
*/

// private fun compareByRewardMore(other: FitnessValue): Int {
// val thisLength = this.extraToMinimize.size
// val otherLength = other.extraToMinimize.size
// val minLen = Math.min(thisLength, otherLength)
//
// if (minLen > 0) {
// for (i in 0..(minLen - 1)) {
// val te = this.extraToMinimize[i]
// val oe = other.extraToMinimize[i]
//
// /*
// We prioritize the improvement of lowest
// heuristics, as more likely to be covered (ie 0)
// first.
// */
//
// if (te < oe) {
// return +1
// } else if (te > oe) {
// return -1
// }
// }
// }
//
// if (thisLength == otherLength) {
// return 0
// }
//
// /*
// up to min size, same values of the heuristics.
// But one test is doing more stuff, as it has more extra
// heuristics. And so we reward it.
//
// However, there is big risk of bloat. So, let's put
// an arbitrary low limit.
// */
// if (minLen >= 3) { //TODO should be a parameter to experiment with
// return 0
// }
//
// if (thisLength > otherLength) {
// return +1
// } else {
// return -1
// }
// }
private fun compareByBestMin(target: Int, other: FitnessValue): Int {
val thisLength = this.extraToMinimize[target]?.size ?: 0
val otherLength = other.extraToMinimize[target]?.size ?: 0
val minLen = min(thisLength, otherLength)

if (minLen > 0) {
for (i in 0 until minLen) {
val te = this.extraToMinimize[target]!![i]
val oe = other.extraToMinimize[target]!![i]

/*
We prioritize the improvement of lowest
heuristics, as more likely to be covered (ie 0)
first.
*/

if (te < oe) {
return +1
} else if (te > oe) {
return -1
}
}
}

val thisN = databaseExecutions[target]?.numberOfSqlCommands ?: 0
val otherN = other.databaseExecutions[target]?.numberOfSqlCommands ?: 0

/*
if same min, reward number of SQL commands: if more, the
better.
If even with that we cannot make a choice, then reward
the one with less heuristics, as that mean it had more
success with the SQL commands
*/

return when {
thisN > otherN -> 1
thisN < otherN -> -1
thisLength > otherLength -> -1
thisLength < otherLength -> 1
else -> 0
}
}
}
@@ -337,7 +337,7 @@ class Archive<T> where T : Individual {
val currh = current[0].fitness.getHeuristic(k)
val currsize = current[0].individual.size()
val copySize = copy.individual.size()
val extra = copy.fitness.compareExtraToMinimize(k, current[0].fitness)
val extra = copy.fitness.compareExtraToMinimize(k, current[0].fitness, config.secondaryObjectiveStrategy)

val better = v.distance > currh ||
(v.distance == currh && extra > 0) ||
@@ -391,7 +391,7 @@ class Archive<T> where T : Individual {

list.sortWith(compareBy<EvaluatedIndividual<T>>
{ it.fitness.getHeuristic(target) }
.thenComparator { a, b -> a.fitness.compareExtraToMinimize(target, b.fitness) }
.thenComparator { a, b -> a.fitness.compareExtraToMinimize(target, b.fitness, config.secondaryObjectiveStrategy) }
.thenBy { -it.individual.size() })

val limit = apc.getArchiveTargetLimit()
@@ -92,7 +92,7 @@ abstract class Mutator<T> : TrackOperator where T : Individual {

val reachNew = archive.wouldReachNewTarget(mutated)

if (reachNew || !current.fitness.subsumes(mutated.fitness, targets)) {
if (reachNew || !current.fitness.subsumes(mutated.fitness, targets, config.secondaryObjectiveStrategy)) {
val trackedMutated = if(config.enableTrackEvaluatedIndividual) trackedCurrent.next(this, mutated)!! else mutated
archive.addIfNeeded(trackedMutated)
current = trackedMutated
@@ -3,6 +3,7 @@
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import org.evomaster.core.EMConfig;
import org.evomaster.core.Main;
import org.evomaster.core.database.DbAction;
import org.evomaster.core.problem.rest.HttpVerb;
@@ -154,7 +155,7 @@ public void testSteps() {
noDataFV.averageExtraDistancesToMinimize(0));

for (int target : noDataFV.getViewOfData().keySet()) {
assertTrue(closeDataFV.compareExtraToMinimize(target, noDataFV) >= 0);
assertTrue(closeDataFV.compareExtraToMinimize(target, noDataFV, EMConfig.SecondaryObjectiveStrategy.AVG_DISTANCE) >= 0);
}

//but still not reaching target

0 comments on commit 9fbe059

Please sign in to comment.
You can’t perform that action at this time.