Skip to content
Permalink
Browse files

fixed another source of non-determinism

  • Loading branch information
arcuri82 committed Mar 23, 2020
1 parent 70227be commit e15f4a2ce8f788044bed644fb769736e2419dd48
@@ -0,0 +1,24 @@
package org.evomaster.core


/**
* WARNING: here we have mutable static state.
* In general, we should avoid doing something like this.
* But we need to have way to create unique ids withount a dependency injection.
* As this should have extremely low impact on the execution, forgetting to reset
* it at each test case should no impact, apart from very, very special cases (eg,
* when we test for determinism)
*/
class StaticCounter {

companion object{

private var counter = 0

fun getAndIncrease() = counter++

fun reset() {
counter = 0
}
}
}
@@ -457,6 +457,7 @@ class RestSampler : Sampler<RestIndividual>(){
Therefore, to properly test the GET, we might
need to be able to create many elements.
*/
log.trace("Creating POSTs on collection before a GET")
val k = 1 + randomness.nextInt(available)

(0 until k).forEach {
@@ -14,10 +14,16 @@ import org.evomaster.core.search.gene.Gene
import org.evomaster.core.search.service.mutator.MutatedGeneSpecification
import org.evomaster.core.search.service.mutator.StructureMutator
import org.evomaster.core.search.service.mutator.geneMutation.ArchiveMutator
import org.slf4j.Logger
import org.slf4j.LoggerFactory


class RestStructureMutator : StructureMutator() {

companion object {
private val log: Logger = LoggerFactory.getLogger(RestStructureMutator::class.java)
}

@Inject
private lateinit var archiveMutator: ArchiveMutator

@@ -225,6 +231,7 @@ class RestStructureMutator : StructureMutator() {
if (randomness.nextBoolean() || ind.seeActions().size == config.maxTestSize) {

//delete one at random
log.trace("Deleting action from test")
val chosen = randomness.nextInt(ind.seeActions().size)

//save mutated genes
@@ -239,6 +246,7 @@ class RestStructureMutator : StructureMutator() {
} else {

//add one at random
log.trace("Adding action to test")
val sampledAction = sampler.sampleRandomAction(0.05)
val chosen = randomness.nextInt(ind.seeActions().size)
//ind.seeActions().add(chosen, sampledAction)
@@ -93,6 +93,7 @@ class ArrayGene<T>(
//maybe not so important here to complicate code to enable forceNewValue

elements.clear()
log.trace("Randomizing ArrayGene")
val n = randomness.nextInt(maxSize)
(0 until n).forEach {
val gene = template.copy() as T
@@ -114,6 +115,7 @@ class ArrayGene<T>(
gene.randomize(randomness, false)
elements.add(gene)
} else if(elements.size > 0 && randomness.nextBoolean(MODIFY_SIZE)){
log.trace("Remvoving gene in mutation")
elements.removeAt(randomness.nextInt(elements.size))
} else {
val gene = randomness.choose(elements)
@@ -173,6 +175,7 @@ class ArrayGene<T>(
elements.add(gene)
}
delete ->{
log.trace("Removing gene")
elements.removeAt(randomness.nextInt(elements.size))
}
else -> {
@@ -75,6 +75,7 @@ class MapGene<T>(
//maybe not so important here to complicate code to enable forceNewValue

elements.clear()
log.trace("Randomizing MapGene")
val n = randomness.nextInt(maxSize)
(0 until n).forEach {
val gene = template.copy() as T
@@ -99,6 +100,7 @@ class MapGene<T>(
gene.name = "key_${keyCounter++}"
elements.add(gene)
} else if(elements.size > 0 && randomness.nextBoolean(MODIFY_SIZE)){
log.trace("Removing gene in mutation")
elements.removeAt(randomness.nextInt(elements.size))
} else {
val gene = randomness.choose(elements)
@@ -156,6 +158,7 @@ class MapGene<T>(
return
}
delete ->{
log.trace("Deleting gene")
elements.removeAt(randomness.nextInt(elements.size))
return
}
@@ -5,6 +5,7 @@ import org.evomaster.client.java.instrumentation.shared.StringSpecialization
import org.evomaster.client.java.instrumentation.shared.StringSpecialization.*
import org.evomaster.client.java.instrumentation.shared.StringSpecializationInfo
import org.evomaster.client.java.instrumentation.shared.TaintInputName
import org.evomaster.core.StaticCounter
import org.evomaster.core.logging.LoggingUtil
import org.evomaster.core.output.OutputFormat
import org.evomaster.core.parser.RegexHandler
@@ -55,13 +56,6 @@ class StringGene(

private val log: Logger = LoggerFactory.getLogger(StringGene::class.java)

/*
WARNING
mutable static state.
only used to create unique names
*/
private var counter: Int = 0

private const val NEVER_ARCHIVE_MUTATION = -2
private const val CHAR_MUTATION_INITIALIZED = -1
}
@@ -177,7 +171,7 @@ class StringGene(
if (!TaintInputName.isTaintInput(value)
&& randomness.nextBoolean(apc.getBaseTaintAnalysisProbability())) {

value = TaintInputName.getTaintName(counter++)
value = TaintInputName.getTaintName(StaticCounter.getAndIncrease())
return
}

@@ -206,6 +200,7 @@ class StringGene(
p < 0.8 && s.isNotEmpty() -> {
val delta = getDelta(randomness, apc, start = 6, end = 3)
val sign = randomness.choose(listOf(-1, +1))
log.trace("Changing char in: {}", s)
val i = randomness.nextInt(s.length)
val array = s.toCharArray()
array[i] = s[i] + (sign * delta)
@@ -220,6 +215,7 @@ class StringGene(
if (s.isEmpty() || randomness.nextBoolean(0.8)) {
s + randomness.nextWordChar()
} else {
log.trace("Appending char")
val i = randomness.nextInt(s.length)
if (i == 0) {
randomness.nextWordChar() + s
@@ -496,7 +492,7 @@ class StringGene(
if (!TaintInputName.isTaintInput(value)
&& randomness.nextBoolean(apc.getBaseTaintAnalysisProbability())) {

value = TaintInputName.getTaintName(counter++)
value = TaintInputName.getTaintName(StaticCounter.getAndIncrease())
return
}

@@ -5,6 +5,8 @@ import org.evomaster.core.search.gene.Gene
import org.evomaster.core.search.gene.GeneUtils
import org.evomaster.core.search.service.AdaptiveParameterControl
import org.evomaster.core.search.service.Randomness
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.lang.IllegalArgumentException


@@ -15,6 +17,11 @@ class QuantifierRxGene(
val max: Int = 1
) : RxTerm(name) {

companion object {
private val log: Logger = LoggerFactory.getLogger(QuantifierRxGene::class.java)
}


val atoms = mutableListOf<RxAtom>()

/**
@@ -101,6 +108,7 @@ class QuantifierRxGene(
val length = atoms.size

if( length > min && randomness.nextBoolean(0.1)){
log.trace("Removing atom")
atoms.removeAt(randomness.nextInt(length))
} else if(length < limitedMax && randomness.nextBoolean(0.1)){
addNewAtom(randomness, false, listOf())
@@ -79,7 +79,7 @@ class Randomness {

fun nextInt(bound: Int): Int {
val k = random.nextInt(bound)
log.trace("nextInt(): {}", k)
log.trace("nextInt(bound): {} , {}", k, bound)
return k
}

@@ -94,7 +94,7 @@ class Randomness {
while (k == exclude) {
k = nextInt(min, max)
}
log.trace("nextInt(): {}", k)
log.trace("nextInt(min,max,exclude): {}", k)
return k
}

@@ -110,7 +110,7 @@ class Randomness {
}

val k = (min.toLong() + random.nextDouble() * (max.toLong() - min + 1)).toInt()
log.trace("nextInt(): {}", k)
log.trace("nextInt(min,max): {}", k)
return k
}

@@ -207,6 +207,9 @@ class Randomness {
}
temp += v
}

log.trace("Chosen: {}", found)

return found
}

@@ -255,7 +258,11 @@ class Randomness {
selection.addAll(list)
selection.shuffle(random)

return selection.subList(0, n)
val k = selection.subList(0, n)

if(log.isTraceEnabled) log.trace("Chosen: {}", k.joinToString(" "))

return k
}

/**
@@ -270,7 +277,11 @@ class Randomness {
selection.addAll(set)
selection.shuffle(random)

return selection.subList(0, n).toSet()
val k = selection.subList(0, n).toSet()

if(log.isTraceEnabled) log.trace("Chosen: {}", k.joinToString(" "))

return k
}


@@ -292,6 +303,10 @@ class Randomness {
iter.next()
i++
}
return iter.next()

val k = iter.next()
log.trace("Chosen: {}", k)

return k
}
}
@@ -388,6 +388,7 @@ class ArchiveMutator {
if (it != -1) return it
}
}
log.trace("Deciding index")
return randomness.nextInt(gene.value.length)
}

@@ -8,6 +8,7 @@
import org.evomaster.client.java.controller.internal.SutController;
import org.evomaster.client.java.instrumentation.shared.ClassName;
import org.evomaster.core.Main;
import org.evomaster.core.StaticCounter;
import org.evomaster.core.logging.LoggingUtil;
import org.evomaster.core.output.OutputFormat;
import org.evomaster.core.output.compiler.CompilerForTestGenerated;
@@ -41,6 +42,8 @@
protected static RemoteController remoteController;
protected static int controllerPort;

protected int defaultSeed = 42;


@AfterAll
public static void tearDown() {
@@ -85,10 +88,12 @@ protected void runAndCheckDeterminism(int iterations, Consumer<List<String>> lam
"--stoppingCriterion", "FITNESS_EVALUATIONS"
));

StaticCounter.Companion.reset();
String firstRun = LoggingUtil.Companion.runWithDeterministicLogger(
() -> {lambda.accept(args); return Unit.INSTANCE;}
);

StaticCounter.Companion.reset();
String secondRun = LoggingUtil.Companion.runWithDeterministicLogger(
() -> {lambda.accept(args); return Unit.INSTANCE;}
);
@@ -308,7 +313,7 @@ protected void compile(String outputFolderName){

return new ArrayList<>(Arrays.asList(
"--createTests", "" + createTests,
"--seed", "42",
"--seed", "" + defaultSeed,
"--sutControllerPort", "" + controllerPort,
"--maxActionEvaluations", "" + iterations,
"--stoppingCriterion", "FITNESS_EVALUATIONS",
@@ -21,13 +21,23 @@ public static void initClass() throws Exception {
SpringTestBase.initClass(new TaintMultiController());
}

@Test
public void testDeterminism(){

runAndCheckDeterminism(500, (args) -> {
initAndRun(args);
});
}

@Test
public void testRunEM() throws Throwable {

defaultSeed = 13;

runTestHandlingFlakyAndCompilation(
"TaintMultiEM",
"org.bar.TaintMultiEM",
5000,
5_000,
(args) -> {

args.add("--baseTaintAnalysisProbability");

0 comments on commit e15f4a2

Please sign in to comment.
You can’t perform that action at this time.