Apply new optimal set of settings after tuning

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -1123,11 +1123,11 @@ class EMConfig {
 
     @Cfg("Probability of sampling a new individual at random")
     @Probability
-    var probOfRandomSampling = 0.5
+    var probOfRandomSampling = 0.8
 
     @Cfg("The percentage of passed search before starting a more focused, less exploratory one")
     @PercentageAsProbability(true)
-    var focusedSearchActivationTime = 0.5
+    var focusedSearchActivationTime = 0.8
 
     @Cfg("Number of applied mutations on sampled individuals, at the start of the search")
     @Min(0.0)
@@ -1265,7 +1265,7 @@ class EMConfig {
     }
 
     @Cfg("Specify whether when we sample from archive we do look at the most promising targets for which we have had a recent improvement")
-    var feedbackDirectedSampling = FeedbackDirectedSampling.LAST
+    var feedbackDirectedSampling = FeedbackDirectedSampling.FOCUSED_QUICKEST
 
     //Warning: this is off in the tests, as it is a source of non-determinism
     @Cfg("Whether to use timestamp info on the execution time of the tests for sampling (e.g., to reward the quickest ones)")
@@ -1335,7 +1335,7 @@ class EMConfig {
 
     @Cfg("When generating SQL data, how many new rows (max) to generate for each specific SQL Select")
     @Min(1.0)
-    var maxSqlInitActionsPerMissingData = 5
+    var maxSqlInitActionsPerMissingData = 1
 
 
     @Cfg("Force filling data of all columns when inserting new row, instead of only minimal required set.")
@@ -1528,7 +1528,7 @@ class EMConfig {
 
     @Cfg("Specify a probability to apply SQL actions for preparing resources for REST Action")
     @Probability
-    var probOfApplySQLActionToCreateResources = 0.5
+    var probOfApplySQLActionToCreateResources = 0.1
 
     @Experimental
     @Cfg("Specify a maximum number of handling (remove/add) resource size at once, e.g., add 3 resource at most")
@@ -1853,7 +1853,7 @@ class EMConfig {
 
     @Cfg("Probability to use input tracking (i.e., a simple base form of taint-analysis) to determine how inputs are used in the SUT")
     @Probability
-    var baseTaintAnalysisProbability = 0.9
+    var baseTaintAnalysisProbability = 0.5
 
     @Cfg("Whether input tracking is used on sampling time, besides mutation time")
     var taintOnSampling = true

core/src/main/kotlin/org/evomaster/core/search/service/Randomness.kt

@@ -412,7 +412,10 @@ class Randomness {
 
         val k =  selection.subList(0, n)
 
-        if(log.isTraceEnabled) log.trace("Chosen: {}", k.joinToString(" "))
+        //printing actual values here lead to non-deterministic behavior is toString() is non-deterministic,
+        //which is the typical case for custom objects that do not override it, as output string will have
+        // a @ reference number to the heap
+        log.trace("Chosen {} elements from list", n)
 
         return k
     }
@@ -431,7 +434,7 @@ class Randomness {
 
         val k = selection.subList(0, n).toSet()
 
-        if(log.isTraceEnabled) log.trace("Chosen: {}", k.joinToString(" "))
+        log.trace("Chosen {} elements from set", n)
 
         return k
     }
@@ -457,7 +460,7 @@ class Randomness {
         }
 
         val k = iter.next()
-        log.trace("Chosen: {}", k)
+        log.trace("Chosen index: {}", i)
 
         return k
     }

docs/options.md

@@ -68,7 +68,7 @@ There are 3 types of options:
 |`archiveGeneMutation`| __Enum__. Whether to enable archive-based gene mutation. *Valid values*: `NONE, SPECIFIED, SPECIFIED_WITH_TARGETS, SPECIFIED_WITH_SPECIFIC_TARGETS, SPECIFIED_WITH_TARGETS_DIRECTION, SPECIFIED_WITH_SPECIFIC_TARGETS_DIRECTION, ADAPTIVE`. *Default value*: `SPECIFIED_WITH_SPECIFIC_TARGETS`.|
 |`archiveTargetLimit`| __Int__. Limit of number of individuals per target to keep in the archive. *Constraints*: `min=1.0`. *Default value*: `10`.|
 |`avoidNonDeterministicLogs`| __Boolean__. At times, we need to run EvoMaster with printed logs that are deterministic. For example, this means avoiding printing out time-stamps. *Default value*: `false`.|
-|`baseTaintAnalysisProbability`| __Double__. Probability to use input tracking (i.e., a simple base form of taint-analysis) to determine how inputs are used in the SUT. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.9`.|
+|`baseTaintAnalysisProbability`| __Double__. Probability to use input tracking (i.e., a simple base form of taint-analysis) to determine how inputs are used in the SUT. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
 |`bbExperiments`| __Boolean__. Only used when running experiments for black-box mode, where an EvoMaster Driver would be present, and can reset state after each experiment. *Default value*: `false`.|
 |`bloatControlForSecondaryObjective`| __Boolean__. Whether secondary objectives are less important than test bloat control. *Default value*: `false`.|
 |`coveredTargetFile`| __String__. Specify a file which saves covered targets info regarding generated test suite. *Default value*: `coveredTargets.txt`.|
@@ -108,8 +108,8 @@ There are 3 types of options:
 |`extraHeuristicsFile`| __String__. Where the extra heuristics file (if any) is going to be written (in CSV format). *Default value*: `extra_heuristics.csv`.|
 |`extraQueryParam`| __Boolean__. Add an extra query param, to analyze how it is used/read by the SUT. Needed to discover new query params that were not specified in the schema. *Default value*: `true`.|
 |`extractSqlExecutionInfo`| __Boolean__. Enable extracting SQL execution info. *Default value*: `true`.|
-|`feedbackDirectedSampling`| __Enum__. Specify whether when we sample from archive we do look at the most promising targets for which we have had a recent improvement. *Valid values*: `NONE, LAST, FOCUSED_QUICKEST`. *Default value*: `LAST`.|
-|`focusedSearchActivationTime`| __Double__. The percentage of passed search before starting a more focused, less exploratory one. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
+|`feedbackDirectedSampling`| __Enum__. Specify whether when we sample from archive we do look at the most promising targets for which we have had a recent improvement. *Valid values*: `NONE, LAST, FOCUSED_QUICKEST`. *Default value*: `FOCUSED_QUICKEST`.|
+|`focusedSearchActivationTime`| __Double__. The percentage of passed search before starting a more focused, less exploratory one. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.|
 |`forceSqlAllColumnInsertion`| __Boolean__. Force filling data of all columns when inserting new row, instead of only minimal required set. *Default value*: `true`.|
 |`geneMutationStrategy`| __Enum__. Strategy used to define the mutation probability. *Valid values*: `ONE_OVER_N, ONE_OVER_N_BIASED_SQL`. *Default value*: `ONE_OVER_N_BIASED_SQL`.|
 |`geneWeightBasedOnImpactsBy`| __Enum__. Specify a strategy to calculate a weight of a gene based on impacts. *Valid values*: `SORT_COUNTER, SORT_RATIO, COUNTER, RATIO`. *Default value*: `RATIO`.|
@@ -137,7 +137,7 @@ There are 3 types of options:
 |`maxLengthOfTraces`| __Int__. Specify a maxLength of tracking when enableTrackIndividual or enableTrackEvaluatedIndividual is true. Note that the value should be specified with a non-negative number or -1 (for tracking all history). *Constraints*: `min=-1.0`. *Default value*: `10`.|
 |`maxResponseByteSize`| __Int__. Maximum size (in bytes) that EM handles response payloads in the HTTP responses. If larger than that, a response will not be stored internally in EM during the test generation. This is needed to avoid running out of memory. *Default value*: `1000000`.|
 |`maxSearchSuiteSize`| __Int__. Define the maximum number of tests in a suite in the search algorithms that evolve whole suites, e.g. WTS. *Constraints*: `min=1.0`. *Default value*: `50`.|
-|`maxSqlInitActionsPerMissingData`| __Int__. When generating SQL data, how many new rows (max) to generate for each specific SQL Select. *Constraints*: `min=1.0`. *Default value*: `5`.|
+|`maxSqlInitActionsPerMissingData`| __Int__. When generating SQL data, how many new rows (max) to generate for each specific SQL Select. *Constraints*: `min=1.0`. *Default value*: `1`.|
 |`maxTestSize`| __Int__. Max number of 'actions' (e.g., RESTful calls or SQL commands) that can be done in a single test. *Constraints*: `min=1.0`. *Default value*: `10`.|
 |`maxTimeInSeconds`| __Int__. Maximum number of seconds allowed for the search. The more time is allowed, the better results one can expect. But then of course the test generation will take longer. Only applicable depending on the stopping criterion. If this value is 0, the setting 'maxTime' will be used instead. *Constraints*: `min=0.0`. *Default value*: `0`.|
 |`maxlengthOfHistoryForAGM`| __Int__. Specify a maximum length of history when applying archive-based gene mutation. *Default value*: `10`.|
@@ -150,11 +150,11 @@ There are 3 types of options:
 |`mutatedGeneFile`| __String__. Specify a path to save mutation details which is useful for debugging mutation. *DEBUG option*. *Default value*: `mutatedGeneInfo.csv`.|
 |`outputExecutedSQL`| __Enum__. Whether to output executed sql info. *DEBUG option*. *Valid values*: `NONE, ALL_AT_END, ONCE_EXECUTED`. *Default value*: `NONE`.|
 |`populationSize`| __Int__. Define the population size in the search algorithms that use populations (e.g., Genetic Algorithms, but not MIO). *Constraints*: `min=1.0`. *Default value*: `30`.|
-|`probOfApplySQLActionToCreateResources`| __Double__. Specify a probability to apply SQL actions for preparing resources for REST Action. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
+|`probOfApplySQLActionToCreateResources`| __Double__. Specify a probability to apply SQL actions for preparing resources for REST Action. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.1`.|
 |`probOfArchiveMutation`| __Double__. Specify a probability to enable archive-based mutation. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
 |`probOfEnablingResourceDependencyHeuristics`| __Double__. Specify whether to enable resource dependency heuristics, i.e, probOfEnablingResourceDependencyHeuristics > 0.0. Note that the option is available to be enabled only if resource-based smart sampling is enable. This option has an effect on sampling multiple resources and mutating a structure of an individual. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.95`.|
 |`probOfEnablingSingleInsertionForTable`| __Double__. a probability of enabling single insertion strategy to insert rows into database. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
-|`probOfRandomSampling`| __Double__. Probability of sampling a new individual at random. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
+|`probOfRandomSampling`| __Double__. Probability of sampling a new individual at random. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.|
 |`probOfSelectFromDatabase`| __Double__. Specify a probability that enables selection (i.e., SELECT sql) of data from database instead of insertion (i.e., INSERT sql) for preparing resources for REST actions. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.1`.|
 |`probOfSmartSampling`| __Double__. When sampling new test cases to evaluate, probability of using some smart strategy instead of plain random. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.95`.|
 |`probRestDefault`| __Double__. In REST, specify probability of using 'default' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.|

e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java

@@ -26,7 +26,7 @@ public void testRunAdaptiveHypermutation() throws Throwable {
         runTestHandlingFlakyAndCompilation(
                 "hypermtation/TestLowWeightHighImpact",
                 "org.adaptivehypermuation.LowWeightHighImpactTest",
-                500,
+                1000,
                 true,
                 (args) -> {
 

e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/taintnested/TaintNestedEMTest.java

@@ -32,6 +32,8 @@ public void testRunEM() throws Throwable {
                     args.add("true");
                     args.add("--discoveredInfoRewardedInFitness");
                     args.add("true");
+                    args.add("--baseTaintAnalysisProbability");
+                    args.add("0.9");
 
                     Solution<RestIndividual> solution = initAndRun(args);
 

e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/testability/TestabilityEMTest.java

@@ -35,8 +35,8 @@ public void testRunEM() throws Throwable {
                 15_000,
                 true,
                 (args) -> {
-                    args.add("--baseTaintAnalysisProbability");
-                    args.add("0.9");
+//                    args.add("--baseTaintAnalysisProbability");
+//                    args.add("0.9");
 
                     args.add("--enableTrackEvaluatedIndividual");
                     args.add("false");

e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/valid/ValidEMTest.java

@@ -24,6 +24,9 @@ public void testRunEM() throws Throwable {
                 1000,
                 (args) -> {
 
+                    args.add("--baseTaintAnalysisProbability");
+                    args.add("0.9");
+
                     Solution<RestIndividual> solution = initAndRun(args);
 
                     assertHasAtLeastOne(solution, HttpVerb.POST, 400, "/api/valid", null);

e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/queryparamarray/QueryParamArrayEMTest.kt

@@ -22,7 +22,7 @@ class QueryParamArrayEMTest : SpringTestBase() {
         runTestHandlingFlakyAndCompilation(
             "QueryParamArrayEMT",
             "org.foo.QueryParamArrayEMT",
-            20
+            50
         ) { args: List<String> ->
 
             val solution = initAndRun(args)

e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/hypermutation/AdaptiveHypermutationEMTest.java

@@ -55,6 +55,9 @@ public void testRunEM() throws Throwable {
                     args.add("--minimize");
                     args.add("false");
 
+                    args.add("--focusedSearchActivationTime");
+                    args.add("0.5");
+
                     Solution<RPCIndividual> solution = initAndRun(args);
 
                     boolean ok = check(solution, "lowWeightHighCoverage",1);

e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/hypermutation/HypermutationEMTest.java

@@ -57,6 +57,9 @@ public void testRunEM() throws Throwable {
                     args.add("--minimize");
                     args.add("false");
 
+                    args.add("--probOfRandomSampling");
+                    args.add("0.0");
+
                     Solution<RPCIndividual> solution = initAndRun(args);
 
                     boolean ok = check(solution, "differentWeight",0);