From a1c175e6e1e685cecf4912afb74d70a36f4da499 Mon Sep 17 00:00:00 2001 From: Amid Gol Date: Thu, 11 Apr 2024 19:23:37 +0200 Subject: [PATCH 1/9] Apply new optimal set of settings after tuning This change is based on the results of the tuning paper. We found a new set of configs to be optimal, detecting a higher number of faults while keeping a competitive degree of line and branch coverage. This setting is referred to as "best faults" in the paper. --- core/src/main/kotlin/org/evomaster/core/EMConfig.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 2869d8b15f..1188a0e534 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -1094,11 +1094,11 @@ class EMConfig { @Cfg("Probability of sampling a new individual at random") @Probability - var probOfRandomSampling = 0.5 + var probOfRandomSampling = 0.8 @Cfg("The percentage of passed search before starting a more focused, less exploratory one") @PercentageAsProbability(true) - var focusedSearchActivationTime = 0.5 + var focusedSearchActivationTime = 0.8 @Cfg("Number of applied mutations on sampled individuals, at the start of the search") @Min(0.0) @@ -1236,7 +1236,7 @@ class EMConfig { } @Cfg("Specify whether when we sample from archive we do look at the most promising targets for which we have had a recent improvement") - var feedbackDirectedSampling = FeedbackDirectedSampling.LAST + var feedbackDirectedSampling = FeedbackDirectedSampling.FOCUSED_QUICKEST //Warning: this is off in the tests, as it is a source of non-determinism @Cfg("Whether to use timestamp info on the execution time of the tests for sampling (e.g., to reward the quickest ones)") @@ -1306,7 +1306,7 @@ class EMConfig { @Cfg("When generating SQL data, how many new rows (max) to generate for each specific SQL Select") @Min(1.0) - var maxSqlInitActionsPerMissingData = 5 + var maxSqlInitActionsPerMissingData = 1 @Cfg("Force filling data of all columns when inserting new row, instead of only minimal required set.") @@ -1499,7 +1499,7 @@ class EMConfig { @Cfg("Specify a probability to apply SQL actions for preparing resources for REST Action") @Probability - var probOfApplySQLActionToCreateResources = 0.5 + var probOfApplySQLActionToCreateResources = 0.1 @Experimental @Cfg("Specify a maximum number of handling (remove/add) resource size at once, e.g., add 3 resource at most") @@ -1824,7 +1824,7 @@ class EMConfig { @Cfg("Probability to use input tracking (i.e., a simple base form of taint-analysis) to determine how inputs are used in the SUT") @Probability - var baseTaintAnalysisProbability = 0.9 + var baseTaintAnalysisProbability = 0.5 @Cfg("Whether input tracking is used on sampling time, besides mutation time") var taintOnSampling = true From b253abc1cac462bcf7241cc7d77024d52dc0789b Mon Sep 17 00:00:00 2001 From: Amid Gol Date: Tue, 16 Apr 2024 22:11:51 +0200 Subject: [PATCH 2/9] Update options.md --- docs/options.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/options.md b/docs/options.md index 70e5c25388..7b2fd4a3a1 100644 --- a/docs/options.md +++ b/docs/options.md @@ -68,7 +68,7 @@ There are 3 types of options: |`archiveGeneMutation`| __Enum__. Whether to enable archive-based gene mutation. *Valid values*: `NONE, SPECIFIED, SPECIFIED_WITH_TARGETS, SPECIFIED_WITH_SPECIFIC_TARGETS, SPECIFIED_WITH_TARGETS_DIRECTION, SPECIFIED_WITH_SPECIFIC_TARGETS_DIRECTION, ADAPTIVE`. *Default value*: `SPECIFIED_WITH_SPECIFIC_TARGETS`.| |`archiveTargetLimit`| __Int__. Limit of number of individuals per target to keep in the archive. *Constraints*: `min=1.0`. *Default value*: `10`.| |`avoidNonDeterministicLogs`| __Boolean__. At times, we need to run EvoMaster with printed logs that are deterministic. For example, this means avoiding printing out time-stamps. *Default value*: `false`.| -|`baseTaintAnalysisProbability`| __Double__. Probability to use input tracking (i.e., a simple base form of taint-analysis) to determine how inputs are used in the SUT. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.9`.| +|`baseTaintAnalysisProbability`| __Double__. Probability to use input tracking (i.e., a simple base form of taint-analysis) to determine how inputs are used in the SUT. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`bbExperiments`| __Boolean__. Only used when running experiments for black-box mode, where an EvoMaster Driver would be present, and can reset state after each experiment. *Default value*: `false`.| |`bloatControlForSecondaryObjective`| __Boolean__. Whether secondary objectives are less important than test bloat control. *Default value*: `false`.| |`coveredTargetFile`| __String__. Specify a file which saves covered targets info regarding generated test suite. *Default value*: `coveredTargets.txt`.| @@ -108,8 +108,8 @@ There are 3 types of options: |`extraHeuristicsFile`| __String__. Where the extra heuristics file (if any) is going to be written (in CSV format). *Default value*: `extra_heuristics.csv`.| |`extraQueryParam`| __Boolean__. Add an extra query param, to analyze how it is used/read by the SUT. Needed to discover new query params that were not specified in the schema. *Default value*: `true`.| |`extractSqlExecutionInfo`| __Boolean__. Enable extracting SQL execution info. *Default value*: `true`.| -|`feedbackDirectedSampling`| __Enum__. Specify whether when we sample from archive we do look at the most promising targets for which we have had a recent improvement. *Valid values*: `NONE, LAST, FOCUSED_QUICKEST`. *Default value*: `LAST`.| -|`focusedSearchActivationTime`| __Double__. The percentage of passed search before starting a more focused, less exploratory one. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| +|`feedbackDirectedSampling`| __Enum__. Specify whether when we sample from archive we do look at the most promising targets for which we have had a recent improvement. *Valid values*: `NONE, LAST, FOCUSED_QUICKEST`. *Default value*: `FOCUSED_QUICKEST`.| +|`focusedSearchActivationTime`| __Double__. The percentage of passed search before starting a more focused, less exploratory one. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.| |`forceSqlAllColumnInsertion`| __Boolean__. Force filling data of all columns when inserting new row, instead of only minimal required set. *Default value*: `true`.| |`geneMutationStrategy`| __Enum__. Strategy used to define the mutation probability. *Valid values*: `ONE_OVER_N, ONE_OVER_N_BIASED_SQL`. *Default value*: `ONE_OVER_N_BIASED_SQL`.| |`geneWeightBasedOnImpactsBy`| __Enum__. Specify a strategy to calculate a weight of a gene based on impacts. *Valid values*: `SORT_COUNTER, SORT_RATIO, COUNTER, RATIO`. *Default value*: `RATIO`.| @@ -137,7 +137,7 @@ There are 3 types of options: |`maxLengthOfTraces`| __Int__. Specify a maxLength of tracking when enableTrackIndividual or enableTrackEvaluatedIndividual is true. Note that the value should be specified with a non-negative number or -1 (for tracking all history). *Constraints*: `min=-1.0`. *Default value*: `10`.| |`maxResponseByteSize`| __Int__. Maximum size (in bytes) that EM handles response payloads in the HTTP responses. If larger than that, a response will not be stored internally in EM during the test generation. This is needed to avoid running out of memory. *Default value*: `1000000`.| |`maxSearchSuiteSize`| __Int__. Define the maximum number of tests in a suite in the search algorithms that evolve whole suites, e.g. WTS. *Constraints*: `min=1.0`. *Default value*: `50`.| -|`maxSqlInitActionsPerMissingData`| __Int__. When generating SQL data, how many new rows (max) to generate for each specific SQL Select. *Constraints*: `min=1.0`. *Default value*: `5`.| +|`maxSqlInitActionsPerMissingData`| __Int__. When generating SQL data, how many new rows (max) to generate for each specific SQL Select. *Constraints*: `min=1.0`. *Default value*: `1`.| |`maxTestSize`| __Int__. Max number of 'actions' (e.g., RESTful calls or SQL commands) that can be done in a single test. *Constraints*: `min=1.0`. *Default value*: `10`.| |`maxTimeInSeconds`| __Int__. Maximum number of seconds allowed for the search. The more time is allowed, the better results one can expect. But then of course the test generation will take longer. Only applicable depending on the stopping criterion. If this value is 0, the setting 'maxTime' will be used instead. *Constraints*: `min=0.0`. *Default value*: `0`.| |`maxlengthOfHistoryForAGM`| __Int__. Specify a maximum length of history when applying archive-based gene mutation. *Default value*: `10`.| @@ -150,11 +150,11 @@ There are 3 types of options: |`mutatedGeneFile`| __String__. Specify a path to save mutation details which is useful for debugging mutation. *DEBUG option*. *Default value*: `mutatedGeneInfo.csv`.| |`outputExecutedSQL`| __Enum__. Whether to output executed sql info. *DEBUG option*. *Valid values*: `NONE, ALL_AT_END, ONCE_EXECUTED`. *Default value*: `NONE`.| |`populationSize`| __Int__. Define the population size in the search algorithms that use populations (e.g., Genetic Algorithms, but not MIO). *Constraints*: `min=1.0`. *Default value*: `30`.| -|`probOfApplySQLActionToCreateResources`| __Double__. Specify a probability to apply SQL actions for preparing resources for REST Action. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| +|`probOfApplySQLActionToCreateResources`| __Double__. Specify a probability to apply SQL actions for preparing resources for REST Action. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.1`.| |`probOfArchiveMutation`| __Double__. Specify a probability to enable archive-based mutation. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`probOfEnablingResourceDependencyHeuristics`| __Double__. Specify whether to enable resource dependency heuristics, i.e, probOfEnablingResourceDependencyHeuristics > 0.0. Note that the option is available to be enabled only if resource-based smart sampling is enable. This option has an effect on sampling multiple resources and mutating a structure of an individual. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.95`.| |`probOfEnablingSingleInsertionForTable`| __Double__. a probability of enabling single insertion strategy to insert rows into database. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| -|`probOfRandomSampling`| __Double__. Probability of sampling a new individual at random. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| +|`probOfRandomSampling`| __Double__. Probability of sampling a new individual at random. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.| |`probOfSelectFromDatabase`| __Double__. Specify a probability that enables selection (i.e., SELECT sql) of data from database instead of insertion (i.e., INSERT sql) for preparing resources for REST actions. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.1`.| |`probOfSmartSampling`| __Double__. When sampling new test cases to evaluate, probability of using some smart strategy instead of plain random. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.95`.| |`probRestDefault`| __Double__. In REST, specify probability of using 'default' values, if any is specified in the schema. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.| From e053465254041368fc801f382521654ee56ed8ea Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 8 May 2024 14:50:12 +0200 Subject: [PATCH 3/9] needed more budget --- .../spring/openapi/v3/queryparamarray/QueryParamArrayEMTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/queryparamarray/QueryParamArrayEMTest.kt b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/queryparamarray/QueryParamArrayEMTest.kt index e6d48a724b..54b0ae76fa 100644 --- a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/queryparamarray/QueryParamArrayEMTest.kt +++ b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/queryparamarray/QueryParamArrayEMTest.kt @@ -22,7 +22,7 @@ class QueryParamArrayEMTest : SpringTestBase() { runTestHandlingFlakyAndCompilation( "QueryParamArrayEMT", "org.foo.QueryParamArrayEMT", - 20 + 50 ) { args: List -> val solution = initAndRun(args) From fab78a09376fec377464a8632e233007f59e45ff Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 22 May 2024 14:38:37 +0200 Subject: [PATCH 4/9] fixed non-determinism --- .../org/evomaster/core/search/service/Randomness.kt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Randomness.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Randomness.kt index 4e29adb812..a597a401fc 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Randomness.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Randomness.kt @@ -412,7 +412,10 @@ class Randomness { val k = selection.subList(0, n) - if(log.isTraceEnabled) log.trace("Chosen: {}", k.joinToString(" ")) + //printing actual values here lead to non-deterministic behavior is toString() is non-deterministic, + //which is the typical case for custom objects that do not override it, as output string will have + // a @ reference number to the heap + log.trace("Chosen {} elements from list", n) return k } @@ -431,7 +434,7 @@ class Randomness { val k = selection.subList(0, n).toSet() - if(log.isTraceEnabled) log.trace("Chosen: {}", k.joinToString(" ")) + log.trace("Chosen {} elements from set", n) return k } @@ -457,7 +460,7 @@ class Randomness { } val k = iter.next() - log.trace("Chosen: {}", k) + log.trace("Chosen index: {}", i) return k } From 2ba9c057180d3c780d44989be906674912e570a4 Mon Sep 17 00:00:00 2001 From: Amid Gol Date: Fri, 31 May 2024 18:25:20 +0200 Subject: [PATCH 5/9] Manipulate parameters for the failing tests --- .../examples/hypermutation/AdaptiveHypermutationTest.java | 2 ++ .../examples/hypermutation/AdaptiveHypermutationEMTest.java | 3 +++ .../spring/rpc/examples/hypermutation/HypermutationEMTest.java | 3 +++ 3 files changed, 8 insertions(+) diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java index cfffb27699..19f0e23e61 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java @@ -49,6 +49,8 @@ public void testRunAdaptiveHypermutation() throws Throwable { args.add("--minimize"); args.add("false"); + args.add("--focusedSearchActivationTime"); + args.add("0.5"); Solution solution = initAndRun(args); diff --git a/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/hypermutation/AdaptiveHypermutationEMTest.java b/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/hypermutation/AdaptiveHypermutationEMTest.java index 339928846a..88162c39fa 100644 --- a/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/hypermutation/AdaptiveHypermutationEMTest.java +++ b/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/hypermutation/AdaptiveHypermutationEMTest.java @@ -55,6 +55,9 @@ public void testRunEM() throws Throwable { args.add("--minimize"); args.add("false"); + args.add("--focusedSearchActivationTime"); + args.add("0.5"); + Solution solution = initAndRun(args); boolean ok = check(solution, "lowWeightHighCoverage",1); diff --git a/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/hypermutation/HypermutationEMTest.java b/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/hypermutation/HypermutationEMTest.java index 57f34f0636..13efb3b859 100644 --- a/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/hypermutation/HypermutationEMTest.java +++ b/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/hypermutation/HypermutationEMTest.java @@ -54,6 +54,9 @@ public void testRunEM() throws Throwable { args.add("--minimize"); args.add("false"); + args.add("--probOfRandomSampling"); + args.add("0.0"); + Solution solution = initAndRun(args); boolean ok = check(solution, "differentWeight",0); From b5b79fd87b5c32772bab5d812b09e7db378176f9 Mon Sep 17 00:00:00 2001 From: Amid Gol Date: Mon, 10 Jun 2024 17:13:48 +0200 Subject: [PATCH 6/9] Update AdaptiveHypermutationTest.java --- .../examples/hypermutation/AdaptiveHypermutationTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java index 19f0e23e61..94b8a1daf0 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/hypermutation/AdaptiveHypermutationTest.java @@ -26,7 +26,7 @@ public void testRunAdaptiveHypermutation() throws Throwable { runTestHandlingFlakyAndCompilation( "hypermtation/TestLowWeightHighImpact", "org.adaptivehypermuation.LowWeightHighImpactTest", - 500, + 1000, true, (args) -> { @@ -49,8 +49,6 @@ public void testRunAdaptiveHypermutation() throws Throwable { args.add("--minimize"); args.add("false"); - args.add("--focusedSearchActivationTime"); - args.add("0.5"); Solution solution = initAndRun(args); From 4d14ce4aa976b3c0c631cbc3086ccc421ce90f26 Mon Sep 17 00:00:00 2001 From: Amid Gol Date: Tue, 18 Jun 2024 21:15:54 +0200 Subject: [PATCH 7/9] Fix failing test in TaintNestedEMTest --- .../e2etests/spring/examples/taintnested/TaintNestedEMTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/taintnested/TaintNestedEMTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/taintnested/TaintNestedEMTest.java index 953dcb9b4e..3cfc8f0c69 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/taintnested/TaintNestedEMTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/taintnested/TaintNestedEMTest.java @@ -32,6 +32,8 @@ public void testRunEM() throws Throwable { args.add("true"); args.add("--discoveredInfoRewardedInFitness"); args.add("true"); + args.add("--baseTaintAnalysisProbability"); + args.add("0.9"); Solution solution = initAndRun(args); From 967dced087e66beddd859dc434b90ed8f58bda7c Mon Sep 17 00:00:00 2001 From: Amid Gol Date: Tue, 18 Jun 2024 22:20:10 +0200 Subject: [PATCH 8/9] Update TestabilityEMTest.java --- .../spring/examples/testability/TestabilityEMTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/testability/TestabilityEMTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/testability/TestabilityEMTest.java index 5d00148e01..624eac567b 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/testability/TestabilityEMTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/testability/TestabilityEMTest.java @@ -35,8 +35,8 @@ public void testRunEM() throws Throwable { 15_000, true, (args) -> { - args.add("--baseTaintAnalysisProbability"); - args.add("0.9"); +// args.add("--baseTaintAnalysisProbability"); +// args.add("0.9"); args.add("--enableTrackEvaluatedIndividual"); args.add("false"); From 360f200ca519df14de4a86240b59605720af4c8d Mon Sep 17 00:00:00 2001 From: Amid Gol Date: Tue, 18 Jun 2024 22:32:44 +0200 Subject: [PATCH 9/9] Update ValidEMTest.java --- .../evomaster/e2etests/spring/examples/valid/ValidEMTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/valid/ValidEMTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/valid/ValidEMTest.java index 6299d88ca0..cfde587fca 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/valid/ValidEMTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/valid/ValidEMTest.java @@ -24,6 +24,9 @@ public void testRunEM() throws Throwable { 1000, (args) -> { + args.add("--baseTaintAnalysisProbability"); + args.add("0.9"); + Solution solution = initAndRun(args); assertHasAtLeastOne(solution, HttpVerb.POST, 400, "/api/valid", null);