Merge pull request #981 from EMResearch/premature-stop

Premature stop

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -46,6 +46,8 @@ class EMConfig {
 
         private val log = LoggerFactory.getLogger(EMConfig::class.java)
 
+        private const val timeRegex = "(\\s*)((?=(\\S+))(\\d+h)?(\\d+m)?(\\d+s)?)(\\s*)"
+
         private const val headerRegex = "(.+:.+)|(^$)"
 
         private const val targetSeparator = ";"
@@ -205,7 +207,7 @@ class EMConfig {
             val max = (m.annotations.find { it is Max } as? Max)?.max
             val probability = m.annotations.find { it is Probability }
             val url = m.annotations.find { it is Url }
-            val regex = (m.annotations.find { it is Regex } as? Regex)?.regex
+            val regex = (m.annotations.find { it is Regex } as? Regex)
 
             var constraints = ""
             if (min != null || max != null || probability != null || url != null || regex != null) {
@@ -223,7 +225,7 @@ class EMConfig {
                     constraints += "URL"
                 }
                 if (regex != null) {
-                    constraints += "regex $regex"
+                    constraints += "regex ${regex.regex}"
                 }
             }
 
@@ -547,6 +549,11 @@ class EMConfig {
         if(security && !minimize){
             throw ConfigProblemException("The use of 'security' requires 'minimize'")
         }
+
+        if(prematureStop.isNotEmpty() && stoppingCriterion != StoppingCriterion.TIME){
+            throw ConfigProblemException("The use of 'prematureStop' is meaningful only if the stopping criterion" +
+                    " 'stoppingCriterion' is based on time")
+        }
     }
 
     private fun checkPropertyConstraints(m: KMutableProperty<*>) {
@@ -857,7 +864,6 @@ class EMConfig {
 
     //----- "Important" options, sorted by priority --------------
 
-
     val defaultMaxTime = "60s"
 
     @Important(1.0)
@@ -873,11 +879,23 @@ class EMConfig {
             " For how long should _EvoMaster_ be left run?" +
             " The default 1 _minute_ is just for demonstration." +
             " __We recommend to run it between 1 and 24 hours__, depending on the size and complexity " +
-            " of the tested application."
+            " of the tested application." +
+            " You can get better results by combining this option with `--prematureStop`." +
+            " For example, something like `--maxTime 24h --prematureStop 1h` will run the search for 24 hours," +
+            " but the it will stop at any point in time in which there has be no improvement in last hour."
     )
-    @Regex("(\\s*)((?=(\\S+))(\\d+h)?(\\d+m)?(\\d+s)?)(\\s*)")
+    @Regex(timeRegex)
     var maxTime = defaultMaxTime
 
+    @Experimental
+    @Cfg("Max amount of time the search is going to wait since last improvement (on metrics we optimize for," +
+            " like fault finding and code/schema coverage)." +
+            " If there is no improvement within this allotted max time, then the search will be prematurely stopped," +
+            " regardless of what specified in --maxTime option.")
+    @Regex("($timeRegex)|(^$)")
+    var prematureStop : String = ""
+
+
     @Important(1.1)
     @Cfg("The path directory of where the generated test classes should be saved to")
     @Folder
@@ -2226,20 +2244,31 @@ class EMConfig {
             return maxTimeInSeconds
         }
 
-        val h = maxTime.indexOf('h')
-        val m = maxTime.indexOf('m')
-        val s = maxTime.indexOf('s')
+        return convertToSeconds(maxTime)
+    }
+
+    fun improvementTimeoutInSeconds() : Int {
+        if(prematureStop.isNullOrBlank()){
+            return Int.MAX_VALUE
+        }
+        return convertToSeconds(prematureStop)
+    }
+
+    private fun convertToSeconds(time: String): Int {
+        val h = time.indexOf('h')
+        val m = time.indexOf('m')
+        val s = time.indexOf('s')
 
         val hours = if (h >= 0) {
-            maxTime.subSequence(0, h).toString().trim().toInt()
+            time.subSequence(0, h).toString().trim().toInt()
         } else 0
 
         val minutes = if (m >= 0) {
-            maxTime.subSequence(if (h >= 0) h + 1 else 0, m).toString().trim().toInt()
+            time.subSequence(if (h >= 0) h + 1 else 0, m).toString().trim().toInt()
         } else 0
 
         val seconds = if (s >= 0) {
-            maxTime.subSequence(if (m >= 0) m + 1 else (if (h >= 0) h + 1 else 0), s).toString().trim().toInt()
+            time.subSequence(if (m >= 0) m + 1 else (if (h >= 0) h + 1 else 0), s).toString().trim().toInt()
         } else 0
 
         return (hours * 60 * 60) + (minutes * 60) + seconds

core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt

@@ -79,6 +79,10 @@ abstract class SearchAlgorithm<T> where T : Individual {
             }
         }
 
+        if(time.isImprovementTimeout()){
+            LoggingUtil.uniqueUserWarn("Premature stop of the search. No improvement in the last ${config.prematureStop}")
+        }
+
         handleAfterSearch()
 
         return archive.extractSolution()

core/src/main/kotlin/org/evomaster/core/search/service/SearchStatusUpdater.kt

@@ -79,6 +79,7 @@ class SearchStatusUpdater : SearchListener{
         val current = String.format("%.3f", time.percentageUsedBudget() * 100)
 
         if(first){
+            println()
             println()
             if(config.e_u1f984){
                 println()
@@ -107,10 +108,15 @@ class SearchStatusUpdater : SearchListener{
             val avgTime = String.format("%.1f", avgTimeAndSize.first)
             val avgSize = String.format("%.1f",avgTimeAndSize.second)
 
+            val sinceLast = time.getSecondsSinceLastImprovement()
+
+            upLineAndErase()
             upLineAndErase()
-            println("* Consumed search budget: $passed%;" +
-                    " covered targets: $coverage;" +
-                    " time per test: ${avgTime}ms ($avgSize actions)")
+            println("* Consumed search budget: $passed%")
+            println("* Covered targets: $coverage;" +
+                    " time per test: ${avgTime}ms ($avgSize actions);" +
+                    " since last improvement: ${sinceLast}s"
+            )
 
             if(config.e_u1f984){
                 updateExtra()

core/src/main/kotlin/org/evomaster/core/search/service/SearchTimeController.kt

@@ -51,6 +51,9 @@ class SearchTimeController {
     var lastActionImprovement = -1
         private set
 
+    var lastActionImprovementTimestamp = -1L
+        private set
+
     var lastActionTimestamp = 0L
         private set
 
@@ -131,6 +134,7 @@ class SearchTimeController {
         recording = true
         searchStarted = true
         startTime = System.currentTimeMillis()
+        lastActionImprovementTimestamp = startTime
     }
 
     fun addListener(listener: SearchListener){
@@ -210,6 +214,7 @@ class SearchTimeController {
     fun newActionImprovement(){
         if(!recording) return
         lastActionImprovement = evaluatedActions
+        lastActionImprovementTimestamp = System.currentTimeMillis()
     }
 
 
@@ -238,7 +243,22 @@ class SearchTimeController {
 
     fun shouldContinueSearch(): Boolean{
 
-        return percentageUsedBudget() < 1.0
+        return percentageUsedBudget() < 1.0 && !isImprovementTimeout()
+    }
+
+    fun isImprovementTimeout() : Boolean{
+
+        if(configuration.prematureStop.isNullOrBlank()){
+            return false
+        }
+
+        val passed = getSecondsSinceLastImprovement()
+
+        return  passed > configuration.improvementTimeoutInSeconds()
+    }
+
+    fun getSecondsSinceLastImprovement() : Int{
+        return ((System.currentTimeMillis() - lastActionImprovementTimestamp) / 1000.0).toInt()
     }
 
     /**

docs/options.md

@@ -33,7 +33,7 @@ There are 3 types of options:
 
 |Options|Description|
 |---|---|
-|`maxTime`| __String__. Maximum amount of time allowed for the search.  The time is expressed with a string where hours (`h`), minutes (`m`) and seconds (`s`) can be specified, e.g., `1h10m120s` and `72m` are both valid and equivalent. Each component (i.e., `h`, `m` and `s`) is optional, but at least one must be specified.  In other words, if you need to run the search for just `30` seconds, you can write `30s`  instead of `0h0m30s`. **The more time is allowed, the better results one can expect**. But then of course the test generation will take longer. For how long should _EvoMaster_ be left run? The default 1 _minute_ is just for demonstration. __We recommend to run it between 1 and 24 hours__, depending on the size and complexity  of the tested application. *Constraints*: `regex (\s*)((?=(\S+))(\d+h)?(\d+m)?(\d+s)?)(\s*)`. *Default value*: `60s`.|
+|`maxTime`| __String__. Maximum amount of time allowed for the search.  The time is expressed with a string where hours (`h`), minutes (`m`) and seconds (`s`) can be specified, e.g., `1h10m120s` and `72m` are both valid and equivalent. Each component (i.e., `h`, `m` and `s`) is optional, but at least one must be specified.  In other words, if you need to run the search for just `30` seconds, you can write `30s`  instead of `0h0m30s`. **The more time is allowed, the better results one can expect**. But then of course the test generation will take longer. For how long should _EvoMaster_ be left run? The default 1 _minute_ is just for demonstration. __We recommend to run it between 1 and 24 hours__, depending on the size and complexity  of the tested application. You can get better results by combining this option with `--prematureStop`. For example, something like `--maxTime 24h --prematureStop 1h` will run the search for 24 hours, but the it will stop at any point in time in which there has be no improvement in last hour. *Constraints*: `regex (\s*)((?=(\S+))(\d+h)?(\d+m)?(\d+s)?)(\s*)`. *Default value*: `60s`.|
 |`outputFolder`| __String__. The path directory of where the generated test classes should be saved to. *Default value*: `src/em`.|
 |`configPath`| __String__. File path for file with configuration settings. Supported formats are YAML and TOML. When EvoMaster starts, it will read such file and import all configurations from it. *Constraints*: `regex .*\.(yml\|yaml\|toml)`. *Default value*: `em.yaml`.|
 |`outputFilePrefix`| __String__. The name prefix of generated file(s) with the test cases, without file type extension. In JVM languages, if the name contains '.', folders will be created to represent the given package structure. Also, in JVM languages, should not use '-' in the file name, as not valid symbol for class identifiers. This prefix be combined with the outputFileSuffix to combined the final name. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `EvoMaster`.|
@@ -240,6 +240,7 @@ There are 3 types of options:
 |`maxTestsPerTestSuite`| __Int__. Specify the maximum number of tests to be generated in one test suite. Note that a negative number presents no limit per test suite. *Default value*: `-1`.|
 |`maximumExistingDataToSampleInDb`| __Int__. Specify a maximum number of existing data in the database to sample when SQL handling is enabled. Note that a negative number means all existing data would be sampled. *Default value*: `-1`.|
 |`mutationTargetsSelectionStrategy`| __Enum__. Specify a strategy to select targets for evaluating mutation. *Valid values*: `FIRST_NOT_COVERED_TARGET, EXPANDED_UPDATED_NOT_COVERED_TARGET, UPDATED_NOT_COVERED_TARGET`. *Default value*: `FIRST_NOT_COVERED_TARGET`.|
+|`prematureStop`| __String__. Max amount of time the search is going to wait since last improvement (on metrics we optimize for, like fault finding and code/schema coverage). If there is no improvement within this allotted max time, then the search will be prematurely stopped, regardless of what specified in --maxTime option. *Constraints*: `regex ((\s*)((?=(\S+))(\d+h)?(\d+m)?(\d+s)?)(\s*))\|(^$)`. *Default value*: `""`.|
 |`probOfHandlingLength`| __Double__. Specify a probability of applying length handling. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|
 |`probOfHarvestingResponsesFromActualExternalServices`| __Double__. a probability of harvesting actual responses from external services as seeds. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|
 |`probOfMutatingResponsesBasedOnActualResponse`| __Double__. a probability of mutating mocked responses based on actual responses. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|

e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/base/BaseMaxTimeAttemptEMTest.kt

@@ -0,0 +1,45 @@
+package org.evomaster.e2etests.spring.openapi.v3.base
+
+import com.foo.rest.examples.spring.openapi.v3.base.BaseController
+import org.evomaster.core.output.OutputFormat
+import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.Timeout
+
+/**
+ * Created by arcuri82 on 03-Mar-20.
+ */
+class BaseMaxTimeAttemptEMTest : SpringTestBase() {
+
+    companion object {
+        @BeforeAll
+        @JvmStatic
+        fun init() {
+            initClass(BaseController())
+        }
+    }
+
+
+    @Timeout(60)
+    @Test
+    fun testRunEM() {
+
+            val args = listOf(
+                    "--createTests", "false",
+                    "--seed", "" + defaultSeed,
+                    "--useTimeInFeedbackSampling", "false",
+                    "--sutControllerPort", "" + controllerPort,
+                    "--stoppingCriterion", "TIME",
+                    "--createConfigPathIfMissing", "false",
+                    "--maxTime", "10m", // way more than the JUnit @Timeout(60)
+                    "--prematureStop", "5s" // short compared to JUnit @Timeout(60)
+            )
+
+            val solution = initAndRun(args)
+
+            assertTrue(solution.individuals.size >= 1)
+
+    }
+}