Skip to content
Permalink
Browse files

Merge pull request #135 from EMResearch/split_files

Split files
  • Loading branch information
arcuri82 committed Jan 11, 2020
2 parents be93d2f + 329cfbc commit 35175d71ae1d55744e888b0b2e0362151e1aab61
@@ -634,7 +634,8 @@ class EMConfig {


enum class TestSuiteSplitType {
NONE
NONE,
CODE
}

@Experimental
@@ -337,12 +337,17 @@ class Main {


assert(controllerInfoDto==null || controllerInfoDto.fullName != null)

val solutions = TestSuiteSplitter.split(solution, config.testSuiteSplitType)
writer.setSwagger(swagger)
solutions.forEach {
writer.writeTests(it, controllerInfoDto?.fullName)
/*solutions.forEach {
if(it.individuals.isNotEmpty()) writer.writeTests(it, controllerInfoDto?.fullName)
}
*/

solutions.filter { !it.individuals.isNullOrEmpty() }
.forEach {writer.writeTests(it, controllerInfoDto?.fullName) }

}

private fun writeStatistics(injector: Injector, solution: Solution<*>) {
@@ -14,6 +14,8 @@ class ExpectationsWriter {
private val responseStructureOracle = "responseStructureOracle"
private lateinit var swagger: Swagger
private lateinit var partialOracles: PartialOracles
//private val portRegex = """\w+:\d{4,5}""".toRegex()
private val portRegex = """(\w+|(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})):\d{1,5}""".toRegex()

fun setFormat(format: OutputFormat){
this.format = format
@@ -102,7 +104,10 @@ class ExpectationsWriter {
val printableElement = handleFieldValuesExpect(name, fieldName, result)
if (printableElement != "null"
&& printableElement != TestCaseWriter.NOT_COVERED_YET
&& !printableTh.contains("logged")
&& !printableTh.contains("logged") // messages that were logged in some external service are unlikely to be consistent between runs and introduce some flakiness
&& !printableTh.contains(portRegex)
// Error messages often contain the host and port. Since different runs are likely to be assigned different ports, this leads to flakiness
//TODO: Bogdan: Note that these are only meant to be temporary fixes, until a better solution can be found.
) {
lines.add(".that($expectationsMasterSwitch, $printableElement)")
}
@@ -120,7 +125,8 @@ class ExpectationsWriter {
if (printableTh != "null"
&& printableTh != TestCaseWriter.NOT_COVERED_YET
&& !printableTh.contains("logged") //again, unpleasant, but IDs logged as part of the error message are a problem
//TODO: find a more elegant way to deal with IDs, object refs, timestamps, etc.
&& !printableTh.contains(portRegex)
//TODO: find a more elegant way to deal with IDs, object refs, timestamps, etc.
) {
lines.add(".that($expectationsMasterSwitch, $printableTh)")
}
@@ -584,6 +584,7 @@ class TestCaseWriter {
if (printableTh != "null"
&& printableTh != NOT_COVERED_YET
&& !printableTh.contains("logged")
&& !printableTh.contains("""\w+:\d{4,5}""".toRegex())
) {
lines.add(".body(\"get($test_index)\", $printableTh)")
}
@@ -622,7 +623,7 @@ class TestCaseWriter {
lines.add(".body(isEmptyOrNullString())")
}else {
lines.add(".body(containsString(\"${
GeneUtils.applyEscapes(bodyString, mode = GeneUtils.EscapeMode.BODY, format = format)
GeneUtils.applyEscapes(bodyString, mode = GeneUtils.EscapeMode.TEXT, format = format)
}\"))")
}
}
@@ -651,6 +652,7 @@ class TestCaseWriter {
if (printableTh != "null"
&& printableTh != NOT_COVERED_YET
&& !printableTh.contains("logged")
&& !printableTh.contains("""\w+:\d{4,5}""".toRegex())
) {
//lines.add(".body(\"\'${it}\'\", ${printableTh})")
if(stringKey != "\'id\'") lines.add(".body(\"${stringKey}\", ${printableTh})")
@@ -1,6 +1,8 @@
package org.evomaster.core.output

import org.evomaster.core.EMConfig
import org.evomaster.core.problem.rest.RestCallResult
import org.evomaster.core.search.Individual
import org.evomaster.core.search.Solution


@@ -19,6 +21,52 @@ object TestSuiteSplitter {

return when(type){
EMConfig.TestSuiteSplitType.NONE -> listOf(solution)
EMConfig.TestSuiteSplitType.CODE -> splitByCode(solution)
}
}

/**
* [splitByCode] splits the Solution into several subsets based on the HTTP codes found in the actions.
* The split is as follows:
* - all individuals that contain at least one action with a 500 code go into a separate file. A 500 code is likely
* to be indicative of a fault, and therefore goes into a separate set.
*
* - all individuals that contain 2xx and 3xx action only are deemed to be successful, and a "successful" subset
* is created for them. These are test cases that indicate no problem.
*
* - remaining test cases are set in a third subset. These are often test cases that don't contain outright bugs
* (i.e. 500 actions) but may include 4xx. User errors and input problems may be interesting, hence the separate file.
* Nevertheless, it is up to individual test engineers to look at these test cases in more depth and decide
* if any further action or investigation is required.
*/
fun <T:Individual> splitByCode(solution: Solution<T>): List<Solution<T>>{
val s500 = solution.individuals.filter {
it.evaluatedActions().any { ac ->
(ac.result as RestCallResult).getStatusCode() == 500
// Note: we only check for 500 - Internal Server Error. Other 5xx codes are possible, but they're not really
// related to bug finding. Test cases that have other errors from the 5xx series will end up in the
// "remainder" subset - as they are neither errors, nor successful runs.
}
}.toMutableList()

val successses = solution.individuals.filter {
!s500.contains(it) &&
it.evaluatedActions().all { ac ->
val code = (ac.result as RestCallResult).getStatusCode()
if(code!=null) code < 400
else false
}
}.toMutableList()

val remainder = solution.individuals.filter {
!s500.contains(it) &&
!successses.contains(it)
}.toMutableList()

return listOf(Solution(s500, "${solution.testSuiteName}_500s"),
Solution(successses, "${solution.testSuiteName}_successes"),
Solution(remainder, "${solution.testSuiteName}_remainder")
)
}

}
@@ -127,4 +127,4 @@ There are 3 types of options:
|`processInterval`| __Int__. Specify how often to save results when a search monitor is enabled. *Default value*: `100`.|
|`resourceSampleStrategy`| __Enum__. Specify whether to enable resource-based strategy to sample an individual during search. Note that resource-based sampling is only applicable for REST problem with MIO algorithm. *Valid values*: `NONE, Customized, EqualProbability, Actions, TimeBudgets, Archive, ConArchive`. *Default value*: `NONE`.|
|`startPerOfCandidateGenesToMutate`| __Double__. Specify a percentage (before starting a focus search) which is used by archived-based gene selection method (e.g., APPROACH_IMPACT) for selecting top percent of genes as potential candidates to mutate. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.9`.|
|`testSuiteSplitType`| __Enum__. Instead of generating a single test file, it could be split in several files, according to different strategies. *Valid values*: `NONE`. *Default value*: `NONE`.|
|`testSuiteSplitType`| __Enum__. Instead of generating a single test file, it could be split in several files, according to different strategies. *Valid values*: `NONE, CODE`. *Default value*: `NONE`.|
@@ -25,6 +25,7 @@
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;

@@ -104,18 +105,86 @@ protected void runTestHandlingFlaky(
});
}

protected void runTestHandlingFlaky(
String outputFolderName,
String fullClassName,
List<String> terminations,
int iterations,
boolean createTests,
Consumer<List<String>> lambda,
int timeoutMinutes) throws Throwable{

/*
Years have passed, still JUnit 5 does not handle global test timeouts :(
https://github.com/junit-team/junit5/issues/80
*/
//

List<ClassName> classNames = new ArrayList<ClassName>(Collections.emptyList());

for (String termination : terminations) {
classNames.add(new ClassName(fullClassName + termination));
}


assertTimeoutPreemptively(Duration.ofMinutes(timeoutMinutes), () -> {
ClassName className = new ClassName(fullClassName);
clearGeneratedFiles(outputFolderName, classNames);

List<String> args = getArgsWithCompilation(iterations, outputFolderName, className, createTests);

handleFlaky(
() -> lambda.accept(new ArrayList<>(args))
);
});
}



protected void runTestHandlingFlakyAndCompilation(
String outputFolderName,
String fullClassName,
int iterations,
Consumer<List<String>> lambda) throws Throwable {

runTestHandlingFlakyAndCompilation(outputFolderName, fullClassName, iterations, true, lambda, 3);
runTestHandlingFlakyAndCompilation(outputFolderName, fullClassName, Arrays.asList(""), iterations, true, lambda, 3);
}

protected void runTestHandlingFlakyAndCompilation(
String outputFolderName,
String fullClassName,
List<String> terminations,
int iterations,
Consumer<List<String>> lambda) throws Throwable {

runTestHandlingFlakyAndCompilation(outputFolderName, fullClassName, terminations, iterations, true, lambda, 3);
}

protected void runTestHandlingFlakyAndCompilation(
String outputFolderName,
String fullClassName,
List<String> terminations,
int iterations,
boolean createTests,
Consumer<List<String>> lambda,
int timeoutMinutes) throws Throwable {

runTestHandlingFlaky(outputFolderName, fullClassName, terminations, iterations, createTests,lambda, timeoutMinutes);


//BMR: this is where I should handle multiples???
if (createTests){
for (String termination : terminations) {
assertTimeoutPreemptively(Duration.ofMinutes(2), () -> {
ClassName className = new ClassName(fullClassName + termination);
clearCompiledFiles(className);
//the first one goes through, but for the second generated files appear to not be clean.
compileRunAndVerifyTests(outputFolderName, className);
});
}
}
}

protected void runTestHandlingFlakyAndCompilation(
String outputFolderName,
String fullClassName,
@@ -158,7 +227,7 @@ protected void compileRunAndVerifyTests(String outputFolderName, ClassName class
assertTrue(summary.getTestsSucceededCount() > 0);
}

protected void clearGeneratedFiles(String outputFolderName, ClassName testClassName){
protected void clearGeneratedFiles(String outputFolderName, List<ClassName> testClassNames){

File folder = new File(outputFolderPath(outputFolderName));
try{
@@ -167,9 +236,24 @@ protected void clearGeneratedFiles(String outputFolderName, ClassName testClassN
throw new RuntimeException(e);
}

String bytecodePath = "target/test-classes/" + testClassName.getAsResourcePath();
File compiledFile = new File(bytecodePath);
compiledFile.delete();
for (ClassName testClassName : testClassNames){
clearCompiledFiles(testClassName);
}

}

protected void clearGeneratedFiles(String outputFolderName, ClassName testClassName){
List<ClassName> classNames = new ArrayList<ClassName>();
classNames.add(testClassName);

clearGeneratedFiles(outputFolderName, classNames);
}

protected void clearCompiledFiles(ClassName testClassName){
String byteCodePath = "target/test-classes/" + testClassName.getAsResourcePath();
File compiledFile = new File(byteCodePath);
boolean result = compiledFile.delete();

}

protected Class<?> loadClass(ClassName className){
@@ -0,0 +1,14 @@
package org.evomaster.e2etests.spring.examples.splitter;

import com.foo.rest.examples.spring.strings.StringsController;
import org.evomaster.e2etests.spring.examples.SpringTestBase;
import org.junit.jupiter.api.BeforeAll;


public class SplitterTestBase extends SpringTestBase {

@BeforeAll
public static void initClass() throws Exception {
SpringTestBase.initClass(new StringsController());
}
}
@@ -0,0 +1,67 @@
package org.evomaster.e2etests.spring.examples.splitter;

import org.evomaster.core.EMConfig;
import org.evomaster.core.output.TestSuiteSplitter;
import org.evomaster.core.problem.rest.RestIndividual;
import org.evomaster.core.search.Solution;
import org.junit.jupiter.api.Test;

import java.util.Arrays;
import java.util.List;

import static org.junit.jupiter.api.Assertions.assertTrue;

public class TestSuiteSplitterTest extends SplitterTestBase {


@Test
public void testRunEM_CODE() throws Throwable {
testRunEMMulti(EMConfig.TestSuiteSplitType.CODE);
}

@Test
public void testRunEM_NONE() throws Throwable {
testRunEM(EMConfig.TestSuiteSplitType.NONE);
}

private void testRunEMMulti(EMConfig.TestSuiteSplitType splitType) throws Throwable {
List<String> terminations = Arrays.asList("_successes", "_remainder");
runTestHandlingFlakyAndCompilation(
"SplitterEM",
"org.bar.splitter.Split_" + splitType,
terminations,
10_000,
(args) -> {
args.add("--testSuiteSplitType");
args.add("" + splitType);
Solution<RestIndividual> solution = initAndRun(args);
assertTrue(solution.getIndividuals().size() >= 1);
List<Solution<?>> splits = TestSuiteSplitter.INSTANCE.split(solution, splitType);
assertTrue(splits.size() >= 1);
}
);
}

private void testRunEM(EMConfig.TestSuiteSplitType splitType) throws Throwable {
runTestHandlingFlakyAndCompilation(
"SplitterEM",
"org.bar.splitter.Split_" + splitType,
10_000,
(args) -> {
args.add("--testSuiteSplitType");
args.add("" + splitType);

Solution<RestIndividual> solution = initAndRun(args);

assertTrue(solution.getIndividuals().size() >= 1);

List<Solution<?>> splits = TestSuiteSplitter.INSTANCE.split(solution, splitType);

assertTrue(splits.size() >= 1);

}
);
}


}

0 comments on commit 35175d7

Please sign in to comment.
You can’t perform that action at this time.