Permalink
Browse files

adding support for flaky tests. reactivating Travis

  • Loading branch information...
arcuri82 committed Aug 21, 2018
1 parent ea0e817 commit ddab43eb02c2cddc963181352baef0ff6075b7f2
View
@@ -12,6 +12,6 @@ cache:
- $HOME/.m2
notifications:
email: false
email: true
script: mvn clean verify --quiet
View
@@ -1,6 +1,4 @@
<!--- REMOVED till Failsafe fully support JUnit 5
[![Build Status](https://travis-ci.org/EMResearch/EvoMaster.svg?branch=master)](https://travis-ci.org/EMResearch/EvoMaster)
-->
[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.evomaster/evomaster-client-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.evomaster/evomaster-client-java)
@@ -11,7 +11,7 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
public class SFTest extends RestTestBase{
public class SFTest extends RestTestBase {
@BeforeAll
public static void initClass() throws Exception {
@@ -20,21 +20,23 @@ public static void initClass() throws Exception {
}
@Test
public void testRunEM() {
public void testRunEM() throws Throwable {
String[] args = new String[]{
"--createTests", "false",
"--seed", "42",
"--sutControllerPort", "" + controllerPort,
"--maxActionEvaluations", "200",
"--stoppingCriterion", "FITNESS_EVALUATIONS"
};
handleFlaky(() -> {
String[] args = new String[]{
"--createTests", "false",
"--seed", "42",
"--sutControllerPort", "" + controllerPort,
"--maxActionEvaluations", "500",
"--stoppingCriterion", "FITNESS_EVALUATIONS"
};
Solution<RestIndividual> solution = (Solution<RestIndividual>) Main.initAndRun(args);
Solution<RestIndividual> solution = (Solution<RestIndividual>) Main.initAndRun(args);
assertTrue(solution.getIndividuals().size() >= 1);
assertTrue(solution.getIndividuals().size() >= 1);
assertHasAtLeastOne(solution, HttpVerb.POST, 400);
assertHasAtLeastOne(solution, HttpVerb.POST, 200);
assertHasAtLeastOne(solution, HttpVerb.POST, 400);
assertHasAtLeastOne(solution, HttpVerb.POST, 200);
});
}
}
@@ -200,4 +200,34 @@ protected void assertNone(Solution<RestIndividual> solution,
assertTrue(ok, msg.toString());
}
/**
* Unfortunately JUnit 5 does not handle flaky tests, and Maven is not upgraded yet.
* See https://github.com/junit-team/junit5/issues/1558#issuecomment-414701182
*
* TODO: once that issue is fixed (if it will ever be fixed), then this method
* will no longer be needed
*
* @param lambda
* @throws Throwable
*/
protected void handleFlaky(Runnable lambda) throws Throwable{
int attempts = 3;
Throwable error = null;
for(int i=0; i<attempts; i++){
try{
lambda.run();
return;
}catch (OutOfMemoryError e){
throw e;
}catch (Throwable t){
error = t;
}
}
throw error;
}
}
@@ -28,45 +28,48 @@ public static void initClass() throws Exception {
@Test
public void testRunEM() {
String[] args = new String[]{
"--createTests", "false",
"--seed", "42",
"--sutControllerPort", "" + controllerPort,
"--maxActionEvaluations", "1000",
"--stoppingCriterion", "FITNESS_EVALUATIONS"
};
Solution<RestIndividual> solution = (Solution<RestIndividual>) Main.initAndRun(args);
assertTrue(solution.getIndividuals().size() >= 1);
ObjectMapper mapper = new ObjectMapper();
//get number of distinct response values
List<Integer> responses = solution.getIndividuals().stream()
.flatMap(i -> i.getResults().stream())
.map(r -> r.getResultValue(RestCallResult.Companion.getBODY()))
.filter(s -> s != null)
.map(s -> {
try {
return mapper.readValue(s, BranchesResponseDto.class);
} catch (IOException e) {
return null;
}
})
.filter(b -> b != null)
.map(b -> b.value)
.distinct()
.sorted()
.collect(Collectors.toList());
long n = responses.size();
//FIXME: should be 9, but 2 branches requiring ==0 cannot be
//covered until we have "working" local search or seeding
assertTrue(n >= 7);
//assertEquals(9, n);
public void testRunEM() throws Throwable {
handleFlaky(() -> {
String[] args = new String[]{
"--createTests", "false",
"--seed", "42",
"--sutControllerPort", "" + controllerPort,
"--maxActionEvaluations", "5000",
"--stoppingCriterion", "FITNESS_EVALUATIONS"
};
Solution<RestIndividual> solution = (Solution<RestIndividual>) Main.initAndRun(args);
assertTrue(solution.getIndividuals().size() >= 1);
ObjectMapper mapper = new ObjectMapper();
//get number of distinct response values
List<Integer> responses = solution.getIndividuals().stream()
.flatMap(i -> i.getResults().stream())
.map(r -> r.getResultValue(RestCallResult.Companion.getBODY()))
.filter(s -> s != null)
.map(s -> {
try {
return mapper.readValue(s, BranchesResponseDto.class);
} catch (IOException e) {
return null;
}
})
.filter(b -> b != null)
.map(b -> b.value)
.distinct()
.sorted()
.collect(Collectors.toList());
long n = responses.size();
//FIXME: should be 9, but 2 branches requiring ==0 cannot be
//covered until we have "working" local search or seeding
assertTrue(n >= 7);
//assertEquals(9, n);
});
}
}
@@ -8,37 +8,39 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
public class HLEMTest extends HLTestBase{
public class HLEMTest extends HLTestBase {
@Test
public void testRunEM() {
String[] args = new String[]{
"--createTests", "false",
"--seed", "42",
"--sutControllerPort", "" + controllerPort,
"--maxActionEvaluations", "1000",
"--stoppingCriterion", "FITNESS_EVALUATIONS"
};
Solution<RestIndividual> solution = (Solution<RestIndividual>) Main.initAndRun(args);
assertTrue(solution.getIndividuals().size() >= 1);
//easy cases
assertNone(solution, HttpVerb.POST, 404);
assertHasAtLeastOne(solution, HttpVerb.GET, 404);
assertHasAtLeastOne(solution, HttpVerb.DELETE, 404);
assertHasAtLeastOne(solution, HttpVerb.PATCH, 404);
assertHasAtLeastOne(solution, HttpVerb.PUT, 201);
assertHasAtLeastOne(solution, HttpVerb.POST, 201);
//need smart sampling
assertHasAtLeastOne(solution, HttpVerb.GET, 200);
assertHasAtLeastOne(solution, HttpVerb.DELETE, 204);
assertHasAtLeastOne(solution, HttpVerb.PATCH, 204);
assertHasAtLeastOne(solution, HttpVerb.PUT, 204);
public void testRunEM() throws Throwable {
handleFlaky(() -> {
String[] args = new String[]{
"--createTests", "false",
"--seed", "42",
"--sutControllerPort", "" + controllerPort,
"--maxActionEvaluations", "2000",
"--stoppingCriterion", "FITNESS_EVALUATIONS"
};
Solution<RestIndividual> solution = (Solution<RestIndividual>) Main.initAndRun(args);
assertTrue(solution.getIndividuals().size() >= 1);
//easy cases
assertNone(solution, HttpVerb.POST, 404);
assertHasAtLeastOne(solution, HttpVerb.GET, 404);
assertHasAtLeastOne(solution, HttpVerb.DELETE, 404);
assertHasAtLeastOne(solution, HttpVerb.PATCH, 404);
assertHasAtLeastOne(solution, HttpVerb.PUT, 201);
assertHasAtLeastOne(solution, HttpVerb.POST, 201);
//need smart sampling
assertHasAtLeastOne(solution, HttpVerb.GET, 200);
assertHasAtLeastOne(solution, HttpVerb.DELETE, 204);
assertHasAtLeastOne(solution, HttpVerb.PATCH, 204);
assertHasAtLeastOne(solution, HttpVerb.PUT, 204);
});
}
}

0 comments on commit ddab43e

Please sign in to comment.