Skip to content
Permalink
Browse files

Merge pull request #127 from EMResearch/expectations

Expectations
  • Loading branch information
arcuri82 committed Nov 18, 2019
2 parents b7d779a + aafb0ac commit 4c9226e6ae37cbd458ef9302a3b0668a2217cf92
Showing with 867 additions and 328 deletions.
  1. +10 −2 .../controller/src/main/java/org/evomaster/client/java/controller/contentMatchers/NumberMatcher.java
  2. +37 −0 .../controller/src/main/java/org/evomaster/client/java/controller/contentMatchers/StringMatcher.java
  3. +37 −0 ...ntroller/src/main/java/org/evomaster/client/java/controller/contentMatchers/SubStringMatcher.java
  4. +0 −1 ...java/controller/src/main/java/org/evomaster/client/java/controller/expect/ExpectationHandler.java
  5. +2 −2 core/src/main/kotlin/org/evomaster/core/EMConfig.kt
  6. +8 −8 core/src/main/kotlin/org/evomaster/core/Main.kt
  7. +159 −0 core/src/main/kotlin/org/evomaster/core/output/ExpectationsWriter.kt
  8. +96 −77 core/src/main/kotlin/org/evomaster/core/output/TestCaseWriter.kt
  9. +2 −2 core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt
  10. +235 −0 core/src/main/kotlin/org/evomaster/core/output/service/ObjectGenerator.kt
  11. +94 −0 core/src/main/kotlin/org/evomaster/core/output/service/PartialOracles.kt
  12. +25 −8 core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt
  13. +31 −4 core/src/main/kotlin/org/evomaster/core/problem/rest/RestActionBuilder.kt
  14. +11 −2 core/src/main/kotlin/org/evomaster/core/problem/rest/RestCallAction.kt
  15. +3 −0 core/src/main/kotlin/org/evomaster/core/problem/rest/UsedObjects.kt
  16. +12 −213 core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestSampler.kt
  17. +2 −2 core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestStructureMutator.kt
  18. +12 −6 core/src/main/kotlin/org/evomaster/core/search/gene/GeneUtils.kt
  19. +59 −0 core/src/test/kotlin/org/evomaster/core/output/MatcherTests.kt
  20. +20 −1 core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt
  21. +4 −0 core/src/test/kotlin/org/evomaster/core/output/WriteJsonTest.kt
  22. +3 −0 core/src/test/kotlin/org/evomaster/core/output/WriteXMLTest.kt
  23. +5 −0 core/src/test/kotlin/org/evomaster/core/output/service/TestSuiteWriterTest.kt
@@ -22,10 +22,10 @@ protected boolean matchesSafely (Number item) {
if (item == null) return false;
else return item.doubleValue() == value;
}
public static Matcher<Number> numberMatches(Number item) {
public static NumberMatcher numberMatches(Number item) {
return new NumberMatcher(item.doubleValue());
}
public static Matcher<Number> numberMatches(String item) {
public static NumberMatcher numberMatches(String item) {
try{
Number value = Double.parseDouble(item);
return new NumberMatcher(value.doubleValue());
@@ -39,4 +39,12 @@ public static boolean numbersMatch(Number item1, Number item2){
NumberMatcher n1 = new NumberMatcher(item1.doubleValue());
return n1.matchesSafely(item2);
}

public static boolean numbersMatch(Number item1, String item2){
if(item1 == null || item2 == null) return false;
NumberMatcher n2 = numberMatches(item2);
return n2.matchesSafely(item1);
}


}
@@ -0,0 +1,37 @@
package org.evomaster.client.java.controller.contentMatchers;

import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeMatcher;

public class StringMatcher extends TypeSafeMatcher<String> {
private final String value;

public StringMatcher(String value) { this.value = value; }

@Override
public void describeTo(Description description){
//The point of this matcher is to provide comparisons for strings in a way that is consistent with the
// numerical matcher. It can also be extended to provide a more detailed type of matching
// (for example, include settings for ignore case, include string, and anything else we might discover we need.
// Should this prove unnecessary, it is to be removed.
description.appendValue(value);
}

@Override
protected boolean matchesSafely(String item){
if(item == null) return false;
else return value.equals(item.toString());
}

public static Matcher<String> stringMatches(String item) {return new StringMatcher((item)); }
public static Matcher<String> stringMatches(Object item) {return new StringMatcher((item.toString())); }

public static boolean stringsMatch(Object item1, Object item2){
if(item1 == null || item2 == null) return false;
StringMatcher s1 = new StringMatcher(item1.toString());
return s1.matchesSafely(item2.toString());
}

}

@@ -0,0 +1,37 @@
package org.evomaster.client.java.controller.contentMatchers;

import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeMatcher;

public class SubStringMatcher extends TypeSafeMatcher<String> {

private final String value;

public SubStringMatcher(String value) { this.value = value; }

@Override
public void describeTo(Description description){
//The point of this matcher is to provide comparisons for strings in a way that is consistent with the
// numerical matcher. It can also be extended to provide a more detailed type of matching
// (for example, include settings for ignore case, include string, and anything else we might discover we need.
// Should this prove unnecessary, it is to be removed.
description.appendValue(value);
}

@Override
protected boolean matchesSafely(String item){
if(item == null) return false;
else return (value.contains(item) || item.contains(value));
}

public static Matcher<String> subStringMatches(String item) {return new SubStringMatcher((item)); }
public static Matcher<String> subStringMatches(Object item) {return new SubStringMatcher((item.toString())); }

public static boolean subStringsMatch(Object item1, Object item2){
if(item1 == null || item2 == null) return false;
SubStringMatcher s1 = new SubStringMatcher(item1.toString());
return s1.matchesSafely(item2.toString());
}

}
@@ -21,7 +21,6 @@


public class ExpectationHandler implements AggregateExpectation, IndividualExpectation {

private boolean masterSwitch = false;
/**
*
@@ -711,8 +711,8 @@ class EMConfig {

@Experimental
@Cfg("Enable Expectation Generation. If enabled, expectations will be generated. " +
"A variable called activeExpectations is added to each test case, with a default value of false. If set to true, an expectation that fails will cause the test case containing it to fail.")
var expectationsActive = false
"A variable called expectationsMasterSwitch is added to the test suite, with a default value of false. If set to true, an expectation that fails will cause the test case containing it to fail.")
var expectationsActive = true

@Cfg("Generate basic assertions. Basic assertions (comparing the returned object to itself) are added to the code. " +
"NOTE: this should not cause any tests to fail.")
@@ -14,10 +14,7 @@ import org.evomaster.core.logging.LoggingUtil
import org.evomaster.core.output.TestSuiteSplitter
import org.evomaster.core.output.service.TestSuiteWriter
import org.evomaster.core.problem.rest.RestIndividual
import org.evomaster.core.problem.rest.service.BlackBoxRestModule
import org.evomaster.core.problem.rest.service.ResourceDepManageService
import org.evomaster.core.problem.rest.service.ResourceRestModule
import org.evomaster.core.problem.rest.service.RestModule
import org.evomaster.core.problem.rest.service.*
import org.evomaster.core.problem.web.service.WebModule
import org.evomaster.core.remote.NoRemoteConnectionException
import org.evomaster.core.remote.SutProblemException
@@ -142,7 +139,11 @@ class Main {

val controllerInfo = checkState(injector)

val config = injector.getInstance(EMConfig::class.java)
val idMapper = injector.getInstance(IdMapper::class.java)

val solution = run(injector)
val faults = solution.overall.potentialFoundFaults(idMapper)

writeOverallProcessData(injector)

@@ -156,10 +157,7 @@ class Main {

writeTests(injector, solution, controllerInfo)

val config = injector.getInstance(EMConfig::class.java)
val idMapper = injector.getInstance(IdMapper::class.java)

val faults = solution.overall.potentialFoundFaults(idMapper)

LoggingUtil.getInfoLogger().apply {
val stc = injector.getInstance(SearchTimeController::class.java)
@@ -335,11 +333,13 @@ class Main {
LoggingUtil.getInfoLogger().info("Going to save $tests to ${config.outputFolder}")

val writer = injector.getInstance(TestSuiteWriter::class.java)
val swagger = injector.getInstance(RestSampler::class.java).getSwagger()


assert(controllerInfoDto==null || controllerInfoDto.fullName != null)

val solutions = TestSuiteSplitter.split(solution, config.testSuiteSplitType)

writer.setSwagger(swagger)
solutions.forEach {
writer.writeTests(it, controllerInfoDto?.fullName)
}
@@ -0,0 +1,159 @@
package org.evomaster.core.output

import com.google.gson.Gson
import io.swagger.models.Swagger
import org.evomaster.core.output.service.PartialOracles
import org.evomaster.core.problem.rest.RestCallAction
import org.evomaster.core.problem.rest.RestCallResult
import org.evomaster.core.search.gene.GeneUtils
import javax.ws.rs.core.MediaType

class ExpectationsWriter {
private var format: OutputFormat = OutputFormat.JAVA_JUNIT_4
private val expectationsMasterSwitch = "expectationsMasterSwitch"
private val responseStructureOracle = "responseStructureOracle"
private lateinit var swagger: Swagger
private lateinit var partialOracles: PartialOracles

fun setFormat(format: OutputFormat){
this.format = format
}

fun setSwagger(swagger: Swagger){
this.swagger = swagger
}

fun setPartialOracles(partialOracles: PartialOracles){
this.partialOracles = partialOracles
}

fun addDeclarations(lines: Lines){
lines.addEmpty()
when{
format.isJava() -> lines.append("ExpectationHandler expectationHandler = expectationHandler()")
format.isKotlin() -> lines.append("val expectationHandler: ExpectationHandler = expectationHandler()")

}
lines.indented {
lines.add(".expect($expectationsMasterSwitch)")
if (format.isJava()) lines.append(";")
}

}

fun handleGenericFirstLine(call: RestCallAction, lines: Lines, res: RestCallResult, name: String, header: String){
lines.addEmpty()
when {
format.isKotlin() -> lines.append("val $name: ValidatableResponse = ")
format.isJava() -> lines.append("ValidatableResponse $name = ")
}
lines.append("given()$header")
}

fun handleExpectationSpecificLines(call: RestCallAction, lines: Lines, res: RestCallResult, name: String){
lines.addEmpty()
when{
format.isKotlin() -> lines.add("val json_$name: JsonPath = $name")
format.isJava() -> lines.add("JsonPath json_$name = $name")
}

lines.append(".extract().response().jsonPath()")
if(format.isJava()) {lines.append(";")}
}

fun handleExpectations(call: RestCallAction, lines: Lines, result: RestCallResult, active: Boolean, name: String) {

/*
TODO: This is a WiP to show the basic idea of the expectations:
An exception is thrown ONLY if the expectations are set to active.
If inactive, the condition will still be processed (with a goal to later adding to summaries or
other information processing/handling that might be needed), but it does not cause
the test case to fail regardless of truth value.
The example below aims to show this behaviour and provide a reminder.
As it is still work in progress, expect quite significant changes to this.
*/

lines.add("expectationHandler")
lines.indented {
lines.add(".expect()")
addExpectationsWithoutObjects(result, lines, name)
partialOracles.responseStructure(call, lines, result, name)
if (format.isJava()) { lines.append(";")}
}
}

private fun addExpectationsWithoutObjects(result: RestCallResult, lines: Lines, name: String) {
if (result.getBodyType() != null) {
// if there is a body, add expectations based on the body type. Right now only application/json is supported
when {
result.getBodyType()!!.isCompatible(MediaType.APPLICATION_JSON_TYPE) -> {
when (result.getBody()?.first()) {
'[' -> {
// This would be run if the JSON contains an array of objects
val resContents = Gson().fromJson(result.getBody(), ArrayList::class.java)
val printableTh = "numbersMatch(" +
"json_$name.getJsonObject(\"size\"), " +
"${resContents.size})"
lines.add(".that($expectationsMasterSwitch, ($printableTh))")
//TODO: individual objects in this collection also need handling
resContents.forEachIndexed { index, result ->
val fieldName = "get($index)"
val printableElement = handleFieldValuesExpect(name, fieldName, result)
if (printableElement != "null"
&& printableElement != TestCaseWriter.NOT_COVERED_YET
&& !printableTh.contains("logged")
) {
lines.add(".that($expectationsMasterSwitch, $printableElement)")
}
}

}
'{' -> {
// This would be run if the JSON contains a single object
val resContents = Gson().fromJson(result.getBody(), Object::class.java)

(resContents as Map<*, *>).keys
.filter { !it.toString().contains("timestamp") && !it.toString().contains("cache") }
.forEach {
val printableTh = handleFieldValuesExpect(name, "\'${it.toString()}\'", resContents[it])
if (printableTh != "null"
&& printableTh != TestCaseWriter.NOT_COVERED_YET
&& !printableTh.contains("logged") //again, unpleasant, but IDs logged as part of the error message are a problem
//TODO: find a more elegant way to deal with IDs, object refs, timestamps, etc.
) {
lines.add(".that($expectationsMasterSwitch, $printableTh)")
}
}
}
else -> {
// this shouldn't be run if the JSON is okay. Panic! Update: could also be null. Pause, then panic!
if(result.getBody() != null) lines.add(".that($expectationsMasterSwitch, subStringsMatch($name.extract().response().asString(), \"${GeneUtils.applyEscapes(result.getBody().toString(), mode = GeneUtils.EscapeMode.ASSERTION, format = format)}\"))")
else lines.add(".that($expectationsMasterSwitch, json_$name.toString().isEmpty())")
}
}
}
result.getBodyType()!!.isCompatible(MediaType.TEXT_PLAIN_TYPE) -> {
if(result.getBody() != null) lines.add(".that($expectationsMasterSwitch, subStringsMatch($name.extract().response().asString(), \"${GeneUtils.applyEscapes(result.getBody().toString(), mode = GeneUtils.EscapeMode.ASSERTION, format = format)}\"))")
else lines.add(".that($expectationsMasterSwitch, json_$name.toString().isEmpty())")
}
}
}
}

private fun handleFieldValuesExpect(objectName: String, fieldName: String, resContentsItem: Any?): String{
if (resContentsItem == null) {
return TestCaseWriter.NOT_COVERED_YET
}
else{
when(resContentsItem::class) {
Double::class -> return "numbersMatch(json_$objectName.getJsonObject(\"$fieldName\")," +
" ${resContentsItem as Double})"
String::class -> return "subStringsMatch(json_$objectName.getJsonObject(\"$fieldName\")," +
"\"${GeneUtils.applyEscapes((resContentsItem as String), mode = GeneUtils.EscapeMode.EXPECTATION, format = format)}\")"
else -> return TestCaseWriter.NOT_COVERED_YET
}
}
}

}

0 comments on commit 4c9226e

Please sign in to comment.
You can’t perform that action at this time.