Report status code before schema

webtau-feature-testing/examples/scenarios/rest/openapi/api-spec.json

@@ -44,13 +44,13 @@
           "200": {
             "description": "customer already exists",
             "schema": {
-              "$ref": "#/definitions/Employee"
+              "$ref": "#/definitions/Id"
             }
           },
           "201": {
             "description": "customer created successfully",
             "schema": {
-              "$ref": "#/definitions/Employee"
+              "$ref": "#/definitions/Id"
             }
           }
         }
@@ -72,6 +72,17 @@
           "type": "string"
         }
       }
+    },
+    "Id": {
+      "type": "object",
+      "required": [
+        "id"
+      ],
+      "properties": {
+        "id": {
+          "type": "string"
+        }
+      }
     }
   }
 }

webtau-feature-testing/test-expectations/scenarios/rest/openapi/disableOpenApiValidation/run-details.json

@@ -7,13 +7,13 @@
   }, {
     "scenario" : "disable request validation",
     "stepsSummary" : {
-      "numberOfFailed" : 2
+      "numberOfSuccessful" : 3
     }
   }, {
     "scenario" : "disable response validation",
     "stepsSummary" : {
       "numberOfSuccessful" : 3
     }
   } ],
-  "exitCode" : 1
-}
+  "exitCode" : 0
+}

webtau-http-groovy/src/test/groovy/com/twosigma/webtau/http/HttpGroovyTest.groovy

@@ -23,6 +23,7 @@ import com.twosigma.webtau.http.datanode.DataNode
 import com.twosigma.webtau.http.datanode.GroovyDataNode
 import com.twosigma.webtau.http.testserver.TestServerResponse
 import com.twosigma.webtau.http.validation.HttpResponseValidator
+import com.twosigma.webtau.http.validation.HttpValidationHandlers
 import com.twosigma.webtau.utils.ResourceUtils
 import com.twosigma.webtau.utils.UrlUtils
 import org.junit.*
@@ -982,6 +983,75 @@ class HttpGroovyTest implements HttpConfiguration {
         http.lastValidationResult.errorMessage.should == ~/java.lang.IllegalArgumentException: Request header is null/
     }
 
+    @Test
+    void "implicit status code mismatch reported before additional validators"() {
+        HttpValidationHandlers.register { result -> throw new AssertionError((Object)"schema validation error") }
+        try {
+            code {
+                http.get("/notfound") {}
+            } should throwException(AssertionError, ~/(?s)header.statusCode.*404.*200/)
+        } finally {
+            HttpValidationHandlers.reset()
+        }
+    }
+
+    @Test
+    void "explicit status code mismatch reported before additional validators"() {
+        HttpValidationHandlers.register { result -> throw new AssertionError((Object)"schema validation error") }
+        try {
+            code {
+                http.get("/notfound") {
+                    statusCode.should == 200
+                }
+            } should throwException(AssertionError, ~/(?s)header.statusCode.*404.*200/)
+        } finally {
+            HttpValidationHandlers.reset()
+        }
+    }
+
+    @Test
+    void "status code mismatch reported before additional validators and failing body assertions"() {
+        HttpValidationHandlers.register { result -> throw new AssertionError((Object)"schema validation error") }
+        try {
+            code {
+                http.get("/notfound") {
+                    id.should == 'foo'
+                }
+            } should throwException(AssertionError, ~/(?s)header.statusCode.*404.*200/)
+        } finally {
+            HttpValidationHandlers.reset()
+        }
+    }
+
+    @Test
+    void "assertion and additional validation errors"() {
+        HttpValidationHandlers.register { result -> throw new AssertionError((Object)"schema validation error") }
+        try {
+            code {
+                http.get("/notfound") {
+                    statusCode.should == 404
+                    id.should == 'foo'
+                }
+            } should throwException(AssertionError, ~/expected: "foo"/)
+        } finally {
+            HttpValidationHandlers.reset()
+        }
+    }
+
+    @Test
+    void "additional validator errors reported if status code is correct"() {
+        HttpValidationHandlers.register { result -> throw new AssertionError((Object)"schema validation error") }
+        try {
+            code {
+                http.get("/notfound") {
+                    statusCode.should == 404
+                }
+            } should throwException(AssertionError, ~/schema validation error/)
+        } finally {
+            HttpValidationHandlers.reset()
+        }
+    }
+
     @Override
     String fullUrl(String url) {
         if (UrlUtils.isFull(url)) {

webtau-http/src/main/java/com/twosigma/webtau/http/Http.java

@@ -436,8 +436,6 @@ private <R> R validateAndRecord(HttpValidationResult validationResult,
         validationResult.setResponseHeaderNode(header);
         validationResult.setResponseBodyNode(body);
 
-        HttpValidationHandlers.validate(validationResult);
-
         ExpectationHandler recordAndThrowHandler = (valueMatcher, actualPath, actualValue, message) -> {
             validationResult.addMismatch(message);
             return ExpectationHandler.Flow.PassToNext;
@@ -458,6 +456,8 @@ private <R> R validateAndRecord(HttpValidationResult validationResult,
                 return null;
             });
 
+            HttpValidationHandlers.validate(validationResult);
+
             return extracted;
         } catch (Throwable e) {
             ExpectationHandlers.withAdditionalHandler((valueMatcher, actualPath, actualValue, message) -> {

webtau-http/src/main/java/com/twosigma/webtau/http/validation/HttpValidationHandlers.java

@@ -23,6 +23,15 @@
 public class HttpValidationHandlers {
     private static final List<HttpValidationHandler> configurations = ServiceLoaderUtils.load(HttpValidationHandler.class);
 
+    public static void register(HttpValidationHandler handler) {
+        configurations.add(handler);
+    }
+
+    public static void reset() {
+        configurations.clear();
+        configurations.addAll(ServiceLoaderUtils.load(HttpValidationHandler.class));
+    }
+
     public static void validate(HttpValidationResult validationResult) {
         configurations.forEach(c -> c.validate(validationResult));
     }