Report status code before schema (#338)

testingisdocumenting · May 21, 2019 · 4594c00 · 4594c00
1 parent b204eb2
commit 4594c00
Show file tree

Hide file tree

Showing 11 changed files with 117 additions and 24 deletions.
diff --git a/webtau-config/src/main/java/com/twosigma/webtau/cfg/WebTauConfig.java b/webtau-config/src/main/java/com/twosigma/webtau/cfg/WebTauConfig.java
@@ -47,7 +47,7 @@ public class WebTauConfig {
     private final ConfigValue disableFollowingRedirects = declareBoolean("disableRedirects", "disable following of redirects from HTTP calls");
     private final ConfigValue maxRedirects = declare("maxRedirects", "Maximum number of redirects to follow for an HTTP call", () -> 20);
     private final ConfigValue userAgent = declare("userAgent", "User agent to send on HTTP requests",
-            () -> "webtau/" + getClass().getPackage().getImplementationVersion());
+            () -> "webtau/" + WebTauMeta.getVersion());
     private final ConfigValue removeWebtauFromUserAgent = declare("removeWebtauFromUserAgent",
             "By default webtau appends webtau and its version to the user-agent, this disables that part",
             () -> false);

diff --git a/.../com/twosigma/webtau/meta/WebTauMeta.java → ...a/com/twosigma/webtau/cfg/WebTauMeta.java b/.../com/twosigma/webtau/meta/WebTauMeta.java → ...a/com/twosigma/webtau/cfg/WebTauMeta.java
@@ -14,9 +14,7 @@
  * limitations under the License.
  */
 
-package com.twosigma.webtau.meta;
-
-import com.twosigma.webtau.utils.ResourceUtils;
+package com.twosigma.webtau.cfg;
 
 public class WebTauMeta {
     private static final WebTauMeta INSTANCE = new WebTauMeta();
@@ -27,6 +25,6 @@ public static String getVersion() {
     }
 
     private WebTauMeta() {
-        version = ResourceUtils.textContent("webtau.version").trim();
+        version = getClass().getPackage().getImplementationVersion();
     }
 }
diff --git a/webtau-core/pom.xml b/webtau-core/pom.xml
@@ -53,13 +53,6 @@
     </dependencies>
 
     <build>
-        <resources>
-            <resource>
-                <directory>src/main/resources</directory>
-                <filtering>true</filtering>
-            </resource>
-        </resources>
-
         <plugins>
             <plugin>
                 <groupId>org.codehaus.gmavenplus</groupId>

diff --git a/webtau-core/src/main/resources/webtau.version b/webtau-core/src/main/resources/webtau.version
diff --git a/webtau-feature-testing/examples/scenarios/rest/openapi/api-spec.json b/webtau-feature-testing/examples/scenarios/rest/openapi/api-spec.json
@@ -44,13 +44,13 @@
           "200": {
             "description": "customer already exists",
             "schema": {
-              "$ref": "#/definitions/Employee"
+              "$ref": "#/definitions/Id"
             }
           },
           "201": {
             "description": "customer created successfully",
             "schema": {
-              "$ref": "#/definitions/Employee"
+              "$ref": "#/definitions/Id"
             }
           }
         }
@@ -72,6 +72,17 @@
           "type": "string"
         }
       }
+    },
+    "Id": {
+      "type": "object",
+      "required": [
+        "id"
+      ],
+      "properties": {
+        "id": {
+          "type": "string"
+        }
+      }
     }
   }
 }
diff --git a/...esting/test-expectations/scenarios/rest/openapi/disableOpenApiValidation/run-details.json b/...esting/test-expectations/scenarios/rest/openapi/disableOpenApiValidation/run-details.json
@@ -7,13 +7,13 @@
   }, {
     "scenario" : "disable request validation",
     "stepsSummary" : {
-      "numberOfFailed" : 2
+      "numberOfSuccessful" : 3
     }
   }, {
     "scenario" : "disable response validation",
     "stepsSummary" : {
       "numberOfSuccessful" : 3
     }
   } ],
-  "exitCode" : 1
-}
+  "exitCode" : 0
+}
diff --git a/webtau-groovy/src/main/groovy/com/twosigma/webtau/cfg/WebTauCliArgsConfig.groovy b/webtau-groovy/src/main/groovy/com/twosigma/webtau/cfg/WebTauCliArgsConfig.groovy
@@ -16,7 +16,6 @@
 
 package com.twosigma.webtau.cfg
 
-import com.twosigma.webtau.meta.WebTauMeta
 import org.apache.commons.cli.CommandLine
 import org.apache.commons.cli.DefaultParser
 import org.apache.commons.cli.HelpFormatter

diff --git a/webtau-http-groovy/src/test/groovy/com/twosigma/webtau/http/HttpGroovyTest.groovy b/webtau-http-groovy/src/test/groovy/com/twosigma/webtau/http/HttpGroovyTest.groovy
@@ -23,6 +23,8 @@ import com.twosigma.webtau.http.datanode.DataNode
 import com.twosigma.webtau.http.datanode.GroovyDataNode
 import com.twosigma.webtau.http.testserver.TestServerResponse
 import com.twosigma.webtau.http.validation.HttpResponseValidator
+import com.twosigma.webtau.http.validation.HttpValidationHandler
+import com.twosigma.webtau.http.validation.HttpValidationHandlers
 import com.twosigma.webtau.utils.ResourceUtils
 import com.twosigma.webtau.utils.UrlUtils
 import org.junit.*
@@ -990,6 +992,75 @@ class HttpGroovyTest implements HttpConfiguration {
         http.lastValidationResult.errorMessage.should == ~/java.lang.IllegalArgumentException: Request header is null/
     }
 
+    private static void withFailingHandler(Closure closure) {
+        HttpValidationHandler handler = { result -> throw new AssertionError((Object)"schema validation error") }
+        HttpValidationHandlers.withAdditionalHandler(handler, closure)
+    }
+
+    static String expected404 = '\n' +
+        'mismatches:\n' +
+        '\n' +
+        'header.statusCode:   actual: 404 <java.lang.Integer>\n' +
+        '                   expected: 200 <java.lang.Integer>'
+
+    @Test
+    void "reports implicit status code mismatch instead of additional validator errors"() {
+        withFailingHandler {
+            code {
+                http.get("/notfound") {}
+            } should throwException(AssertionError, expected404)
+        }
+    }
+
+    @Test
+    void "reports explicit status code mismatch instead of additional validator errors"() {
+        withFailingHandler {
+            code {
+                http.get("/notfound") {
+                    statusCode.should == 200
+                }
+            } should throwException(AssertionError, expected404)
+        }
+    }
+
+    @Test
+    void "reports status code mismatch instead of additional validator errors or failing body assertions"() {
+        withFailingHandler {
+            code {
+                http.get("/notfound") {
+                    id.should == 'foo'
+                }
+            } should throwException(AssertionError, expected404)
+        }
+    }
+
+    @Test
+    void "reports body assertions instead of additional validation errors"() {
+        withFailingHandler() {
+            code {
+                http.get("/notfound") {
+                    statusCode.should == 404
+                    id.should == 'foo'
+                }
+            } should throwException(AssertionError, '\n' +
+                'mismatches:\n' +
+                '\n' +
+                'body.id:   actual: null\n' +
+                '         expected: "foo" <java.lang.String>')
+        }
+    }
+
+    @Test
+    void "reports additional validator errors if status code is correct"() {
+        withFailingHandler {
+            code {
+                http.get("/notfound") {
+                    statusCode.should == 404
+                }
+            } should throwException(AssertionError, 'schema validation error')
+        }
+    }
+
     @Override
     String fullUrl(String url) {
         if (UrlUtils.isFull(url)) {

diff --git a/webtau-http/src/main/java/com/twosigma/webtau/http/Http.java b/webtau-http/src/main/java/com/twosigma/webtau/http/Http.java
@@ -496,8 +496,6 @@ private <R> R validateAndRecord(HttpValidationResult validationResult,
         validationResult.setResponseHeaderNode(header);
         validationResult.setResponseBodyNode(body);
 
-        HttpValidationHandlers.validate(validationResult);
-
         ExpectationHandler recordAndThrowHandler = (valueMatcher, actualPath, actualValue, message) -> {
             validationResult.addMismatch(message);
             return ExpectationHandler.Flow.PassToNext;
@@ -518,6 +516,8 @@ private <R> R validateAndRecord(HttpValidationResult validationResult,
                 return null;
             });
 
+            HttpValidationHandlers.validate(validationResult);
+
             return extracted;
         } catch (Throwable e) {
             ExpectationHandlers.withAdditionalHandler((valueMatcher, actualPath, actualValue, message) -> {

diff --git a/webtau-http/src/main/java/com/twosigma/webtau/http/validation/HttpValidationHandlers.java b/webtau-http/src/main/java/com/twosigma/webtau/http/validation/HttpValidationHandlers.java
@@ -18,12 +18,34 @@
 
 import com.twosigma.webtau.utils.ServiceLoaderUtils;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
 
 public class HttpValidationHandlers {
-    private static final List<HttpValidationHandler> configurations = ServiceLoaderUtils.load(HttpValidationHandler.class);
+    private static final List<HttpValidationHandler> globalHandlers = ServiceLoaderUtils.load(HttpValidationHandler.class);
+    private static final ThreadLocal<List<HttpValidationHandler>> localHandlers = ThreadLocal.withInitial(ArrayList::new);
+
+    public static <R> R withAdditionalHandler(HttpValidationHandler handler, Supplier<R> code) {
+        try {
+            addLocal(handler);
+            return code.get();
+        } finally {
+            removeLocal(handler);
+        }
+    }
 
     public static void validate(HttpValidationResult validationResult) {
-        configurations.forEach(c -> c.validate(validationResult));
+        Stream.concat(localHandlers.get().stream(), globalHandlers.stream())
+                .forEach(c -> c.validate(validationResult));
+    }
+
+    private static void addLocal(HttpValidationHandler handler) {
+        localHandlers.get().add(handler);
+    }
+
+    private static void removeLocal(HttpValidationHandler handler) {
+        localHandlers.get().remove(handler);
     }
 }
diff --git a/webtau-report/src/main/java/com/twosigma/webtau/report/HtmlReportGenerator.java b/webtau-report/src/main/java/com/twosigma/webtau/report/HtmlReportGenerator.java
@@ -17,9 +17,9 @@
 package com.twosigma.webtau.report;
 
 import com.twosigma.webtau.cfg.ConfigValue;
+import com.twosigma.webtau.cfg.WebTauMeta;
 import com.twosigma.webtau.console.ConsoleOutputs;
 import com.twosigma.webtau.console.ansi.Color;
-import com.twosigma.webtau.meta.WebTauMeta;
 import com.twosigma.webtau.utils.FileUtils;
 import com.twosigma.webtau.utils.JsonUtils;
 import com.twosigma.webtau.utils.ResourceUtils;