[plugin-web-app-to-rest-api] Fix relative URL resolution in resource …

…steps (#1888)
vividus-framework · Aug 23, 2021 · 8a8b25b · 8a8b25b
1 parent 0ba131f
commit 8a8b25b
Show file tree

Hide file tree

Showing 6 changed files with 106 additions and 22 deletions.
diff --git a/docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc b/docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc
@@ -10,6 +10,39 @@ The plugin provides the integration between web application testing functionalit
 implementation(group: 'org.vividus', name: 'vividus-plugin-web-app-to-rest-api', version: '{current-version}')
 ----
 
+== Steps
+
+=== Validate resources
+
+Validates resources on web pages
+
+Resource validation logic:
+
+. If the `pages` row contains relative URL then it gets resolved against URL in `web-application.main-page-url` property, i.e. if the main page URL is `https://elderscrolls.bethesda.net/` and relative URL is `/skyrim10` the resulting URL will be `https://elderscrolls.bethesda.net/skyrim10`
+. Collect elements by the CSS selector from each page
+. Get either `href` or `src` attribute value from each element, if neither of the attributes exists the validation fails
+. For each received value execute https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD[HEAD] request
+.. If the status code is https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/200[200 OK] then the resource validation is considered as passed
+.. If the status code is one of https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404[404 Not Found], https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/405[405 Method Not Allowed], https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/501[501 Not Implemented], https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503[503 Service Unavailable] then GET request will be executed
+.. If the https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/GET[GET] status code is https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/200[200 OK] then the resource validation is considered as passed, otherwise failed
+
+[source,gherkin]
+----
+Then all resources by selector `$cssSelector` are valid on:$pages
+----
+
+. `$cssSelector` - The https://www.w3schools.com/cssref/css_selectors.asp[CSS selector]
+. `$pages` - The pages to validate resources on
+
+.Validate resources
+[source,gherkin]
+----
+Then all resources by selector a are valid on:
+|pages                        |
+|https://vividus.org/         |
+|/test-automation-made-awesome|
+----
+
 == Table Transformers
 
 === FROM_SITEMAP

diff --git a/vividus-plugin-html/src/main/java/org/vividus/util/HtmlUtils.java b/vividus-plugin-html/src/main/java/org/vividus/util/HtmlUtils.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2019-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,7 +28,12 @@ private HtmlUtils()
 
     public static Elements getElements(String html, String cssSelector)
     {
-        Document document = Jsoup.parse(html);
+        return getElements("", html, cssSelector);
+    }
+
+    public static Elements getElements(String baseUri, String html, String cssSelector)
+    {
+        Document document = Jsoup.parse(html, baseUri);
         return document.select(cssSelector);
     }
 }
diff --git a/vividus-plugin-html/src/test/java/org/vividus/util/HtmlUtilsTests.java b/vividus-plugin-html/src/test/java/org/vividus/util/HtmlUtilsTests.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2019-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,11 +17,19 @@
 package org.vividus.util;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
 
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
 import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
 
 class HtmlUtilsTests
 {
+    private static final String TITLE = "title";
     private static final String HTML =
             "<!DOCTYPE html>\n"
           + "<html>\n"
@@ -34,6 +42,25 @@ class HtmlUtilsTests
     @Test
     void shouldGetElementsBySelector()
     {
-        assertEquals("Title of the document", HtmlUtils.getElements(HTML, "title").get(0).text());
+        assertEquals("Title of the document", HtmlUtils.getElements(HTML, TITLE).get(0).text());
+    }
+
+    @Test
+    void shouldGetElementsBySelectorHtmlWithBaseUrl()
+    {
+        try (MockedStatic<Jsoup> jsoup = mockStatic(Jsoup.class))
+        {
+            String baseUri = "base-uri";
+            Document document = mock(Document.class);
+            Elements elements = mock(Elements.class);
+
+            jsoup.when(() -> Jsoup.parse(HTML, baseUri)).thenReturn(document);
+            when(document.select(TITLE)).thenReturn(elements);
+
+            assertEquals(elements, HtmlUtils.getElements(baseUri, HTML, TITLE));
+
+            jsoup.verify(() -> Jsoup.parse(HTML, baseUri));
+            jsoup.verifyNoMoreInteractions();
+        }
     }
 }
diff --git a/vividus-plugin-html/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/vividus-plugin-html/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
@@ -0,0 +1 @@
+mock-maker-inline
diff --git a/...b-app-to-rest-api/src/main/java/org/vividus/bdd/steps/integration/ResourceCheckSteps.java b/...b-app-to-rest-api/src/main/java/org/vividus/bdd/steps/integration/ResourceCheckSteps.java
@@ -16,8 +16,11 @@
 
 package org.vividus.bdd.steps.integration;
 
+import static org.vividus.util.HtmlUtils.getElements;
+
 import java.io.IOException;
 import java.net.URI;
+import java.util.Collection;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -43,14 +46,14 @@
 import org.vividus.softassert.SoftAssert;
 import org.vividus.testcontext.ContextCopyingExecutor;
 import org.vividus.ui.web.configuration.WebApplicationConfiguration;
-import org.vividus.util.HtmlUtils;
 import org.vividus.util.UriUtils;
 import org.vividus.validator.model.WebPageResourceValidation;
 
 public class ResourceCheckSteps
 {
     private static final Set<String> ALLOWED_SCHEMES = Set.of("http", "https");
-    private static final String EXCLUDE_PATTERN = "#";
+    private static final String URL_FRAGMENT = "#";
+    private static final String HREF_ATTR = "href";
 
     private final ResourceValidator<WebPageResourceValidation> resourceValidator;
     private final AttachmentPublisher attachmentPublisher;
@@ -79,8 +82,8 @@ public ResourceCheckSteps(ResourceValidator<WebPageResourceValidation> resourceV
 
     public void init()
     {
-        excludeHrefsPattern = Pattern.compile(uriToIgnoreRegex.map(p -> p + "|" + EXCLUDE_PATTERN)
-                                                           .orElse(EXCLUDE_PATTERN));
+        excludeHrefsPattern = Pattern.compile(uriToIgnoreRegex.map(p -> p + "|" + URL_FRAGMENT)
+                                                           .orElse(URL_FRAGMENT));
     }
 
     /**
@@ -101,7 +104,7 @@ public void init()
     public void checkResources(String cssSelector, String html) throws InterruptedException, ExecutionException
     {
         execute(() -> {
-            Stream<Element> resourcesToValidate = getElements(cssSelector, html);
+            Collection<Element> resourcesToValidate = getElements(html, cssSelector);
             Stream<WebPageResourceValidation> validations = createResourceValidations(resourcesToValidate,
                 p -> new WebPageResourceValidation(p.getLeft(), p.getRight()));
             validateResources(validations);
@@ -124,16 +127,11 @@ private WebPageResourceValidation validate(WebPageResourceValidation r)
                 : resourceValidator.perform(r);
     }
 
-    private Stream<Element> getElements(String cssSelector, String html)
-    {
-        return HtmlUtils.getElements(html, cssSelector).parallelStream();
-    }
-
-    private Stream<WebPageResourceValidation> createResourceValidations(Stream<Element> elements,
+    private Stream<WebPageResourceValidation> createResourceValidations(Collection<Element> elements,
             Function<Pair<URI, String>, WebPageResourceValidation> resourceValidationFactory)
     {
-        return elements.map(e ->
-            Pair.of(getElementAttribute(e, "href").orElseGet(() -> e.attr("src")).trim(), getSelector(e)))
+        return elements.parallelStream().map(e ->
+            Pair.of(getHrefAttribute(e).orElseGet(() -> e.attr("src")).trim(), getSelector(e)))
                        .filter(p -> !p.getKey().isEmpty() || softAssert.recordFailedAssertion(
                         "Element by selector " + p.getValue() + " doesn't contain href/src attributes"))
                        .map(p -> Pair.of(createUri(p.getKey()), p.getValue()))
@@ -160,9 +158,22 @@ private String getSelector(Element element)
         }
     }
 
-    private Optional<String> getElementAttribute(Element element, String attributeKey)
+    private static Optional<String> getHrefAttribute(Element element)
     {
-        return element.hasAttr(attributeKey) ? Optional.of(element.attr(attributeKey)) : Optional.empty();
+        String href = element.attr(HREF_ATTR);
+        if (!href.isEmpty())
+        {
+            if (URL_FRAGMENT.equals(href))
+            {
+                return Optional.of(href);
+            }
+
+            String absUrl = element.absUrl(HREF_ATTR);
+            // For scripts e.g. href="javascript:alert('...');" the abs url will be empty
+            return Optional.of(absUrl.isEmpty() ? href : absUrl);
+        }
+
+        return Optional.empty();
     }
 
     private URI createUri(String uri)
@@ -210,7 +221,7 @@ public void checkResources(String cssSelector, ExamplesTable pages) throws Inter
                              httpRequestExecutor.executeHttpRequest(HttpMethod.GET, pageURL, Optional.empty());
                              return Optional.ofNullable(httpTestContext.getResponse())
                                             .map(HttpResponse::getResponseBodyAsString)
-                                            .map(b -> createResourceValidations(getElements(cssSelector, b),
+                                            .map(b -> createResourceValidations(getElements(pageURL, b, cssSelector),
                                                 p -> new WebPageResourceValidation(p.getLeft(), p.getRight(), pageURL)))
                                             .orElseGet(() ->
                                                         Stream.of(brokenResourceValidation(pageURL, Optional.empty())));

diff --git a/...-to-rest-api/src/test/java/org/vividus/bdd/steps/integration/ResourceCheckStepsTests.java b/...-to-rest-api/src/test/java/org/vividus/bdd/steps/integration/ResourceCheckStepsTests.java
@@ -26,6 +26,7 @@
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;
@@ -38,6 +39,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 
 import org.jbehave.core.model.ExamplesTable;
 import org.junit.jupiter.api.Assertions;
@@ -56,6 +58,7 @@
 import org.vividus.softassert.SoftAssert;
 import org.vividus.testcontext.ContextCopyingExecutor;
 import org.vividus.ui.web.configuration.WebApplicationConfiguration;
+import org.vividus.util.Sleeper;
 import org.vividus.validator.model.WebPageResourceValidation;
 
 @ExtendWith(MockitoExtension.class)
@@ -193,7 +196,11 @@ void shouldCheckResourcesFromPages() throws IOException, InterruptedException, E
         resourceCheckSteps.init();
         ExamplesTable examplesTable =
                 new ExamplesTable("|pages|\n|https://first.page|\n|https://second.page|");
-        when(webApplicationConfiguration.getMainApplicationPageUrl()).thenReturn(VIVIDUS_URI);
+        lenient().doAnswer(call ->
+        {
+            Sleeper.sleep(5, TimeUnit.SECONDS);
+            return null;
+        }).when(httpRequestExecutor).executeHttpRequest(HttpMethod.GET, SECOND_PAGE_URL, Optional.empty());
         resourceCheckSteps.checkResources(LINK_SELECTOR, examplesTable);
         verify(httpRequestExecutor).executeHttpRequest(HttpMethod.GET, SECOND_PAGE_URL, Optional.empty());
         verify(httpRequestExecutor).executeHttpRequest(HttpMethod.GET, FIRST_PAGE_URL, Optional.empty());
@@ -204,9 +211,9 @@ void shouldCheckResourcesFromPages() throws IOException, InterruptedException, E
             assertThat(validationsToReport, hasSize(8));
             Iterator<WebPageResourceValidation> resourceValidations = validationsToReport.iterator();
             validate(resourceValidations.next(), SERENITY_URI, HTTP_ID, CheckStatus.PASSED);
+            validate(resourceValidations.next(), URI.create(FIRST_PAGE_URL + "/faq"), RELATIVE_ID, CheckStatus.PASSED);
             validate(resourceValidations.next(), VIVIDUS_URI, HTTPS_ID, CheckStatus.PASSED);
             validate(resourceValidations.next(), VIVIDUS_ABOUT_URI, ABOUT_ID, CheckStatus.PASSED);
-            validate(resourceValidations.next(), FAQ_URI, RELATIVE_ID, CheckStatus.PASSED);
             validate(resourceValidations.next(), SHARP_URI, SHARP_ID, CheckStatus.FILTERED);
             validate(resourceValidations.next(), FTP_URI, FTP_ID, CheckStatus.FILTERED);
             validate(resourceValidations.next(), JS_URI, JS_ID, CheckStatus.FILTERED);