diff --git a/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformerTests.java b/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformerTests.java index 5419df4780..6adde8e63d 100644 --- a/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformerTests.java +++ b/vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HeadlessCrawlerTableTransformerTests.java @@ -90,7 +90,8 @@ class HeadlessCrawlerTableTransformerTests private static final String CRAWLING_RELATIVE_URL = "/page"; - private static final String OUTGOING_ABSOLUT_URL = "http://some.url/path"; + private static final String OUTGOING_RELATIVE_URL = "/path"; + private static final String OUTGOING_ABSOLUTE_URL = "http://some.url" + OUTGOING_RELATIVE_URL; private static final String EXCLUDE_EXTENSIONS_REGEX = "js|css"; private static final String EXCLUDE_URLS_REGEX = ".*broken-link*"; @@ -129,7 +130,7 @@ void testFetchUrlsSuccessfully(String mainAppPageRelativeUrl, Set seedRe transformer.setExcludeUrlsRegex(EXCLUDE_URLS_REGEX); transformer.setMainPageUrlProperty(MAIN_APP_PROP); Set urls = testFetchUrls(mainAppPageRelativeUrl, expectedSeedRelativeUrls); - assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUT_URL))); + assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUTE_URL))); verifyNoInteractions(redirectsProvider); assertThat(logger.getLoggingEvents(), is(List.of(getMainAppPageWarn()))); } @@ -153,7 +154,7 @@ void shouldFilterUrlsWhenLastRedirectUrlAlreadyInTheSet() throws IOException, In transformer.setFilterRedirects(true); transformer.setSeedRelativeUrls(toSet(PATH2, PATH3)); transformer.setMainPageUrlProperty(MAIN_APP_PROP); - URI outgoingURI = URI.create(OUTGOING_ABSOLUT_URL); + URI outgoingURI = URI.create(OUTGOING_ABSOLUTE_URL); when(redirectsProvider.getRedirects(outgoingURI)).thenReturn(List.of(outgoingURI)); Set urls = testFetchUrls(ROOT, asList(PATH2, SLASH_PATH3)); assertThat(urls, equalTo(Set.of())); @@ -167,11 +168,11 @@ void shouldTreatInvalidStatusCodeAsNoRedirects() throws IOException, Interrupted transformer.setFilterRedirects(true); transformer.setSeedRelativeUrls(toSet(PATH2, PATH3)); transformer.setMainPageUrlProperty(MAIN_APP_PROP); - URI outgoingURI = URI.create(OUTGOING_ABSOLUT_URL); + URI outgoingURI = URI.create(OUTGOING_ABSOLUTE_URL); var httpResponseException = new HttpResponseException(HttpStatus.SC_NOT_FOUND, ""); when(redirectsProvider.getRedirects(outgoingURI)).thenThrow(httpResponseException); Set urls = testFetchUrls(ROOT, List.of(PATH2, SLASH_PATH3)); - assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUT_URL))); + assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUTE_URL))); assertThat(logger.getLoggingEvents(), is(List.of(getMainAppPageWarn(), warn(httpResponseException, "Exception during redirects receiving")))); } @@ -182,10 +183,10 @@ void shouldNotFilterUrlsWhenLastRedirectUrlNotInTheSet() throws IOException, Int transformer.setFilterRedirects(true); transformer.setSeedRelativeUrls(toSet(PATH2, PATH3)); transformer.setMainPageUrlProperty(MAIN_APP_PROP); - URI outgoingURI = URI.create(OUTGOING_ABSOLUT_URL); + URI outgoingURI = URI.create(OUTGOING_ABSOLUTE_URL); when(redirectsProvider.getRedirects(outgoingURI)).thenReturn(List.of(URI.create("http://some.url/other"))); Set urls = testFetchUrls(ROOT, asList(PATH2, SLASH_PATH3)); - assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUT_URL))); + assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUTE_URL))); assertThat(logger.getLoggingEvents(), is(List.of(getMainAppPageWarn(), info(REDIRECT_FILTER_LOG, System.lineSeparator(), "http://some.url/path -> http://some.url/other")))); } @@ -195,7 +196,7 @@ void testFetchUrlsTwice() throws IOException, InterruptedException { transformer.setSeedRelativeUrls(toSet(SEED)); Set urls = testFetchUrls(DEFAULT_RELATIVE_URL, List.of(SEED)); - assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUT_URL))); + assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUTE_URL))); TableProperties tableProperties = buildTableProperties(); Set urls2 = transformer.fetchUrls(tableProperties); verifyNoMoreInteractions(crawlControllerFactory); @@ -214,10 +215,10 @@ void testFetchUrlsTwiceWithSameProperties() throws IOException, InterruptedExcep transformer.setSeedRelativeUrls(toSet(seedRelativeUrlsProperty)); Set urls = runUrlFetching(mainAppPage, tableProperties, List.of(seedRelativeUrlsProperty), crawlController, ordered); - assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUT_URL))); + assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUTE_URL))); Set urls2 = transformer.fetchUrls(tableProperties); verifyNoMoreInteractions(crawlControllerFactory, crawlController); - assertThat(urls2, equalTo(Set.of(OUTGOING_ABSOLUT_URL))); + assertThat(urls2, equalTo(Set.of(OUTGOING_ABSOLUTE_URL))); assertSame(urls, urls2); verifyNoInteractions(redirectsProvider); } @@ -228,7 +229,7 @@ void testFetchUrlsWhenSeedRelativeUrlsAreSetViaConfiguration() throws IOExceptio String seedRelativeUrl = "/fromConfig"; transformer.setSeedRelativeUrls(Set.of(seedRelativeUrl)); Set urls = testFetchUrls(DEFAULT_RELATIVE_URL, List.of(seedRelativeUrl)); - assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUT_URL))); + assertThat(urls, equalTo(Set.of(OUTGOING_ABSOLUTE_URL))); verifyNoInteractions(redirectsProvider); } @@ -261,7 +262,7 @@ private Set runUrlFetching(String mainAppPage, TableProperties tableProp { LinkCrawler linkCrawler = ((LinkCrawlerFactory) factory).newInstance(); HtmlParseData htmlParseData = new HtmlParseData(); - String outgoingUrl = UriUtils.buildNewUrl(mainAppPage, OUTGOING_ABSOLUT_URL).toString(); + String outgoingUrl = UriUtils.buildNewUrl(mainAppPage, OUTGOING_RELATIVE_URL).toString(); htmlParseData.setOutgoingUrls(Set.of(createWebUrl(outgoingUrl))); String crawlingPageUrl = UriUtils.buildNewUrl(mainAppPage, CRAWLING_RELATIVE_URL).toString(); WebURL crawlingPageWebUrl = createWebUrl(crawlingPageUrl);