Add multipage crawl

qwazr · Feb 24, 2017 · dfe65e7 · dfe65e7
1 parent 5974e7c
commit dfe65e7
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 3 deletions.
diff --git a/src/test/java/com/qwazr/crawler/web/test/WebCrawlerTest.java b/src/test/java/com/qwazr/crawler/web/test/WebCrawlerTest.java
@@ -73,6 +73,8 @@ private WebCrawlDefinition getNewWebCrawl() {
 		webCrawl.setJavascriptEnabled(false);
 		webCrawl.setImplicitlyWait(0);
 		webCrawl.setDownloadImages(false);
+		webCrawl.setMaxUrlNumber(10);
+		webCrawl.setMaxDepth(2);
 		webCrawl.entry_url = WebAppTestServer.URL;
 		return webCrawl;
 	}
@@ -90,7 +92,7 @@ private void crawlWait(final String sessionName, final int crawlCount) throws In
 	public void test300SimpleCrawl() throws InterruptedException {
 		final String sessionName = RandomStringUtils.randomAlphanumeric(10);
 		remote.runSession(sessionName, getNewWebCrawl());
-		crawlWait(sessionName, 1);
+		crawlWait(sessionName, 3);
 	}
 
 	@Test
@@ -101,8 +103,8 @@ public void test400CrawlEvent() throws InterruptedException {
 		webCrawl.scripts.put(WebCrawlDefinition.EventEnum.before_crawl,
 				new WebCrawlDefinition.Script(BeforeCrawl.class.getName()));
 		remote.runSession(sessionName, webCrawl);
-		crawlWait(sessionName, 1);
-		Assert.assertEquals(2, BeforeCrawl.count.get());
+		crawlWait(sessionName, 3);
+		Assert.assertEquals(4, BeforeCrawl.count.get());
 	}
 
 	@Test

diff --git a/src/test/statics/html/index.html b/src/test/statics/html/index.html
@@ -1,5 +1,13 @@
 <html>
 <body>
 <h1>Hello World</h1>
+<ul>
+    <li>
+        <a href="page1.html">page1</a>
+    </li>
+    <li>
+        <a href="page2.html">page2</a>
+    </li>
+</ul>
 </body>
 </html>
diff --git a/src/test/statics/html/page1.html b/src/test/statics/html/page1.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+<h1>Page 1</h1>
+</body>
+</html>
diff --git a/src/test/statics/html/page2.html b/src/test/statics/html/page2.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+<h1>Page 1</h1>
+</body>
+</html>