RobotsTxt API supports variable user-agent

qwazr · Jun 18, 2017 · 53b3ddb · 53b3ddb
1 parent 83485af
commit 53b3ddb
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 11 deletions.
diff --git a/src/main/java/com/qwazr/crawler/web/WebCrawlThread.java b/src/main/java/com/qwazr/crawler/web/WebCrawlThread.java
@@ -366,7 +366,7 @@ private RobotsTxt.Status checkRobotsTxt(CurrentURI currentURI)
 				robotsTxt = RobotsTxt.download(driver.getProxy(), robotsTxtUserAgent, robotsTxtURI);
 				robotsTxtMap.put(robotsTxtURI, robotsTxt);
 			}
-			return robotsTxt.getStatus(uri);
+			return robotsTxt.getStatus(uri, robotsTxtUserAgent);
 		} finally {
 			timeTracker.next("Robots.txt check");
 		}

diff --git a/src/main/java/com/qwazr/crawler/web/robotstxt/RobotsTxt.java b/src/main/java/com/qwazr/crawler/web/robotstxt/RobotsTxt.java
@@ -58,17 +58,14 @@ public enum Status {
 	private static final Logger logger = LoggerFactory.getLogger(RobotsTxt.class);
 
 	private final RobotsTxtUserAgentMap userAgentMap;
-	private final String userAgent;
 	private final int httpStatusCode;
 
-	public RobotsTxt(final String userAgent, final InputStream input, final Charset charset) throws IOException {
-		this.userAgent = userAgent;
+	public RobotsTxt(final InputStream input, final Charset charset) throws IOException {
 		this.userAgentMap = RobotsTxtUserAgentMap.of(input, charset);
 		this.httpStatusCode = 200;
 	}
 
-	RobotsTxt(final String userAgent, final int statusCode) {
-		this.userAgent = userAgent;
+	RobotsTxt(final int statusCode) {
 		this.userAgentMap = null;
 		this.httpStatusCode = statusCode;
 	}
@@ -102,14 +99,16 @@ public static URI getRobotsURI(final URI uri) throws MalformedURLException, URIS
 	 * @throws MalformedURLException
 	 * @throws URISyntaxException
 	 */
-	public final Status getStatus(final URI uri) throws MalformedURLException, URISyntaxException {
-		final Status status = getStatusNoLogs(uri);
+	public final Status getStatus(final URI uri, final String userAgent)
+			throws MalformedURLException, URISyntaxException {
+		final Status status = getStatusNoLogs(uri, userAgent);
 		if (logger.isInfoEnabled())
 			logger.info("Check robots.txt returns " + status.name() + " for " + uri);
 		return status;
 	}
 
-	private Status getStatusNoLogs(final URI uri) throws MalformedURLException, URISyntaxException {
+	private Status getStatusNoLogs(final URI uri, final String userAgent)
+			throws MalformedURLException, URISyntaxException {
 		switch (httpStatusCode) {
 		case 400:
 		case 404:
@@ -149,7 +148,7 @@ public static RobotsTxt download(final WebCrawlDefinition.ProxyDefinition proxy,
 					final ContentType contentType = ContentType.getOrDefault(entity);
 					final Charset charset = contentType.getCharset();
 					try (final InputStream is = entity.getContent()) {
-						return new RobotsTxt(userAgent, is, charset == null ? CharsetUtils.CharsetUTF8 : charset);
+						return new RobotsTxt(is, charset == null ? CharsetUtils.CharsetUTF8 : charset);
 					}
 				} finally {
 					IOUtils.close((CloseableHttpResponse) response);
@@ -164,7 +163,7 @@ public static RobotsTxt download(final WebCrawlDefinition.ProxyDefinition proxy,
 				if (logger.isInfoEnabled())
 					logger.info("Get wrong status (" + sc + " code for: " + uri);
 			}
-			return new RobotsTxt(userAgent, sc);
+			return new RobotsTxt(sc);
 		}
 	}