Permalink
Browse files

I spel good

  • Loading branch information...
ScottMansfield committed Jun 19, 2015
1 parent dbe7dbd commit a02a2ee296aebe65419ef4fec75501325961d363
Showing with 2 additions and 2 deletions.
  1. +2 −2 widow-core/src/main/java/com/widowcrawler/core/siteattr/RobotsTxtManager.java
@@ -52,7 +52,7 @@ public static RobotsTxt getByDomain(String domain) {
if (!fetchesInProgress.containsKey(domain)) {
fetchFuture = new FutureTask<>(() -> {
logger.info("Fetching and parsing robots.txt for domain " + domain);
RobotsTxt robotsTxt = Terminator.parse(makeRobotsTxtRL(domain));
RobotsTxt robotsTxt = Terminator.parse(makeRobotsTxtURL(domain));
robotstxtByDomain.put(domain, robotsTxt);
return Boolean.TRUE;
});
@@ -81,7 +81,7 @@ public static RobotsTxt getByDomain(String domain) {
}
}

private static String makeRobotsTxtRL(String domain) {
private static String makeRobotsTxtURL(String domain) {
return String.format("http://%s/robots.txt", domain);
}
}

0 comments on commit a02a2ee

Please sign in to comment.