updateRobots deem S_CONNECT_FAILED 404

internetarchive · Oct 6, 2017 · d4b8566 · d4b8566
1 parent 0ec055c
commit d4b8566
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java
@@ -157,6 +157,12 @@ public synchronized void updateRobots(CrawlURI curi) {
             gotSomething = true;
         }
 
+        // special deeming for connection-failure
+         if (curi.getFetchStatus() == S_CONNECT_FAILED) {
+             curi.setFetchStatus(S_DEEMED_NOT_FOUND);
+             gotSomething = true;
+         }
+
        if (!gotSomething) {
            // robots.txt fetch failed and exceptions (ignore/deeming) don't apply; no valid robots info yet
            validRobots = false;