From 1f4978e0b7b0a27f4579deabe7eb0697ae06f3ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B8ren=20Vejrup=20Carlsen?= <svc@kb.dk>
Date: Fri, 25 May 2018 13:02:46 +0200
Subject: [PATCH] NAS-2750 - logging level changed

---
 .../harvesting/ExtendedDNSFetcher.java         | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/harvester/heritrix3/heritrix3-extensions/src/main/java/dk/netarkivet/harvester/harvesting/ExtendedDNSFetcher.java b/harvester/heritrix3/heritrix3-extensions/src/main/java/dk/netarkivet/harvester/harvesting/ExtendedDNSFetcher.java
index 2549948e35..33e06a0ba4 100644
--- a/harvester/heritrix3/heritrix3-extensions/src/main/java/dk/netarkivet/harvester/harvesting/ExtendedDNSFetcher.java
+++ b/harvester/heritrix3/heritrix3-extensions/src/main/java/dk/netarkivet/harvester/harvesting/ExtendedDNSFetcher.java
@@ -68,6 +68,7 @@ public class ExtendedDNSFetcher extends Processor {
     /**
      * If a DNS lookup fails, whether or not to fall back to InetAddress
      * resolution, which may use local 'hosts' files or other mechanisms.
+     * It is disabled by default.
      */
     {
         setAcceptNonDnsResolves(false);
@@ -85,6 +86,7 @@ public void setAcceptNonDnsResolves(boolean acceptNonDnsResolves) {
      * 
      * This should not generally be used in production as it will prevent 
      * DNS lookups from being recorded properly.
+     * It is disabled by default.
      * 
      */
     {
@@ -111,7 +113,7 @@ public void setServerCache(ServerCache serverCache) {
     
     /**
      * Whether or not to perform an on-the-fly digest hash of retrieved
-     * content-bodies.
+     * content-bodies. It is enabled by default
      */
     {
         setDigestContent(true);
@@ -125,7 +127,7 @@ public void setDigestContent(boolean digest) {
 
     /**
      * Which algorithm (for example MD5 or SHA-1) to use to perform an 
-     * on-the-fly digest hash of retrieved content-bodies.
+     * on-the-fly digest hash of retrieved content-bodies. The default is 'sha1'
      */
     protected String digestAlgorithm = "sha1"; 
     public String getDigestAlgorithm() {
@@ -136,7 +138,7 @@ public void setDigestAlgorithm(String digestAlgorithm) {
     }
     
     /**
-     * Whether or not to prevalidate dnsname as a valid host 
+     * Whether or not to prevalidate dnsname as a valid host. It is disabled by default.
      */
     {
         setPrevalidateHostname(false);
@@ -148,11 +150,6 @@ public boolean getPrevalidateHostname() {
     public void setPrevalidateHostname(boolean prevalidateHostname) {
         kp.put("prevalidateHostname",prevalidateHostname);
     }
-    
-    
-    
-    
-    
 
     private static final long DEFAULT_TTL_FOR_NON_DNS_RESOLVES
         = 6 * 60 * 60; // 6 hrs
@@ -200,7 +197,10 @@ protected void innerProcess(CrawlURI curi) {
         
         if (getPrevalidateHostname()) {
             if (!validHostName(dnsName)) {
-                setUnresolvable(curi, targetHost);
+                targetHost.setIP(null, 0);
+                curi.setFetchStatus(S_DOMAIN_UNRESOLVABLE); // or S_UNFETCHABLE_URI
+                logger.info("URI '" + curi.getURI() + "' rejected, as hostname '" + dnsName + "' is considered invalid");
+                return;
             }
         }