added dns resolve to HTTPClient POST using a dns cache to prevent tha…

…t that not-thread-safe built-in dns cache inside apache http client is used git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7513 6c8d7289-2bf4-0310-a012-ef5d649a1542
yacy · Feb 22, 2011 · cd19d05 · cd19d05
1 parent d28f804
commit cd19d05
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 9 deletions.
diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java
@@ -1786,7 +1786,7 @@ private Document[] parseDocument(final Response response) throws InterruptedExce
 
                 // process the next hyperlink
                 nextUrl = nextEntry.getKey();
-                String u = nextUrl.toNormalform(true, true, true);
+                String u = nextUrl.toNormalform(true, true, false, true);
                 if (!(u.startsWith("http://") || u.startsWith("ftp://") || u.startsWith("smb://") || u.startsWith("file://"))) continue;
                 // enqueue the hyperlink into the pre-notice-url db
                 try {

diff --git a/source/net/yacy/cora/document/MultiProtocolURI.java b/source/net/yacy/cora/document/MultiProtocolURI.java
@@ -845,19 +845,19 @@ private static CharType charType(char c) {
     }
 
     public String toNormalform(final boolean excludeReference, final boolean stripAmp) {
-        return toNormalform(excludeReference, stripAmp, false);
+        return toNormalform(excludeReference, stripAmp, false, false);
     }
 
     private static final Pattern ampPattern = Pattern.compile("&amp;");
-    public String toNormalform(final boolean excludeReference, final boolean stripAmp, final boolean removeSessionID) {
-        String result = toNormalform0(excludeReference, removeSessionID); 
+    public String toNormalform(final boolean excludeReference, final boolean stripAmp, final boolean resolveHost, final boolean removeSessionID) {
+        String result = toNormalform0(excludeReference, resolveHost, removeSessionID); 
         if (stripAmp) {
             result = ampPattern.matcher(result).replaceAll("&");
         }
         return result;
     }
 
-    private String toNormalform0(final boolean excludeReference, final boolean removeSessionID) {
+    private String toNormalform0(final boolean excludeReference, final boolean resolveHost, final boolean removeSessionID) {
         // generates a normal form of the URL
         boolean defaultPort = false;
         if (this.protocol.equals("mailto")) {
@@ -882,7 +882,12 @@ private String toNormalform0(final boolean excludeReference, final boolean remov
                 u.append(this.userInfo);
                 u.append("@");
             }
-            u.append(this.getHost().toLowerCase());
+            if (resolveHost) {
+                u.append(Domains.dnsResolve(this.getHost().toLowerCase()).getHostAddress());
+            } else {
+                u.append(this.getHost().toLowerCase());
+            }
+
         }
         if (!defaultPort) {
             u.append(":");

diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java
@@ -481,6 +481,7 @@ private void execute(final HttpUriRequest httpUriRequest) throws IOException {
     	        //assert entity.getContentLength() >= 0;
     	        assert !hrequest.expectContinue();
     	    }
+
 			httpResponse = httpClient.execute(httpUriRequest, httpContext);
         } catch (Exception e) {
             //e.printStackTrace();

diff --git a/source/net/yacy/cora/protocol/http/HTTPConnector.java b/source/net/yacy/cora/protocol/http/HTTPConnector.java
@@ -80,7 +80,7 @@ public byte[] post(final MultiProtocolURI url, final int timeout, final String v
 		client.setUserAgent(this.userAgent);
 		client.setHost(vhost);
 
-		return client.POSTbytes(url.toNormalform(false, false), post, usegzip);
+		return client.POSTbytes(url.toNormalform(false, false, true, false), post, usegzip);
 	}
 
 }
diff --git a/source/net/yacy/kelondro/data/meta/DigestURI.java b/source/net/yacy/kelondro/data/meta/DigestURI.java
@@ -205,7 +205,7 @@ private final byte[] urlHashComputation() {
         final StringBuilder hashs = new StringBuilder(12);
         assert hashs.length() == 0;
         // form the 'local' part of the hash
-        String normalform = toNormalform(true, true, true);
+        String normalform = toNormalform(true, true, false, true);
         String b64l = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(normalform));
         if (b64l.length() < 5) return null;
         hashs.append(b64l.substring(0, 5)); // 5 chars

diff --git a/test/net/yacy/cora/document/MultiProtocolURITest.java b/test/net/yacy/cora/document/MultiProtocolURITest.java
@@ -28,7 +28,7 @@ public class MultiProtocolURITest {
 		for (int i=0; i<testURIs.length; i++) {
 			MultiProtocolURI uri = new MultiProtocolURI(testURIs[i][0]);
 
-   			assertEquals(uri.toNormalform(true, true, true), testURIs[i][1]);
+   			assertEquals(uri.toNormalform(true, true, false, true), testURIs[i][1]);
 		}
 	}
 }