remove reference in crawl entries

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5623 6c8d7289-2bf4-0310-a012-ef5d649a1542
yacy · Feb 19, 2009 · 4f9dae2 · 4f9dae2
1 parent 1ba4301
commit 4f9dae2
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 2 deletions.
diff --git a/source/de/anomic/crawler/CrawlEntry.java b/source/de/anomic/crawler/CrawlEntry.java
@@ -132,6 +132,7 @@ public CrawlEntry(
         assert url != null;
         assert initiator != null;
         assert profileHandle == null || profileHandle.length() == yacySeedDB.commonHashLength : profileHandle + " != " + yacySeedDB.commonHashLength;
+        url.removeRef(); // remove anchor reference
         this.initiator     = initiator;
         this.url           = url;
         this.refhash       = (referrerhash == null) ? "" : referrerhash;

diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java
@@ -508,7 +508,7 @@ public int size() {
 
     protected final class crawlWorker extends Thread {
 
-        public CrawlEntry entry;
+        private CrawlEntry entry;
         private final Integer code;
 
         public crawlWorker(final CrawlEntry entry) {

diff --git a/source/de/anomic/yacy/yacyURL.java b/source/de/anomic/yacy/yacyURL.java
@@ -526,6 +526,10 @@ public String getRef() {
         return ref;
     }
 
+    public void removeRef() {
+    	ref = null;
+    }
+
     public String getUserInfo() {
         return userInfo;
     }
@@ -680,7 +684,7 @@ private final String urlHashComputation() {
         // the url hash computation needs a DNS lookup to check if the addresses domain is local
         // that causes that this method may be very slow
 
-        assert this.hash == null; // should only be called if the hash was not computed bevore
+        assert this.hash == null; // should only be called if the hash was not computed before
 
         final int id = serverDomains.getDomainID(this.host); // id=7: tld is local
         final boolean isHTTP = this.protocol.equals("http");
@@ -894,6 +898,7 @@ public static void main(final String[] args) {
             environment = test[i][0];
             url = test[i][1];
             try {aURL = yacyURL.newURL(environment, url);} catch (final MalformedURLException e) {aURL = null;}
+            if (aURL != null) System.out.println("normalized: " + aURL.toNormalform(true, true));
             if (environment == null) {
                 try {jURL = new java.net.URL(url);} catch (final MalformedURLException e) {jURL = null;}
             } else {