- refactoring of CrawlStacker (to prepare it for new multi-Threading …

…to remove DNS lookup bottleneck)

- fix of shallBeOwnWord target computation heuristic


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5392 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/ConfigAccounts_p.java

@@ -62,7 +62,7 @@ public static serverObjects respond(final httpRequestHeader header, final server
             }
 
             if (localhostAccess) {
-                if (sb.acceptLocalURLs) {
+                if (sb.crawlStacker.acceptLocalURLs()) {
                     // in this case it is not allowed to use a localhostAccess option
                     prop.put("commitIntranetWarning", 1);
                     localhostAccess = false;

htroot/IndexTransfer_p.java

@@ -105,14 +105,12 @@ public static serverObjects respond(final httpRequestHeader header, final server
         } else {
             if (!prop.containsKey("running_status")) prop.put("running_status","Not running");
         }
-
-
-
+
         //List known hosts
         yacySeed seed;
         int hc = 0;
         if ((sb.webIndex.seedDB != null) && (sb.webIndex.seedDB.sizeConnected() > 0)) {
-            final Iterator<yacySeed> e = yacyPeerSelection.getAcceptRemoteIndexSeeds(sb.webIndex.seedDB, "AAAAAAAAAAAA", sb.webIndex.seedDB.sizeConnected());
+            final Iterator<yacySeed> e = yacyPeerSelection.getAcceptRemoteIndexSeeds(sb.webIndex.seedDB, null, sb.webIndex.seedDB.sizeConnected(), false);
             final TreeMap<String, String> hostList = new TreeMap<String, String>();
             while (e.hasNext()) {
                 seed = e.next();

htroot/rct_p.java

@@ -72,7 +72,7 @@ public static serverObjects respond(final httpRequestHeader header, final server
                             loaddate = new Date();
                         }
                         final yacyURL referrer = null; // referrer needed!
-                        final String urlRejectReason = sb.acceptURL(url);
+                        final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(url);
                         if (urlRejectReason == null) {
                             // stack url
                             if (sb.getLog().isFinest()) sb.getLog().logFinest("crawlOrder: stack: url='" + url + "'");

htroot/yacy/crawlReceipt.java

@@ -127,7 +127,7 @@ public static serverObjects respond(final httpRequestHeader header, final server
         }
 
         // check if the entry is in our network domain
-        final String urlRejectReason = sb.acceptURL(comp.url());
+        final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(comp.url());
         if (urlRejectReason != null) {
             log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (" + urlRejectReason + ") for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
             prop.put("delay", "9999");

htroot/yacy/transferURL.java

@@ -128,7 +128,7 @@ public static serverObjects respond(final httpRequestHeader header, final server
                 }
 
                 // check if the entry is in our network domain
-                final String urlRejectReason = sb.acceptURL(comp.url());
+                final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(comp.url());
                 if (urlRejectReason != null) {
                     if (yacyCore.log.isFine()) yacyCore.log.logFine("transferURL: blocked URL '" + comp.url() + "' (" + urlRejectReason + ") from peer " + otherPeerName);
                     lEntry = null;

source/de/anomic/crawler/CrawlQueues.java

@@ -393,7 +393,7 @@ public boolean remoteCrawlLoaderJob() {
             } catch (final ParseException e) {
                 loaddate = new Date();
             }
-            final String urlRejectReason = sb.acceptURL(url);
+            final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(url);
             if (urlRejectReason == null) {
                 // stack url
                 if (sb.getLog().isFinest()) sb.getLog().logFinest("crawlOrder: stack: url='" + url + "'");