some enhancements to the remote crawl trigger

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2030 6c8d7289-2bf4-0310-a012-ef5d649a1542
yacy · Apr 20, 2006 · 60e5aff · 60e5aff
1 parent dbe96e6
commit 60e5aff
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 19 deletions.
diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java
@@ -1161,19 +1161,6 @@ public boolean limitCrawlTriggerJob() {
             log.logInfo("shifted " + toshift + " jobs from global crawl to local crawl");
         }
 
-
-        if (sbQueue.size() >= indexingSlots) {
-            log.logFine("LimitCrawl: too many processes in indexing queue, dismissed to protect emergency case (" +
-            "sbQueueSize=" + sbQueue.size() + ")");
-            return false;
-        }
-        if (cacheLoader.size() >= crawlSlots) {
-            log.logFine("LimitCrawl: too many processes in loader queue, dismissed to protect emergency case (" +
-            "cacheLoader=" + cacheLoader.size() + ")");
-            return false;
-        }
-
-
         // if the server is busy, we do crawling more slowly
         //if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
 
@@ -1214,7 +1201,20 @@ public boolean limitCrawlTriggerJob() {
                 if (success) return true;
             }
 
-            processLocalCrawling(urlEntry, profile, stats);
+            processLocalCrawling(urlEntry, profile, stats); // emergency case
+
+            if (sbQueue.size() >= indexingSlots) {
+                log.logFine("LimitCrawl: too many processes in indexing queue, delayed to protect emergency case (" +
+                "sbQueueSize=" + sbQueue.size() + ")");
+                return false;
+            }
+
+            if (cacheLoader.size() >= crawlSlots) {
+                log.logFine("LimitCrawl: too many processes in loader queue, delayed to protect emergency case (" +
+                "cacheLoader=" + cacheLoader.size() + ")");
+                return false;
+            }
+
             return true;
         } catch (IOException e) {
             log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage());
@@ -1706,7 +1706,7 @@ private boolean processRemoteCrawlTrigger(plasmaCrawlNURL.Entry urlEntry) {
 
         // do the request
         try {
-            HashMap page = yacyClient.crawlOrder(remoteSeed, urlEntry.url(), urlPool.getURL(urlEntry.referrerHash()));
+            HashMap page = yacyClient.crawlOrder(remoteSeed, urlEntry.url(), urlPool.getURL(urlEntry.referrerHash()), 6000);
 
             // check success
             /*

diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java
@@ -733,11 +733,11 @@ public static String transfer(String targetAddress, String filename, byte[] file
         return "wrong protocol: " + protocol;
     }
 
-    public static HashMap crawlOrder(yacySeed targetSeed, URL url, URL referrer) {
-        return crawlOrder(targetSeed,new URL[]{url},new URL[]{referrer}); 
+    public static HashMap crawlOrder(yacySeed targetSeed, URL url, URL referrer, int timeout) {
+        return crawlOrder(targetSeed, new URL[]{url}, new URL[]{referrer}, timeout); 
     }
 
-    public static HashMap crawlOrder(yacySeed targetSeed, URL[] url, URL[] referrer) {
+    public static HashMap crawlOrder(yacySeed targetSeed, URL[] url, URL[] referrer, int timeout) {
         // this post a message to the remote message board
         if (targetSeed == null) { return null; }
         if (yacyCore.seedDB.mySeed == null) { return null; }
@@ -777,7 +777,7 @@ public static HashMap crawlOrder(yacySeed targetSeed, URL[] url, URL[] referrer)
             return nxTools.table(
                     httpc.wput(
                             new URL("http://" + address + "/yacy/crawlOrder.html"),
-                            60000, 
+                            timeout, 
                             null, 
                             null, 
                             (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,