extended fail-safe memory-managament. prevents too much allocation, t…

…oo often GC and should help for the 100%CPU-bug

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@303 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/Performance_p.html

@@ -21,10 +21,12 @@ <h2>Performance</h2>
 <td class="small">Total<br>Cycles</td>
 <td class="small">Idle<br>Cycles</td>
 <td class="small">Busy<br>Cycles</td>
+<td class="small">Short Mem<br>Cycles</td>
 <td class="small">Sleep Time<br>per Cycle<br>(milliseconds)</td>
 <td class="small">Exec Time<br>per Busy-Cycle<br>(milliseconds)</td>
 <td class="small">Delay between<br>idle loops</td>
 <td class="small">Delay between<br>busy loops</td>
+<td class="small">Minimum of<br>Required Memory</td>
 <td class="small">Full Description</td>
 </tr>
 #{table}#
@@ -40,10 +42,12 @@ <h2>Performance</h2>
 <td class="small" align="right">#[totalcycles]#</td>
 <td class="small" align="right">#[idlecycles]#</td>
 <td class="small" align="right">#[busycycles]#</td>
+<td class="small" align="right">#[memscycles]#</td>
 <td class="small" align="right">#[sleeppercycle]#</td>
 <td class="small" align="right">#[execpercycle]#</td>
 <td class="small" align="right"><input name="#[threadname]#_idlesleep" type="text" align="right" size="7" maxlength="7" value="#[idlesleep]#"> milliseconds</td>
 <td class="small" align="right"><input name="#[threadname]#_busysleep" type="text" align="right" size="7" maxlength="7" value="#[busysleep]#"> milliseconds</td>
+<td class="small" align="right"><input name="#[threadname]#_memprereq" type="text" align="right" size="8" maxlength="8" value="#[memprereq]#"> bytes</td>
 <td class="small" align="left">#[longdescr]#</td>
 </tr>
 #{/table}#

htroot/Performance_p.java

@@ -81,11 +81,11 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
 
         // set templates for latest news from the threads
         long blocktime, sleeptime, exectime;
-        long idlesleep, busysleep;
+        long idlesleep, busysleep, memprereq;
         int queuesize;
         threads = switchboard.threadNames();
         int c = 0;
-        long idleCycles, busyCycles;
+        long idleCycles, busyCycles, memshortageCycles;
         while (threads.hasNext()) {
             threadName = (String) threads.next();
             thread = switchboard.getThread(threadName);
@@ -102,37 +102,43 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
             exectime = thread.getExecTime();
             idleCycles = thread.getIdleCycles();
             busyCycles = thread.getBusyCycles();
+            memshortageCycles = thread.getOutOfMemoryCycles();
             prop.put("table_" + c + "_blocktime", blocktime / 1000);
             prop.put("table_" + c + "_blockpercent", "" + (100 * blocktime / blocktime_total));
             prop.put("table_" + c + "_sleeptime", sleeptime / 1000);
             prop.put("table_" + c + "_sleeppercent", "" + (100 * sleeptime / sleeptime_total));
             prop.put("table_" + c + "_exectime", exectime / 1000);
             prop.put("table_" + c + "_execpercent", "" + (100 * exectime / exectime_total));
-            prop.put("table_" + c + "_totalcycles", "" + (idleCycles + busyCycles));
+            prop.put("table_" + c + "_totalcycles", "" + (idleCycles + busyCycles + memshortageCycles));
             prop.put("table_" + c + "_idlecycles", "" + idleCycles);
             prop.put("table_" + c + "_busycycles", "" + busyCycles);
+            prop.put("table_" + c + "_memscycles", "" + memshortageCycles);
             prop.put("table_" + c + "_sleeppercycle", ((idleCycles + busyCycles) == 0) ? "-" : ("" + (sleeptime / (idleCycles + busyCycles))));
             prop.put("table_" + c + "_execpercycle", (busyCycles == 0) ? "-" : ("" + (exectime / busyCycles)));
 
             if ((post != null) && (post.containsKey("delaysubmit"))) {
                 // load with new values
-                idlesleep = Long.parseLong((String) post.get(threadName + "_idlesleep", "1"));
-                busysleep = Long.parseLong((String) post.get(threadName + "_busysleep", "1"));
-
+                idlesleep = Long.parseLong((String) post.get(threadName + "_idlesleep",  "100"));
+                busysleep = Long.parseLong((String) post.get(threadName + "_busysleep", "1000"));
+                memprereq = Long.parseLong((String) post.get(threadName + "_memprereq",    "0"));
+
 		// check values to prevent short-cut loops
 		if (idlesleep == 0) idlesleep = 1000;
 
                 // on-the-fly re-configuration
-                switchboard.setThreadSleep(threadName, idlesleep, busysleep);
+                switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
                 switchboard.setConfig(threadName + "_idlesleep", idlesleep);
                 switchboard.setConfig(threadName + "_busysleep", busysleep);
+                switchboard.setConfig(threadName + "_memprereq", memprereq);
             } else {
                 // load with old values
                 idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000"));
                 busysleep = Long.parseLong(switchboard.getConfig(threadName + "_busysleep", "1000"));
+                memprereq = Long.parseLong(switchboard.getConfig(threadName + "_memprereq", "1000"));
             }
             prop.put("table_" + c + "_idlesleep", idlesleep);
             prop.put("table_" + c + "_busysleep", busysleep);
+            prop.put("table_" + c + "_memprereq", memprereq);
 
             c++;
         }

source/de/anomic/kelondro/kelondroRecords.java

@@ -77,8 +77,9 @@
 public class kelondroRecords {
 
     // constants
-    private static int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
-
+    private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
+    public  static final long memBlock = 5000000; // do not fill cache further if the amount of available memory is less that this
+
     // static seek pointers
     private static long POS_MAGIC      = 0;                     // 1 byte, byte: file type magic
     private static long POS_BUSY       = POS_MAGIC      + 1;    // 1 byte, byte: marker for synchronization
@@ -352,10 +353,12 @@ private void checkCacheSpace() {
         // check for space in cache
         // should be only called within a synchronized(XcacheHeaders) environment
         if (XcacheSize == 0) return;
-        while (XcacheHeaders.size() >= XcacheSize) {
+        Handle delkey;
+        while ((XcacheHeaders.size() >= XcacheSize) ||
+               ((XcacheHeaders.size() > 0) && (Runtime.getRuntime().freeMemory() < memBlock))) {
             // delete one entry
             try {
-                Handle delkey = (Handle) XcacheScore.getMinObject(); // error (see below) here
+                delkey = (Handle) XcacheScore.getMinObject(); // error (see below) here
                 XcacheScore.deleteScore(delkey);
                 XcacheHeaders.remove(delkey);
             } catch (NoSuchElementException e) {
@@ -365,6 +368,7 @@ private void checkCacheSpace() {
                 this.XcacheScore = new kelondroMScoreCluster();
                 this.XcacheHeaders = new HashMap();
             }
+            delkey = null;
         }
     }
 
@@ -662,6 +666,7 @@ private void updateNode() {
                 synchronized (XcacheHeaders) {
                     // remember size to evaluate a cache size check need
                     int sizeBefore = XcacheHeaders.size();
+                    //long memBefore = Runtime.getRuntime().freeMemory();
                     // generate cache entry
                     byte[][] cacheValue;
                     if (values == null) {
@@ -676,10 +681,11 @@ private void updateNode() {
                     cacheNode.ohBytes = this.ohBytes;
                     cacheNode.ohHandle = this.ohHandle;
                     // store the cache entry
-                    XcacheHeaders.put(cacheNode.handle, cacheNode);
-                    XcacheScore.setScore(handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
+                    boolean newentry = XcacheHeaders.put(cacheNode.handle, cacheNode) == null;
+                    XcacheScore.setScore(cacheNode.handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
                     // delete the cache entry
                     cacheNode = null;
+                    //System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : ""));
                     // check cache size
                     if (XcacheHeaders.size() > sizeBefore) checkCacheSpace();
                     //System.out.println("kelondroRecords cache4" + filename + ": " + XcacheHeaders.size() + " entries, " + XcacheSize + " allowed.");

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -148,7 +148,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
 
 
     // load slots
-    public static int crawlSlots = 20;
+    public static int crawlSlots = 12;
 
     // couloured list management
     public static TreeSet blueList = null;
@@ -578,12 +578,6 @@ public boolean coreCrawlJob() {
             //log.logDebug("CoreCrawl: queue is empty");
             return false;
         }
-        if (Runtime.getRuntime().freeMemory() < 2000000) {
-            log.logDebug("CoreCrawl: not enough memory available, dismissed (" +
-                    "free=" + Runtime.getRuntime().freeMemory() + ")");
-            System.gc();
-            return false;
-        }
         if (queueStack.size() >= crawlSlots) {
             log.logDebug("CoreCrawl: too many processes in queue, dismissed (" +
                     "queueStack=" + queueStack.size() + ")");
@@ -639,12 +633,6 @@ public boolean limitCrawlTriggerJob() {
             //log.logDebug("LimitCrawl: queue is empty");
             return false;
         }
-        if (Runtime.getRuntime().freeMemory() < 2000000) {
-            log.logDebug("limitCrawlTrigger: not enough memory available, dismissed (" +
-                    "free=" + Runtime.getRuntime().freeMemory() + ")");
-            System.gc();
-            return false;
-        }
         // if the server is busy, we do crawling more slowly
         //if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
 
@@ -717,12 +705,6 @@ public boolean remoteTriggeredCrawlJob() {
             //log.logDebug("GlobalCrawl: queue is empty");
             return false;
         }
-        if (Runtime.getRuntime().freeMemory() < 2000000) {
-            log.logDebug("remoteTriggeredCrawl: not enough memory available, dismissed (" +
-                    "free=" + Runtime.getRuntime().freeMemory() + ")");
-            System.gc();
-            return false;
-        }
         /*
         if (queueStack.size() > 0) {
             log.logDebug("GlobalCrawl: any processe is in queue, dismissed (" +

source/de/anomic/server/serverAbstractSwitch.java

@@ -254,31 +254,52 @@ public void undeployAction(String actionName) {
 	log.logInfo("Undeployed Action '" + action.getShortDescription() + "', (" + switchActions.size() + " actions registered)");
     }
 
-    public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, serverThread newThread, long startupDelay) {
+    public void deployThread(
+            String threadName,
+            String threadShortDescription,
+            String threadLongDescription,
+            serverThread newThread,
+            long startupDelay) {
         deployThread(threadName, threadShortDescription, threadLongDescription,
                      newThread, startupDelay,
-                     Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue")), 
-                     Long.parseLong(getConfig(threadName + "_busysleep" , "novalue")));
+                     Long.parseLong(getConfig(threadName + "_idlesleep" ,     "100")), 
+                     Long.parseLong(getConfig(threadName + "_busysleep" ,    "1000")),
+                     Long.parseLong(getConfig(threadName + "_memprereq" , "1000000")));
     }
 
-    public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, serverThread newThread, long startupDelay, long initialIdleSleep, long initialBusySleep) {
+    public void deployThread(
+            String threadName,
+            String threadShortDescription,
+            String threadLongDescription,
+            serverThread newThread,
+            long startupDelay,
+            long initialIdleSleep,
+            long initialBusySleep,
+            long initialMemoryPreRequisite) {
         if (newThread.isAlive()) throw new RuntimeException("undeployed threads must not live; they are started as part of the deployment");
         newThread.setStartupSleep(startupDelay);
-        long sleep;
+        long x;
         try {
-            sleep = Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue"));
-            newThread.setIdleSleep(sleep);
+            x = Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue"));
+            newThread.setIdleSleep(x);
         } catch (NumberFormatException e) {
             newThread.setIdleSleep(initialIdleSleep);
             setConfig(threadName + "_idlesleep", initialIdleSleep);
         }
         try {
-            sleep = Long.parseLong(getConfig(threadName + "_busysleep" , "novalue"));
-            newThread.setBusySleep(sleep);
+            x = Long.parseLong(getConfig(threadName + "_busysleep" , "novalue"));
+            newThread.setBusySleep(x);
         } catch (NumberFormatException e) {
             newThread.setBusySleep(initialBusySleep);
             setConfig(threadName + "_busysleep", initialBusySleep);
         }
+        try {
+            x = Long.parseLong(getConfig(threadName + "_memprereq" , "novalue"));
+            newThread.setMemPreReqisite(x);
+        } catch (NumberFormatException e) {
+            newThread.setMemPreReqisite(initialMemoryPreRequisite);
+            setConfig(threadName + "_memprereq", initialMemoryPreRequisite);
+        }
         newThread.setLog(log);
         newThread.setDescription(threadShortDescription, threadLongDescription);
         workerThreads.put(threadName, newThread);
@@ -290,11 +311,12 @@ public serverThread getThread(String threadName) {
         return (serverThread) workerThreads.get(threadName);
     }
 
-    public void setThreadSleep(String threadName, long idleMillis, long busyMillis) {
+    public void setThreadPerformance(String threadName, long idleMillis, long busyMillis, long memprereqBytes) {
         serverThread thread = (serverThread) workerThreads.get(threadName);
         if (thread != null) {
             thread.setIdleSleep(idleMillis);
             thread.setBusySleep(busyMillis);
+            thread.setMemPreReqisite(memprereqBytes);
         }
     }
 

source/de/anomic/server/serverAbstractThread.java

@@ -57,10 +57,10 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
     private long startup = 0, idlePause = 0, busyPause = 0, blockPause = 0;
     private boolean running = true;
     private serverLog log = null;
-    private long idletime = 0, busytime = 0;
+    private long idletime = 0, busytime = 0, memprereq = 0;
     private String shortDescr = "", longDescr = "";
     private long threadBlockTimestamp = System.currentTimeMillis();
-    private long idleCycles = 0, busyCycles = 0;
+    private long idleCycles = 0, busyCycles = 0, outofmemoryCycles = 0;
 
     protected final void announceThreadBlockApply() {
         // shall only be used, if a thread blocks for an important reason
@@ -107,6 +107,11 @@ public final void setBusySleep(long milliseconds) {
         busyPause = milliseconds;
     }
 
+    public void setMemPreReqisite(long freeBytes) {
+        // sets minimum required amount of memory for the job execution
+        memprereq = freeBytes;
+    }
+
     public final String getShortDescription() {
         return this.shortDescr;
     }
@@ -125,6 +130,12 @@ public final long getBusyCycles() {
         return this.busyCycles;
     }
 
+    public long getOutOfMemoryCycles() {
+        // returns the total number of cycles where
+        // a job execution was omitted because of memory shortage
+        return this.outofmemoryCycles;
+    }
+
     public final long getBlockTime() {
         // returns the total time that this thread has been blocked so far
         return this.blockPause;
@@ -206,8 +217,10 @@ public void run() {
         long innerpause;
         long timestamp;
         boolean isBusy;
+        Runtime rt = Runtime.getRuntime();
+
         while (running) {
-            try {
+            if (rt.freeMemory() > memprereq) try {
                 // do job
                 timestamp = System.currentTimeMillis();
                 isBusy = this.job();
@@ -224,6 +237,14 @@ public void run() {
                 // if the exception is too bad it should call terminate()
                 this.jobExceptionHandler(e);
                 busyCycles++;
+            } else {
+                // omit job, not enough memory
+                // process scheduled pause
+                timestamp = System.currentTimeMillis();
+                ratz(this.idlePause);
+                idletime += System.currentTimeMillis() - timestamp;
+                outofmemoryCycles++;
+                if (rt.freeMemory() <= memprereq) System.gc(); // give next loop a chance
             }
         }
         this.close();

source/de/anomic/server/serverInstantThread.java

@@ -117,6 +117,7 @@ public static serverThread oneTimeJob(Object env, String jobExec, serverLog log,
         thread.setStartupSleep(startupDelay);
         thread.setIdleSleep(-1);
         thread.setBusySleep(-1);
+        thread.setMemPreReqisite(0);
         thread.setLog(log);
         thread.start();
         return thread;

source/de/anomic/server/serverSwitch.java

@@ -73,9 +73,11 @@ public void deployThread(String threadName,
 			     String threadShortDescription,
 			     String threadLongDescription,
                              serverThread newThread,
-                             long startupDelay, long initialIdleSleep, long initialBusySleep);
+                             long startupDelay,
+                             long initialIdleSleep, long initialBusySleep,
+                             long initialMemoryPreRequisite);
     public serverThread getThread(String threadName);
-    public void setThreadSleep(String threadName, long idleMillis, long busyMillis);
+    public void setThreadPerformance(String threadName, long idleMillis, long busyMillis, long memprereq);
     public void terminateThread(String threadName, boolean waitFor);
     public void terminateAllThreads(boolean waitFor);
     public Iterator /*of serverThread-Names (String)*/ threadNames();

source/de/anomic/server/serverThread.java

@@ -65,6 +65,9 @@ public interface serverThread {
     public void setBusySleep(long milliseconds);
     // sets a sleep time for pauses between two jobs if the job returns true (busy)
 
+    public void setMemPreReqisite(long freeBytes);
+    // sets minimum required amount of memory for the job execution
+
     public String getShortDescription();
     // returns short description string for online display
 
@@ -77,6 +80,10 @@ public interface serverThread {
     public long getBusyCycles();
     // returns the total number of cycles of job execution with busy-result
 
+    public long getOutOfMemoryCycles();
+    // returns the total number of cycles where
+    // a job execution was omitted because of memory shortage
+
     public long getBlockTime();
     // returns the total time that this thread has been blocked so far