Skip to content

Commit

Permalink
because of a bug in the JRE 1.4.2 there was no memory protection
Browse files Browse the repository at this point in the history 
see http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4686462
this commit fixes the bug by using a memory-computation patch.
All uses of Runtime.maxMemory had been replaced by serverMemory.max
The bug is not present any more in Java 1.5

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2419 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Aug 18, 2006
1 parent 4eca0f8 commit 0187c60
Show file tree
Hide file tree
Showing 10 changed files with 67 additions and 31 deletions.
3 changes: 2 additions & 1 deletion htroot/PerformanceMemory_p.java
View file
Expand Up @@ -50,6 +50,7 @@
import de.anomic.http.httpc;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.serverFileUtils;
Expand Down Expand Up @@ -138,7 +139,7 @@ public static serverObjects respond(httpHeader header, serverObjects post, serve
long memoryTotalAfterInitBGC = Long.parseLong(env.getConfig("memoryTotalAfterInitBGC", "0"));
long memoryTotalAfterInitAGC = Long.parseLong(env.getConfig("memoryTotalAfterInitAGC", "0"));
long memoryTotalAfterStartup = Long.parseLong(env.getConfig("memoryTotalAfterStartup", "0"));
long memoryMax = Runtime.getRuntime().maxMemory();
long memoryMax = serverMemory.max;

prop.put("memoryMax", memoryMax / MB);
prop.put("memoryAvailAfterStartup", (memoryMax - memoryTotalAfterStartup + memoryFreeAfterStartup) / MB);
Expand Down
3 changes: 2 additions & 1 deletion htroot/Status.java
View file
Expand Up @@ -56,6 +56,7 @@
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
Expand Down Expand Up @@ -270,7 +271,7 @@ else if (jobType.equals("globalCrawlTrigger"))
// memory usage and system attributes
prop.put("freeMemory", bytesToString(rt.freeMemory()));
prop.put("totalMemory", bytesToString(rt.totalMemory()));
prop.put("maxMemory", bytesToString(rt.maxMemory()));
prop.put("maxMemory", bytesToString(serverMemory.max));
prop.put("processors", rt.availableProcessors());

// proxy traffic
Expand Down
4 changes: 2 additions & 2 deletions source/de/anomic/kelondro/kelondroAttrSeq.java
View file
Expand Up @@ -64,6 +64,7 @@


import de.anomic.server.serverFileUtils;
import de.anomic.server.serverMemory;

public class kelondroAttrSeq {

Expand Down Expand Up @@ -448,12 +449,11 @@ public String toString() {
}
}

private static final Runtime runtime = Runtime.getRuntime();
private static final long cc = 0;
private static boolean shortmemstate = false;
private static boolean shortmem() {
if ((cc % 300) == 0) {
shortmemstate = (runtime.freeMemory() < 20000000L);
shortmemstate = (serverMemory.available() < 20000000L);
}
return shortmemstate;
}
Expand Down
6 changes: 4 additions & 2 deletions source/de/anomic/kelondro/kelondroObjectCache.java
View file
Expand Up @@ -58,6 +58,8 @@

import java.util.TreeMap;

import de.anomic.server.serverMemory;

public class kelondroObjectCache {

private final TreeMap cache;
Expand Down Expand Up @@ -288,7 +290,7 @@ public void flushc() {
((k = (String) ages.getMinObject()) != null) &&
((ages.size() > maxSize) ||
(((System.currentTimeMillis() - longEmit(ages.getScore(k))) > maxAge) &&
(Runtime.getRuntime().freeMemory() < minMem)))
(serverMemory.available() < minMem)))
) {
cache.remove(k);
ages.deleteScore(k);
Expand All @@ -304,7 +306,7 @@ public void flushh() {
((k = (String) hasnot.getMinObject()) != null) &&
((hasnot.size() > maxSize) ||
(((System.currentTimeMillis() - longEmit(hasnot.getScore(k))) > maxAge) &&
(Runtime.getRuntime().freeMemory() < minMem)))
(serverMemory.available() < minMem)))
) {
hasnot.deleteScore(k);
hasnotFlush++;
Expand Down
12 changes: 3 additions & 9 deletions source/de/anomic/kelondro/kelondroRecords.java
View file
Expand Up @@ -78,6 +78,8 @@
import java.util.TreeSet;
import java.util.logging.Logger;

import de.anomic.server.serverMemory;

public class kelondroRecords {

// constants
Expand Down Expand Up @@ -449,14 +451,6 @@ private void initCache(long buffersize, long preloadTime) {
}
}

private static final long max = Runtime.getRuntime().maxMemory();
private static final Runtime runtime = Runtime.getRuntime();

public static long availableMemory() {
// memory that is available including increasing total memory up to maximum
return max - runtime.totalMemory() + runtime.freeMemory();
}

public File file() {
if (filename == null) return null;
return new File(filename);
Expand Down Expand Up @@ -837,7 +831,7 @@ private boolean cacheSpace() {
// returns false if the cache is considered to be full
if (cacheSize == 0) return false; // no caching
if (cacheHeaders.size() == 0) return true; // nothing there to flush
if ((cacheHeaders.size() < cacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space
if ((cacheHeaders.size() < cacheSize) && (serverMemory.available() >= memBlock)) return true; // no need to flush cache space

// just delete any of the entries
cacheHeaders.removeOne();
Expand Down
4 changes: 3 additions & 1 deletion source/de/anomic/kelondro/kelondroRowBufferedSet.java
View file
Expand Up @@ -28,6 +28,8 @@
import java.util.Iterator;
import java.util.TreeMap;

import de.anomic.server.serverMemory;

public class kelondroRowBufferedSet extends kelondroRowSet {

private static final long memBlockLimit = 2000000; // do not fill cache further if the amount of available memory is less that this
Expand Down Expand Up @@ -136,7 +138,7 @@ public kelondroRow.Entry put(kelondroRow.Entry newentry) {
if (oldentry == null) {
// this was not anywhere
buffer.put(key, newentry);
if (((buffer.size() > bufferFlushMinimum) && (kelondroRecords.availableMemory() > memBlockLimit)) ||
if (((buffer.size() > bufferFlushMinimum) && (serverMemory.available() > memBlockLimit)) ||
(buffer.size() > bufferFlushLimit)) flush();
} else {
// replace old entry
Expand Down
3 changes: 2 additions & 1 deletion source/de/anomic/plasma/plasmaRankingCRProcess.java
View file
Expand Up @@ -59,6 +59,7 @@
import de.anomic.kelondro.kelondroRowSet;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverDate;
import de.anomic.server.serverMemory;
import de.anomic.tools.bitfield;

public class plasmaRankingCRProcess {
Expand Down Expand Up @@ -376,7 +377,7 @@ public static int genrci(File cr_in, File rci_out) throws IOException {
count++;
if ((count % 1000) == 0) {
l = java.lang.Math.max(1, (System.currentTimeMillis() - start) / 1000);
System.out.println("processed " + count + " citations, " + (count / l) + " per second, rci.size = " + rci.size() + ", " + ((size - count) / (count / l)) + " seconds remaining; mem = " + Runtime.getRuntime().freeMemory());
System.out.println("processed " + count + " citations, " + (count / l) + " per second, rci.size = " + rci.size() + ", " + ((size - count) / (count / l)) + " seconds remaining; mem = " + serverMemory.available());
}
i.remove();
}
Expand Down
5 changes: 2 additions & 3 deletions source/de/anomic/server/serverAbstractThread.java
View file
Expand Up @@ -245,7 +245,7 @@ public void run() {
long timestamp;
long memstamp0, memstamp1;
boolean isBusy;
Runtime rt = Runtime.getRuntime();
//Runtime rt = Runtime.getRuntime();

while (running) {
if ((this.intermissionObedient) && (this.intermission > 0) && (this.intermission != Long.MAX_VALUE)) {
Expand All @@ -266,7 +266,7 @@ public void run() {
timestamp = System.currentTimeMillis();
ratz(this.idlePause);
idletime += System.currentTimeMillis() - timestamp;
} else if (rt.freeMemory() > memprereq) try {
} else if (serverMemory.available() > memprereq) try {
// do job
timestamp = System.currentTimeMillis();
memstamp0 = serverMemory.used();
Expand Down Expand Up @@ -305,7 +305,6 @@ public void run() {
ratz(this.idlePause);
idletime += System.currentTimeMillis() - timestamp;
outofmemoryCycles++;
if (rt.freeMemory() <= memprereq) System.gc(); // give next loop a chance
}
}
this.close();
Expand Down
44 changes: 42 additions & 2 deletions source/de/anomic/server/serverMemory.java
View file
Expand Up @@ -49,7 +49,8 @@

public class serverMemory {

public static final long max = Runtime.getRuntime().maxMemory();
public static boolean vm15 = System.getProperty("java.vm.version").startsWith("1.5");
public static final long max = (vm15) ? Runtime.getRuntime().maxMemory() : computedMaxMemory(); // patch for maxMemory bug in Java 1.4.2
private static final Runtime runtime = Runtime.getRuntime();

public static long free() {
Expand Down Expand Up @@ -90,5 +91,44 @@ public static String bytesToString(long byteCount) {
} catch (Exception e) {
return "unknown";
}
}
}

private static int computedMaxMemory() {
// there is a bug in java 1.4.2 for maxMemory()
// see for bug description:
// http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4686462
// to get the correct maxMemory, we force a OutOfMemoryError here to measure the 'real' maxMemory()
int mb = 1024 * 1024;
byte[][] x = new byte[2048][];
for (int i = 0; i < x.length; i++) {
try {
x[i] = new byte[mb];
} catch (OutOfMemoryError e) {
x = null; // free memory
//System.out.println("* computed maxMemory = " + i + " mb");
return (int) Math.max(i * mb, Runtime.getRuntime().totalMemory());
}
}
return 2048 * mb;
}

public static void main(String[] args) {
// try this with a jvm 1.4.2 and with a jvm 1.5 and compare results
int mb = 1024 * 1024;
System.out.println("vm: " + System.getProperty("java.vm.version"));
System.out.println("computed max = " + (computedMaxMemory() / mb) + " mb");
int alloc = 10000;
Runtime rt = Runtime.getRuntime();
byte[][] x = new byte[100000][];
for (int i = 0; i < 100000; i++) {
x[i] = new byte[alloc];
if (i % 100 == 0) System.out.println("used = " + (i * alloc / mb) +
", total = " + (rt.totalMemory() / mb) +
", free = " + (rt.freeMemory() / mb) +
", max = " + (rt.maxMemory() / mb) +
", avail = " + ((rt.maxMemory() - rt.totalMemory() + rt.freeMemory()) / mb));
}

}

}
14 changes: 5 additions & 9 deletions source/yacy.java
View file
Expand Up @@ -90,6 +90,7 @@
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverMemory;
import de.anomic.server.serverPlainSwitch;
import de.anomic.server.serverSwitch;
import de.anomic.server.serverSystem;
Expand Down Expand Up @@ -705,7 +706,7 @@ public static void minimizeUrlDB(String homePath, int dbcache) {
plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(new File(dbroot, "urlHash.temp.db"), cache, 10000);

Runtime rt = Runtime.getRuntime();
int cacheMem = (int)((rt.maxMemory()-rt.totalMemory())/1024)-(2*cache + 8*1024);
int cacheMem = (int)((serverMemory.max-rt.totalMemory())/1024)-(2*cache + 8*1024);
if (cacheMem < 2048) throw new OutOfMemoryError("Not enough memory available to start clean up.");

plasmaWordIndex wordIndex = new plasmaWordIndex(dbroot, indexRoot, cacheMem, 10000, log, sps.getConfigBool("useCollectionIndex", false));
Expand Down Expand Up @@ -1256,15 +1257,10 @@ public static void main(String args[]) {

// check memory amount
System.gc();
long startupMemFree = Runtime.getRuntime().freeMemory(); // the
// amount of
// free
// memory in
// the Java
// Virtual
// Machine
long startupMemFree = Runtime.getRuntime().freeMemory(); // the amount of free memory in the Java Virtual Machine
long startupMemTotal = Runtime.getRuntime().totalMemory(); // the total amount of memory in the Java virtual machine; may vary over time

serverMemory.available(); // force initialization of class serverMemory

// go into headless awt mode
System.setProperty("java.awt.headless", "true");
//which XML Parser?
Expand Down

0 comments on commit 0187c60

Please sign in to comment.