Permalink
Browse files

Prevent integer overflow in table statistics and use strong typing

  • Loading branch information...
luccioman committed Jun 19, 2017
1 parent 0f80c97 commit 0487336ec30a5bda5f5fbf67b322538e6c6fa3df
Showing with 91 additions and 28 deletions.
  1. +11 −15 htroot/PerformanceMemory_p.java
  2. +80 −13 source/net/yacy/kelondro/table/Table.java
@@ -38,6 +38,7 @@
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.kelondro.index.RAMIndex;
import net.yacy.kelondro.table.Table;
import net.yacy.kelondro.table.Table.TableStatistics;
import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.Switchboard;
@@ -124,27 +125,23 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
// write table for Table index sizes
Iterator<String> i = Table.filenames();
String filename;
Map<Table.StatKeys, String> mapx;
TableStatistics stats;
int p;
c = 0;
long mem, totalmem = 0;
long totalmem = 0;
while (i.hasNext()) {
filename = i.next();
mapx = Table.memoryStats(filename);
stats = Table.memoryStats(filename);
totalmem += stats.getTotalMem();
prop.put("EcoList_" + c + "_tableIndexPath", ((p = filename.indexOf("DATA",0)) < 0) ? filename : filename.substring(p));
prop.putNum("EcoList_" + c + "_tableSize", mapx.get(Table.StatKeys.tableSize));
prop.putNum("EcoList_" + c + "_tableSize", stats.getTableSize());
String v = mapx.get(Table.StatKeys.tableKeyMem);
mem = v == null ? 0 : Long.parseLong(v);
totalmem += mem;
prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem));
prop.put("EcoList_" + c + "_tableKeyChunkSize", mapx.get(Table.StatKeys.tableKeyChunkSize));
prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(stats.getKeyMem()));
prop.put("EcoList_" + c + "_tableKeyChunkSize", Formatter.bytesToString(stats.getKeyChunkSize()));
v = mapx.get(Table.StatKeys.tableValueMem);
mem = v == null ? 0 : Long.parseLong(v);
totalmem += mem;
prop.put("EcoList_" + c + "_tableValueMem", Formatter.bytesToString(mem));
prop.put("EcoList_" + c + "_tableValueChunkSize", mapx.get(Table.StatKeys.tableValueChunkSize));
prop.put("EcoList_" + c + "_tableValueMem", Formatter.bytesToString(stats.getValueMem()));
prop.put("EcoList_" + c + "_tableValueChunkSize", Formatter.bytesToString(stats.getValueChunkSize()));
c++;
}
@@ -154,7 +151,6 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
// write object cache table
final Iterator<Map.Entry<String, RAMIndex>> oi = RAMIndex.objects();
c = 0;
mem = 0;
Map.Entry<String, RAMIndex> oie;
RAMIndex cache;
long hitmem, totalhitmem = 0;
@@ -31,7 +31,6 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.ConcurrentModificationException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -72,6 +71,8 @@
// static tracker objects
private final static ConcurrentLog log = new ConcurrentLog("TABLE");
/** Map all active table instances by file name */
private final static TreeMap<String, Table> tableTracker = new TreeMap<String, Table>();
private final static long maxarraylength = 134217727L; // (2^27-1) that may be the maximum size of array length in some JVMs
@@ -363,31 +364,97 @@ public static long tableSize(final File tablefile, final int recordsize, final b
return tableTracker.keySet().iterator();
}
public static final Map<StatKeys, String> memoryStats(final String filename) {
public static final TableStatistics memoryStats(final String filename) {
// returns a map for each file in the tracker;
// the map represents properties for each record objects,
// i.e. for cache memory allocation
final Table theTABLE = tableTracker.get(filename);
return theTABLE.memoryStats();
}
public enum StatKeys {
tableSize, tableKeyChunkSize, tableKeyMem, tableValueChunkSize, tableValueMem
/**
* Table memory usage statistics
*/
public class TableStatistics {
/** The table entries number */
private int tableSize = 0;
/** Size of a key chunk in bytes */
private int keyChunkSize = 0;
/** Total size of keys in bytes */
private long keyMem = 0;
/** Size of a value chunk in bytes */
private int valueChunkSize = 0;
/** Total size of values in bytes */
private long valueMem = 0;
/**
* @return the size of a key chunk in bytes
*/
public int getKeyChunkSize() {
return this.keyChunkSize;
}
/**
* @return the total size of the table in-memory keys in bytes
*/
public long getKeyMem() {
return this.keyMem;
}
/**
* @return the table entries number
*/
public int getTableSize() {
return this.tableSize;
}
/**
* @return the size of a value chunk in bytes
*/
public int getValueChunkSize() {
return this.valueChunkSize;
}
/**
* @return the total size of the table in-memory values in bytes
*/
public long getValueMem() {
return this.valueMem;
}
/**
* @return the total memory used by the table in bytes
*/
public long getTotalMem() {
return this.keyMem + this.valueMem;
}
}
private final Map<StatKeys, String> memoryStats() {
/**
* @return computed table current memory usage statistics
*/
private final TableStatistics memoryStats() {
// returns statistical data about this object
synchronized (this) {
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
}
final HashMap<StatKeys, String> map = new HashMap<StatKeys, String>(8);
if (this.index == null) return map; // possibly closed or beeing closed
map.put(StatKeys.tableSize, Integer.toString(this.index.size()));
map.put(StatKeys.tableKeyChunkSize, (this.index instanceof RowHandleMap) ? Integer.toString(((RowHandleMap) this.index).row().objectsize) : "-1");
map.put(StatKeys.tableKeyMem, (this.index instanceof RowHandleMap) ? Integer.toString(((RowHandleMap) this.index).row().objectsize * this.index.size()) : "-1");
map.put(StatKeys.tableValueChunkSize, (this.table == null) ? "0" : Integer.toString(this.table.row().objectsize));
map.put(StatKeys.tableValueMem, (this.table == null) ? "0" : Integer.toString(this.table.row().objectsize * this.table.size()));
return map;
TableStatistics stats = new TableStatistics();
if (this.index == null) return stats; // possibly closed or being closed
stats.tableSize = this.index.size();
if(this.index instanceof RowHandleMap) {
stats.keyChunkSize = (((RowHandleMap) this.index).row().objectsize);
stats.keyMem = (long)((RowHandleMap) this.index).row().objectsize * (long)this.index.size();
}
if(table != null) {
stats.valueChunkSize = this.table.row().objectsize;
stats.valueMem = (long)this.table.row().objectsize * (long)this.table.size();
}
return stats;
}
public boolean usesFullCopy() {

0 comments on commit 0487336

Please sign in to comment.