Skip to content

Commit

Permalink
- refactoring of IntegerHandleIndex and LongHandleIndex: both classes…
Browse files Browse the repository at this point in the history 
… had been merged into the new HandleMap class, which handles (key<byte[]>,n-byte-long) pairs with arbitraty key and value length. This will be useful to get a memory-enhanced/minimized database table indexing.

- added a analysis method that counts bytes that could be saved in case the new HandleMap can be applied in the most efficient way. Look for the log messages beginning with "HeapReader saturation": in most cases we could save about 30% RAM!
- removed the old FlexTable database structure. It was not used any more.
- removed memory statistics in PerformanceMemory about flex tables and node caches (node caches were used by Tree Tables, which are also not used any more)
- add a stub for a steering of navigation functions. That should help to switch off naviagtion computation in cases where it is not demanded by a client

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6034 6c8d7289-2bf4-0310-a012-ef5d649a1542
  • Loading branch information
orbiter committed Jun 7, 2009
1 parent bead000 commit c079b18
Show file tree
Hide file tree
Showing 25 changed files with 219 additions and 1,138 deletions.
54 changes: 0 additions & 54 deletions htroot/PerformanceMemory_p.html
View file
Expand Up @@ -99,60 +99,6 @@ <h2>Performance Settings for Memory</h2>
</tr>
</table>

<p><strong>FlexTable RAM Index:</strong></p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<td>Table</td>
<td>Count</td>
<td>Chunk Size</td>
<td>Used Memory</td>
</tr>
#{TableList}#
<tr class="TableCellLight">
<td align="left" class="TableCellDark">#[tableIndexPath]#</td>
<td align="right">#[tableIndexCount]#</td>
<td align="right">#[tableIndexChunkSize]#</td>
<td align="right">#[tableIndexMem]#</td>
</tr>
#{/TableList}#
<tr class="TableCellDark">
<td colspan="6">Total Mem = #[TableIndexTotalMem]# MB</td>
</tr>
</table>

<p><strong>Node Caches:</strong></p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<td>Table</td>
<td>Size</td>
<td>Chunk Size</td>
<td>Used Memory</td>
<td>Read Hit</td>
<td>Read Miss</td>
<td>Write Unique</td>
<td>Write Double</td>
<td>Deletes</td>
<td>Flushes</td>
</tr>
#{NodeList}#
<tr class="TableCellLight">
<td align="left" class="TableCellDark">#[nodeCachePath]#</td>
<td align="right">#[nodeCacheCount]#</td>
<td align="right">#[nodeChunkSize]#</td>
<td align="right">#[nodeCacheMem]#</td>
<td align="right">#[nodeCacheReadHit]#</td>
<td align="right">#[nodeCacheReadMiss]#</td>
<td align="right">#[nodeCacheWriteUnique]#</td>
<td align="right">#[nodeCacheWriteDouble]#</td>
<td align="right">#[nodeCacheDeletes]#</td>
<td align="right">#[nodeCacheFlushes]#</td>
</tr>
#{/NodeList}#
<tr class="TableCellDark">
<td colspan="10">Total Mem = #[nodeCacheTotalMem]# MB; Stop Grow when less than #[nodeCacheStopGrow]# MB available left; Start Shrink when less than #[nodeCacheStartShrink]# MB availabe left</td>
</tr>
</table>

<p><strong>Object Read Caches:</strong></p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
Expand Down
50 changes: 2 additions & 48 deletions htroot/PerformanceMemory_p.java
View file
Expand Up @@ -31,9 +31,7 @@

import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.blob.Cache;
import de.anomic.kelondro.table.CachedRecords;
import de.anomic.kelondro.table.EcoTable;
import de.anomic.kelondro.table.FlexTable;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.kelondro.util.FileUtils;
import de.anomic.plasma.plasmaSwitchboard;
Expand Down Expand Up @@ -90,30 +88,12 @@ public static serverObjects respond(final httpRequestHeader header, final server
prop.putNum("memoryUsedAfterInitAGC", (memoryTotalAfterInitAGC - memoryFreeAfterInitAGC) / KB);
prop.putNum("memoryUsedNow", (memoryTotalNow - memoryFreeNow) / MB);

// write table for FlexTable index sizes
Iterator<String> i = FlexTable.filenames();
// write table for EcoTable index sizes
Iterator<String> i = EcoTable.filenames();
String filename;
Map<String, String> map;
int p, c = 0;
long mem, totalmem = 0;
while (i.hasNext()) {
filename = i.next();
map = FlexTable.memoryStats(filename);
mem = Long.parseLong(map.get("tableIndexMem"));
totalmem += mem;
prop.put("TableList_" + c + "_tableIndexPath", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p));
prop.put("TableList_" + c + "_tableIndexChunkSize", map.get("tableIndexChunkSize"));
prop.putNum("TableList_" + c + "_tableIndexCount", map.get("tableIndexCount"));
prop.put("TableList_" + c + "_tableIndexMem", Formatter.bytesToString(mem));
c++;
}
prop.put("TableList", c);
prop.putNum("TableIndexTotalMem", totalmem / (1024 * 1024d));

// write table for EcoTable index sizes
i = EcoTable.filenames();
c = 0;
totalmem = 0;
while (i.hasNext()) {
filename = i.next();
map = EcoTable.memoryStats(filename);
Expand All @@ -135,32 +115,6 @@ public static serverObjects respond(final httpRequestHeader header, final server
prop.put("EcoList", c);
prop.putNum("EcoIndexTotalMem", totalmem / (1024 * 1024d));

// write node cache table
i = CachedRecords.filenames();
c = 0;
totalmem = 0;
while (i.hasNext()) {
filename = i.next();
map = CachedRecords.memoryStats(filename);
mem = Long.parseLong(map.get("nodeCacheMem"));
totalmem += mem;
prop.put("NodeList_" + c + "_nodeCachePath", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p));
prop.put("NodeList_" + c + "_nodeChunkSize", map.get("nodeChunkSize"));
prop.putNum("NodeList_" + c + "_nodeCacheCount", map.get("nodeCacheCount"));
prop.put("NodeList_" + c + "_nodeCacheMem", Formatter.bytesToString(mem));
prop.putNum("NodeList_" + c + "_nodeCacheReadHit", map.get("nodeCacheReadHit"));
prop.putNum("NodeList_" + c + "_nodeCacheReadMiss", map.get("nodeCacheReadMiss"));
prop.putNum("NodeList_" + c + "_nodeCacheWriteUnique", map.get("nodeCacheWriteUnique"));
prop.putNum("NodeList_" + c + "_nodeCacheWriteDouble", map.get("nodeCacheWriteDouble"));
prop.putNum("NodeList_" + c + "_nodeCacheDeletes", map.get("nodeCacheDeletes"));
prop.putNum("NodeList_" + c + "_nodeCacheFlushes", map.get("nodeCacheFlushes"));
c++;
}
prop.put("NodeList", c);
prop.putNum("nodeCacheStopGrow", CachedRecords.getMemStopGrow() / (1024 * 1024d));
prop.putNum("nodeCacheStartShrink", CachedRecords.getMemStartShrink() / (1024 * 1024d));
prop.putNum("nodeCacheTotalMem", totalmem / (1024 * 1024d));

// write object cache table
i = Cache.filenames();
c = 0;
Expand Down
32 changes: 0 additions & 32 deletions htroot/PerformanceMemory_p.xml
View file
Expand Up @@ -37,38 +37,6 @@
<EcoIndexTotalMem>#[EcoIndexTotalMem]#</EcoIndexTotalMem>
</EcoTable>

<FlexTable>
#{TableList}#
<entry>
<tableIndexPath>#[tableIndexPath]#</tableIndexPath>
<tableIndexChunkSize>#[tableIndexChunkSize]#</tableIndexChunkSize>
<tableIndexCount>#[tableIndexCount]#</tableIndexCount>
<tableIndexMem>#[tableIndexMem]#</tableIndexMem>
</entry>
#{/TableList}#
<TableIndexTotalMem>#[TableIndexTotalMem]#</TableIndexTotalMem>
</FlexTable>

<NodeCaches>
#{NodeList}#
<entry>
<nodeCachePath>#[nodeCachePath]#</nodeCachePath>
<nodeChunkSize>#[nodeChunkSize]#</nodeChunkSize>
<nodeCacheCount>#[nodeCacheCount]#</nodeCacheCount>
<nodeCacheMem>#[nodeCacheMem]#</nodeCacheMem>
<nodeCacheReadHit>#[nodeCacheReadHit]#</nodeCacheReadHit>
<nodeCacheReadMiss>#[nodeCacheReadMiss]#</nodeCacheReadMiss>
<nodeCacheWriteUnique>#[nodeCacheWriteUnique]#</nodeCacheWriteUnique>
<nodeCacheWriteDouble>#[nodeCacheWriteDouble]#</nodeCacheWriteDouble>
<nodeCacheDeletes>#[nodeCacheDeletes]#</nodeCacheDeletes>
<nodeCacheFlushes>#[nodeCacheFlushes]#</nodeCacheFlushes>
</entry>
#{/NodeList}#
<nodeCacheTotalMem>#[nodeCacheTotalMem]#</nodeCacheTotalMem>
<nodeCacheStopGrow>#[nodeCacheStopGrow]#</nodeCacheStopGrow>
<nodeCacheStartShrink>#[nodeCacheStartShrink]#</nodeCacheStartShrink>
</NodeCaches>

<ObjectReadCaches>
#{ObjectList}#
<entry>
Expand Down
4 changes: 3 additions & 1 deletion htroot/yacy/search.java
View file
Expand Up @@ -193,6 +193,7 @@ public static serverObjects respond(final httpRequestHeader header, final server
prefer,
plasmaSearchQuery.contentdomParser(contentdom),
language,
"", // no navigation
false,
count,
0,
Expand Down Expand Up @@ -243,7 +244,8 @@ public static serverObjects respond(final httpRequestHeader header, final server
prefer,
plasmaSearchQuery.
contentdomParser(contentdom),
language,
language,
"", // no navigation
false,
count,
0,
Expand Down
5 changes: 5 additions & 0 deletions htroot/yacysearch.java
View file
Expand Up @@ -312,6 +312,10 @@ public static serverObjects respond(final httpRequestHeader header, final server
if (language == null) language = "en";
}

// navigation
String navigation = (post == null) ? "" : post.get("nav", "");

// the query
final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring.trim()); // converts also umlaute

int maxDistance = (querystring.indexOf('"') >= 0) ? maxDistance = query.length - 1 : Integer.MAX_VALUE;
Expand Down Expand Up @@ -385,6 +389,7 @@ public static serverObjects respond(final httpRequestHeader header, final server
prefermask,
contentdomCode,
language,
navigation,
fetchSnippets,
itemsPerPage,
offset,
Expand Down
6 changes: 3 additions & 3 deletions source/de/anomic/data/URLAnalysis.java
View file
Expand Up @@ -51,7 +51,7 @@
import java.util.zip.GZIPOutputStream;

import de.anomic.kelondro.index.HandleSet;
import de.anomic.kelondro.index.IntegerHandleIndex;
import de.anomic.kelondro.index.HandleMap;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.text.MetadataRepository;
import de.anomic.kelondro.text.ReferenceContainerArray;
Expand Down Expand Up @@ -396,7 +396,7 @@ public static void sortsplit(String urlfile) {

public static void incell(File cellPath, String statisticPath) {
try {
IntegerHandleIndex idx = ReferenceContainerArray.referenceHashes(
HandleMap idx = ReferenceContainerArray.referenceHashes(
cellPath,
Segment.wordReferenceFactory,
Base64Order.enhancedCoder,
Expand All @@ -411,7 +411,7 @@ public static void incell(File cellPath, String statisticPath) {

public static int diffurlcol(String metadataPath, String statisticFile, String diffFile) throws IOException {
System.out.println("INDEX DIFF URL-COL startup");
IntegerHandleIndex idx = new IntegerHandleIndex(URLMetadataRow.rowdef.primaryKeyLength, URLMetadataRow.rowdef.objectOrder, new File(statisticFile), 0);
HandleMap idx = new HandleMap(URLMetadataRow.rowdef.primaryKeyLength, URLMetadataRow.rowdef.objectOrder, 4, new File(statisticFile), 0);
MetadataRepository mr = new MetadataRepository(new File(metadataPath));
HandleSet hs = new HandleSet(URLMetadataRow.rowdef.primaryKeyLength, URLMetadataRow.rowdef.objectOrder, 0, 1000000);
System.out.println("INDEX DIFF URL-COL loaded dump, starting diff");
Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/data/blogBoard.java
View file
Expand Up @@ -67,7 +67,7 @@ public blogBoard(final File actpath, final File newFile) throws IOException {
new File(actpath.getParent()).mkdir();
new File(newFile.getParent()).mkdir();
if (database == null) {
database = new MapView(BLOBTree.toHeap(actpath, true, true, keyLength, recordSize, '_', NaturalOrder.naturalOrder, true, false, false, newFile), 500, '_');
database = new MapView(BLOBTree.toHeap(actpath, true, true, keyLength, recordSize, '_', NaturalOrder.naturalOrder, false, false, newFile), 500, '_');
}
}

Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/data/blogBoardComments.java
View file
Expand Up @@ -70,7 +70,7 @@ public blogBoardComments(final File actpath, final File newFile) throws IOExcept
new File(actpath.getParent()).mkdir();
new File(newFile.getParent()).mkdir();
if (database == null) {
database = new MapView(BLOBTree.toHeap(actpath, true, true, keyLength, recordSize, '_', NaturalOrder.naturalOrder, false, false, false, newFile), 500, '_');
database = new MapView(BLOBTree.toHeap(actpath, true, true, keyLength, recordSize, '_', NaturalOrder.naturalOrder, false, false, newFile), 500, '_');
}
}

Expand Down
6 changes: 3 additions & 3 deletions source/de/anomic/data/bookmarksDB.java
View file
Expand Up @@ -116,17 +116,17 @@ public bookmarksDB(
tagCache=new TreeMap<String, Tag>();
bookmarksFile.getParentFile().mkdirs();
//this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false));
this.bookmarksTable = new MapView(BLOBTree.toHeap(bookmarksFile, true, true, 12, 256, '_', NaturalOrder.naturalOrder, true, false, false, bookmarksFileNew), 1000, '_');
this.bookmarksTable = new MapView(BLOBTree.toHeap(bookmarksFile, true, true, 12, 256, '_', NaturalOrder.naturalOrder, false, false, bookmarksFileNew), 1000, '_');

// tags
tagsFile.getParentFile().mkdirs();
final boolean tagsFileExisted = tagsFile.exists();
this.tagsTable = new MapView(BLOBTree.toHeap(tagsFile, true, true, 12, 256, '_', NaturalOrder.naturalOrder, true, false, false, tagsFileNew), 500, '_');
this.tagsTable = new MapView(BLOBTree.toHeap(tagsFile, true, true, 12, 256, '_', NaturalOrder.naturalOrder, false, false, tagsFileNew), 500, '_');
if (!tagsFileExisted) rebuildTags();

// dates
final boolean datesExisted = datesFile.exists();
this.datesTable = new MapView(BLOBTree.toHeap(datesFile, true, true, 20, 256, '_', NaturalOrder.naturalOrder, true, false, false, datesFileNew), 500, '_');
this.datesTable = new MapView(BLOBTree.toHeap(datesFile, true, true, 20, 256, '_', NaturalOrder.naturalOrder, false, false, datesFileNew), 500, '_');
if (!datesExisted) rebuildDates();

// autoReCrawl
Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/data/messageBoard.java
View file
Expand Up @@ -55,7 +55,7 @@ public messageBoard(final File path, final File pathNew) throws IOException {
new File(path.getParent()).mkdir();
new File(pathNew.getParent()).mkdir();
if (database == null) {
database = new MapView(BLOBTree.toHeap(path, true, true, categoryLength + dateFormat.length() + 2, recordSize, '_', NaturalOrder.naturalOrder, true, false, false, pathNew), 500, '_');
database = new MapView(BLOBTree.toHeap(path, true, true, categoryLength + dateFormat.length() + 2, recordSize, '_', NaturalOrder.naturalOrder, false, false, pathNew), 500, '_');
}
sn = 0;
}
Expand Down
2 changes: 1 addition & 1 deletion source/de/anomic/data/userDB.java
View file
Expand Up @@ -60,7 +60,7 @@ public userDB(final File userTableFile, final File userTableFileNew) throws IOEx
this.userTableFile = userTableFileNew;
userTableFile.getParentFile().mkdirs();
userTableFileNew.getParentFile().mkdirs();
this.userTable = new MapView(BLOBTree.toHeap(userTableFile, true, true, 128, 256, '_', NaturalOrder.naturalOrder, true, false, false, userTableFile), 10, '_');
this.userTable = new MapView(BLOBTree.toHeap(userTableFile, true, true, 128, 256, '_', NaturalOrder.naturalOrder, false, false, userTableFile), 10, '_');
}

void resetDatabase() {
Expand Down
4 changes: 2 additions & 2 deletions source/de/anomic/data/wiki/wikiBoard.java
View file
Expand Up @@ -57,11 +57,11 @@ public wikiBoard(
final File bkppath, final File bkppathNew) throws IOException {
new File(actpath.getParent()).mkdirs();
if (datbase == null) {
datbase = new MapView(BLOBTree.toHeap(actpath, true, true, keyLength, recordSize, '_', NaturalOrder.naturalOrder, true, false, false, actpathNew), 500, '_');
datbase = new MapView(BLOBTree.toHeap(actpath, true, true, keyLength, recordSize, '_', NaturalOrder.naturalOrder, false, false, actpathNew), 500, '_');
}
new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) {
bkpbase = new MapView(BLOBTree.toHeap(bkppath, true, true, keyLength + dateFormat.length(), recordSize, '_', NaturalOrder.naturalOrder, true, false, false, bkppathNew), 500, '_');
bkpbase = new MapView(BLOBTree.toHeap(bkppath, true, true, keyLength + dateFormat.length(), recordSize, '_', NaturalOrder.naturalOrder, false, false, bkppathNew), 500, '_');
}
}

Expand Down

0 comments on commit c079b18

Please sign in to comment.