some enhancements to caching

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2236 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/kelondro/kelondroBytesIntMap.java

@@ -60,7 +60,7 @@ public int removei(byte[] key) {
             if (System.currentTimeMillis() - this.lastTimeWrote > 10000) this.trim();
             return -1;
         }
-        kelondroRow.Entry indexentry = remove(key);
+        kelondroRow.Entry indexentry = removeMarked(key);
         if (indexentry == null) return -1;
         return (int) indexentry.getColLongB256(1);
     }

source/de/anomic/kelondro/kelondroFlexTable.java

@@ -54,7 +54,7 @@ public kelondroFlexTable(File path, String tablename, kelondroRow rowdef, boolea
             if ((i % 10000) == 0) System.out.print('.');
         }
         this.index.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
-        index.sort();
+        index.shape();
         System.out.println(index.size() + " index entries initialized and sorted");
     }
 

source/de/anomic/kelondro/kelondroIntBytesMap.java

@@ -60,7 +60,7 @@ public byte[] removeb(int ii) {
             if (System.currentTimeMillis() - this.lastTimeWrote > 10000) this.trim();
             return null;
         }
-        kelondroRow.Entry indexentry = super.remove(kelondroNaturalOrder.encodeLong((long) ii, 4));
+        kelondroRow.Entry indexentry = super.removeMarked(kelondroNaturalOrder.encodeLong((long) ii, 4));
         if (indexentry == null) return null;
         return indexentry.getColBytes(1);
     }

source/de/anomic/kelondro/kelondroRecords.java

@@ -81,7 +81,7 @@ public class kelondroRecords {
 
     // constants
     private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
-    private static final long memBlock = 500000; // do not fill cache further if the amount of available memory is less that this
+    private static final long memBlock = 50000000; // do not fill cache further if the amount of available memory is less that this
     public final static boolean useWriteBuffer = false;
 
     // memory calculation
@@ -427,9 +427,9 @@ private void initCache(long buffersize) {
                 this.cacheScore = new kelondroMScoreCluster(); // cache control of CP_HIGH caches
             }
             this.cacheHeaders = new kelondroIntBytesMap[]{
-                    new kelondroIntBytesMap(this.headchunksize, this.cacheSize / 2),
                     new kelondroIntBytesMap(this.headchunksize, 0),
-                    new kelondroIntBytesMap(this.headchunksize, 0)
+                    new kelondroIntBytesMap(this.headchunksize, 0),
+                    new kelondroIntBytesMap(this.headchunksize, this.cacheSize / 2)
                     };
             this.cacheHeaders[0].setOrdering(kelondroNaturalOrder.naturalOrder, 0);
             this.cacheHeaders[1].setOrdering(kelondroNaturalOrder.naturalOrder, 0);
@@ -489,6 +489,20 @@ public int[] cacheNodeStatus() {
         };
     }
 
+    public String cacheNodeStatusString() {
+        return
+              "cacheMaxSize=" + cacheSize +
+            ", CP_HIGH=" + cacheHeaders[CP_HIGH].size() +
+            ", CP_MEDIUM=" + cacheHeaders[CP_MEDIUM].size() +
+            ", CP_LOW=" + cacheHeaders[CP_LOW].size() +
+            ", readHit=" + readHit +
+            ", readMiss=" + readMiss +
+            ", writeUnique=" + writeUnique +
+            ", writeDouble=" + writeDouble +
+            ", cacheDelete=" + cacheDelete +
+            ", cacheFlush=" + cacheFlush;
+    }
+
     private static int[] cacheCombinedStatus(int[] a, int[] b) {
         int[] c = new int[a.length];
         for (int i = a.length - 1; i >= 0; i--) c[i] = a[i] + b[i];
@@ -916,9 +930,9 @@ private boolean cacheSpace(int forPriority) {
                     // we simply clear the cache
                     String error = "cachScore error: " + e.getMessage() + "; cachesize=" + cacheSize + ", cache.size()=[" + cacheHeaders[0].size() + "," + cacheHeaders[1].size() + "," + cacheHeaders[2].size() + "], cacheScore.size()=" + cacheScore.size();
                     cacheScore = new kelondroMScoreCluster();
-                    cacheHeaders[CP_LOW]    = new kelondroIntBytesMap(headchunksize, cacheSize / 2);
+                    cacheHeaders[CP_LOW]    = new kelondroIntBytesMap(headchunksize, 0);
                     cacheHeaders[CP_MEDIUM] = new kelondroIntBytesMap(headchunksize, 0);
-                    cacheHeaders[CP_HIGH]   = new kelondroIntBytesMap(headchunksize, 0);
+                    cacheHeaders[CP_HIGH]   = new kelondroIntBytesMap(headchunksize, cacheSize / 2);
                     cacheHeaders[0].setOrdering(kelondroNaturalOrder.naturalOrder, 0);
                     cacheHeaders[1].setOrdering(kelondroNaturalOrder.naturalOrder, 0);
                     cacheHeaders[2].setOrdering(kelondroNaturalOrder.naturalOrder, 0);

source/de/anomic/kelondro/kelondroRowBufferedSet.java

@@ -164,13 +164,13 @@ public kelondroRow.Entry put(kelondroRow.Entry newentry) {
         }
     }
 
-    public kelondroRow.Entry remove(byte[] a) {
+    public kelondroRow.Entry removeShift(byte[] a) {
         synchronized (buffer) {
             if (useRowCollection) {
                 kelondroRow.Entry oldentry = (kelondroRow.Entry) buffer.remove(a);
                 if (oldentry == null) {
                     // try the collection
-                    return super.remove(a);
+                    return super.removeShift(a);
                 } else {
                     // the entry was in buffer
                     return oldentry;
@@ -181,11 +181,28 @@ public kelondroRow.Entry remove(byte[] a) {
         }
     }
 
-    public void removeAll(kelondroRowCollection c) {
+    public kelondroRow.Entry removeMarked(byte[] a) {
+        synchronized (buffer) {
+            if (useRowCollection) {
+                kelondroRow.Entry oldentry = (kelondroRow.Entry) buffer.remove(a);
+                if (oldentry == null) {
+                    // try the collection
+                    return super.removeMarked(a);
+                } else {
+                    // the entry was in buffer
+                    return oldentry;
+                }
+            } else {
+                return (kelondroRow.Entry) buffer.remove(a); // test
+            }
+        }
+    }
+
+    public void removeMarkedAll(kelondroRowCollection c) {
         // this can be enhanced
         synchronized (buffer) {
             flush();
-            super.removeAll(c);
+            super.removeMarkedAll(c);
         }
     }
 
@@ -195,8 +212,8 @@ public static void main(String[] args) {
         c.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
         for (int i = 0; i < test.length; i++) c.add(test[i].getBytes());
         for (int i = 0; i < test.length; i++) c.add(test[i].getBytes());
-        c.sort();
-        c.remove("fuenf".getBytes());
+        c.removeMarked("fuenf".getBytes());
+        c.shape();
         Iterator i = c.elements();
         String s;
         System.out.print("INPUT-ITERATOR: ");
@@ -207,7 +224,7 @@ public static void main(String[] args) {
         }
         System.out.println("");
         System.out.println("INPUT-TOSTRING: " + c.toString());
-        c.sort();
+        c.shape();
         System.out.println("SORTED        : " + c.toString());
         c.uniq();
         System.out.println("UNIQ          : " + c.toString());
@@ -232,7 +249,7 @@ public static void main(String[] args) {
                     " entries/second, size = " + c.size());
         }
         System.out.println("bevore sort: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
-        c.sort();
+        c.shape();
         System.out.println("after sort: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
         c.uniq();
         System.out.println("after uniq: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");

source/de/anomic/kelondro/kelondroRowCollection.java

@@ -145,13 +145,18 @@ public final void addAll(kelondroRowCollection c) {
         }
     }
 
-    public final void remove(int p) {
+    protected final void removeShift(int pos, int dist, int upBound) {
+        System.arraycopy(chunkcache, (pos + dist) * rowdef.objectsize(),
+                         chunkcache, pos * rowdef.objectsize(),
+                         (upBound - pos - dist) * rowdef.objectsize());
+        if ((pos < sortBound) && (upBound >= sortBound)) sortBound -= dist;
+    }
+
+    public final void removeShift(int p) {
         assert ((p >= 0) && (p < chunkcount) && (chunkcount > 0));
         //System.out.println("REMOVE at pos " + p + ", chunkcount=" + chunkcount + ", sortBound=" + sortBound);
         synchronized (chunkcache) {
-            System.arraycopy(chunkcache, (p + 1) * rowdef.objectsize(), chunkcache, p * rowdef.objectsize(), (chunkcount - p - 1) * rowdef.objectsize());
-            chunkcount--;
-            if (p < sortBound) sortBound--;
+            removeShift(p, 1, chunkcount--);
         }
         this.lastTimeWrote = System.currentTimeMillis();
     }
@@ -299,7 +304,7 @@ private final void isort(int L, int R) {
                 swap(j, j - 1, 0);
     }
 
-    private final int swap(int i, int j, int p) {
+    protected final int swap(int i, int j, int p) {
         if (i == j) return p;
         if (this.chunkcount * this.rowdef.objectsize() < this.chunkcache.length) {
             // there is space in the chunkcache that we can use as buffer
@@ -326,7 +331,7 @@ public void uniq() {
             while (i < chunkcount - 1) {
                 if (compare(i, i + 1) == 0) {
                     //System.out.println("DOUBLE: " + new String(this.chunkcache, this.chunksize * i, this.chunksize));
-                    remove(i);
+                    removeShift(i);
                 } else {
                     i++;
                 }
@@ -368,5 +373,5 @@ private final int compare(int i, int j) {
                 */
         return c;
     }
-    
+
 }