re-write of kelondroCollectionIndex. This is the data structure that

shall replace the current assortment files.
* used the kelondroFlexTable to hold the index of collections
* used kelondroRow definitions to declare all data structures
* fixed several bugs that appeared in kelondroRowSet and kelondroRowCollection during testing


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2344 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/dbtest.java

@@ -187,7 +187,7 @@ public static void main(String[] args) {
             }
             if (dbe.equals("kelondroFlexTable")) {
                 File tablepath = new File(tablename).getParentFile();
-                table = new kelondroFlexTable(tablepath, new File(tablename).getName(), buffer, preload, testRow, true);
+                table = new kelondroFlexTable(tablepath, new File(tablename).getName(), kelondroBase64Order.enhancedCoder, buffer, preload, testRow, true);
             }
             if (dbe.equals("mysql")) {
                 table = new dbTable("mysql", testRow);
@@ -342,7 +342,7 @@ public static void main(String[] args) {
             if (table instanceof kelondroTree) ((kelondroTree) table).close();
             if (table instanceof kelondroFlexTable) ((kelondroFlexTable) table).close();
             if (table instanceof kelondroSplittedTree) ((kelondroSplittedTree) table).close();
-            if (table instanceof dbTable) ((dbTable)table).closeDatabaseConnection();
+            if (table instanceof dbTable) ((dbTable)table).close();
 
             long afterclose = System.currentTimeMillis();
 
@@ -408,11 +408,11 @@ private void openDatabaseConnection(String dbType) throws Exception{
 
     }        
 
-    public void closeDatabaseConnection() throws Exception {
+    public void close() throws IOException {
         try {
             this.theDBConnection.close();
         } catch (Exception e) {
-            throw new Exception ("Unable to close the database connection.");
+            throw new IOException("Unable to close the database connection.");
         }
     }
 

source/de/anomic/index/indexCollectionRI.java

@@ -45,7 +45,7 @@ public indexCollectionRI(File path, String filenameStub, long buffersize, long p
         collectionIndex = new kelondroCollectionIndex(
                 path, filenameStub, 9 /*keyLength*/,
                 kelondroNaturalOrder.naturalOrder, buffersize, preloadTime,
-                4 /*loadfactor*/, rowdef, 8 /*partitions*/);
+                4 /*loadfactor*/, rowdef);
     }
 
     public int size() {

source/de/anomic/index/indexURLEntry.java

@@ -147,7 +147,7 @@ public byte[] toEncodedByteArrayForm(boolean includeHash) {
     }
 
    public String toPropertyForm() {
-        return entry.toPropertyForm(true);
+        return entry.toPropertyForm(true, false);
     }
 
     public Entry toKelondroEntry() {

source/de/anomic/kelondro/kelondroColumn.java

@@ -56,6 +56,7 @@ public kelondroColumn(String celldef) {
         // example: <UDate-3>
 
         // cut quotes etc.
+        celldef = celldef.trim();
         if (celldef.startsWith("<")) celldef = celldef.substring(1);
         if (celldef.endsWith(">")) celldef = celldef.substring(0, celldef.length() - 1);
 
@@ -200,4 +201,34 @@ public String description() {
         return this.description;
     }
 
+    public String toString() {
+        StringBuffer s = new StringBuffer();
+        switch (celltype) {
+        case celltype_boolean:
+            s.append("boolean ");
+            break;
+        case celltype_binary:
+            s.append("byte[] ");
+            break;
+        case celltype_string:
+            s.append("String ");
+            break;
+        case celltype_cardinal:
+            s.append("Cardinal ");
+            break;
+        }
+        s.append(nickname);
+        s.append('-');
+        s.append(cellwidth);
+        s.append(' ');
+        switch (encoder) {
+        case encoder_b64e:
+            s.append(" {b64e}");
+            break;
+        case encoder_b256:
+            s.append(" {b256}");
+            break;
+        }
+        return new String(s);
+    }
 }

source/de/anomic/kelondro/kelondroFixedWidthArray.java

@@ -92,7 +92,7 @@ public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry)
     }
 
     public synchronized kelondroRow.Entry get(int index) throws IOException {
-        if (index >= size()) throw new kelondroException(filename, "out of bounds, index=" + index + ", size=" + size());
+        //if (index >= size()) throw new kelondroException(filename, "out of bounds, index=" + index + ", size=" + size());
         return row().newEntry(getNode(new Handle(index)).getValueRow());
     }
 

source/de/anomic/kelondro/kelondroFlexTable.java

@@ -33,23 +33,25 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
 
     private kelondroBytesIntMap index;
 
-    public kelondroFlexTable(File path, String tablename, long buffersize, long preloadTime, kelondroRow rowdef, boolean exitOnFail) throws IOException {
+    public kelondroFlexTable(File path, String tablename, kelondroOrder objectOrder, long buffersize, long preloadTime, kelondroRow rowdef, boolean exitOnFail) throws IOException {
         super(path, tablename, rowdef, exitOnFail);
         File newpath = new File(path, tablename + ".table");
         File indexfile = new File(newpath, "col.000.index");
         kelondroIndex ki = null;
         String description = new String(this.col[0].getDescription());
-        System.out.println("*** Last Startup time: " + description.substring(4));
+        int p = description.indexOf(';', 4);
+        long stt = (p > 0) ? Long.parseLong(description.substring(4, p)) : 0;
+        System.out.println("*** Last Startup time: " + stt + " milliseconds");
         long start = System.currentTimeMillis();
 
         if (indexfile.exists()) {
             // use existing index file
             System.out.println("*** Using File index " + indexfile);
             ki = new kelondroTree(indexfile, buffersize, preloadTime, 10);
-        } else if (size() > 100000) {
+        } else if (stt > preloadTime) {
             // generate new index file
             System.out.print("*** Generating File index for " + size() + " entries from " + indexfile);
-            ki = initializeTreeIndex(indexfile, buffersize, preloadTime);
+            ki = initializeTreeIndex(indexfile, buffersize, preloadTime, objectOrder);
 
             System.out.println(" -done-");
             System.out.println(ki.size()
@@ -58,7 +60,7 @@ public kelondroFlexTable(File path, String tablename, long buffersize, long prel
         } else {
             // fill the index
             System.out.print("*** Loading RAM index for " + size() + " entries from "+ newpath);
-            ki = initializeRamIndex();
+            ki = initializeRamIndex(objectOrder);
 
             System.out.println(" -done-");
             System.out.println(ki.size()
@@ -71,9 +73,9 @@ public kelondroFlexTable(File path, String tablename, long buffersize, long prel
         super.col[0].setDescription(description.getBytes());
     }
 
-    private kelondroIndex initializeRamIndex() throws IOException {
+    private kelondroIndex initializeRamIndex(kelondroOrder objectOrder) throws IOException {
         kelondroRowBufferedSet ri = new kelondroRowBufferedSet(new kelondroRow(new int[]{super.row().width(0), 4}), 0);
-        ri.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
+        ri.setOrdering(objectOrder, 0);
         Iterator content = super.col[0].contentNodes(-1);
         kelondroRecords.Node node;
         kelondroRow.Entry indexentry;
@@ -92,14 +94,15 @@ private kelondroIndex initializeRamIndex() throws IOException {
         }
         System.out.print(" -ordering- ");
         System.out.flush();
-        ri.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
         ri.shape();
         return ri;
     }
 
 
-    private kelondroIndex initializeTreeIndex(File indexfile, long buffersize, long preloadTime) throws IOException {
-        kelondroTree index = new kelondroTree(indexfile, buffersize, preloadTime, 10, rowdef.width(0), 4, true);
+    private kelondroIndex initializeTreeIndex(File indexfile, long buffersize, long preloadTime, kelondroOrder objectOrder) throws IOException {
+        kelondroTree index = new kelondroTree(indexfile, buffersize, preloadTime, 10,
+                new kelondroRow("byte[] key-" + rowdef.width(0) + ", int reference-4"),
+                objectOrder, 2, 80, true);
         Iterator content = super.col[0].contentNodes(-1);
         kelondroRecords.Node node;
         kelondroRow.Entry indexentry;

source/de/anomic/kelondro/kelondroFlexWidthArray.java

@@ -186,8 +186,9 @@ public void print() throws IOException {
         for (int i = 0; i < size(); i++) {
             System.out.print("row " + i + ": ");
             row = get(i);
-            for (int j = 0; j < row().columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
-            System.out.println();
+            System.out.println(row.toString());
+            //for (int j = 0; j < row().columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
+            //System.out.println();
         }
         System.out.println("EndOfTable");
     }

source/de/anomic/kelondro/kelondroIndex.java

@@ -60,5 +60,5 @@ public interface kelondroIndex {
     public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException;
     public kelondroRow.Entry remove(byte[] key) throws IOException;
     //public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException;
-
+    public void close() throws IOException;
 }

source/de/anomic/kelondro/kelondroRow.java

@@ -130,6 +130,16 @@ public int[] widths() {
         return w;
     }
 
+    public String toString() {
+        StringBuffer s = new StringBuffer();
+        s.append(row[0].toString());
+        for (int i = 1; i < row.length; i++) {
+            s.append(", ");
+            s.append(row[i].toString());
+        }
+        return new String(s);
+    }
+
     public Entry newEntry() {
         return new Entry();
     }
@@ -370,7 +380,7 @@ public byte[] toEncodedBytesForm() {
          }
         */
 
-        public String toPropertyForm(boolean includeBraces) {
+        public String toPropertyForm(boolean includeBraces, boolean decimalCardinal) {
             StringBuffer sb = new StringBuffer();
             if (includeBraces) sb.append("{");
             int encoder, cellwidth;
@@ -395,15 +405,20 @@ public String toPropertyForm(boolean includeBraces) {
                     sb.append(',');
                     continue;
                 case kelondroColumn.celltype_cardinal:
-                    if (encoder == kelondroColumn.encoder_b64e) {
+                    if (decimalCardinal) {
+                        sb.append(row[i].nickname());
+                        sb.append('=');
+                        sb.append(Long.toString(bytes2long(rowinstance, colstart[i], cellwidth)));
+                        sb.append(',');
+                        continue;
+                    } else if (encoder == kelondroColumn.encoder_b64e) {
                         sb.append(row[i].nickname());
                         sb.append('=');
                         long c = bytes2long(rowinstance, colstart[i], cellwidth);
                         sb.append(kelondroBase64Order.enhancedCoder.encodeLongSmart(c, cellwidth).getBytes());
                         sb.append(',');
                         continue;
-                    }
-                    throw new kelondroException("ROW", "toEncodedForm of celltype cardinal has no encoder (" + encoder + ")");
+                    } else throw new kelondroException("ROW", "toEncodedForm of celltype cardinal has no encoder (" + encoder + ")");
                 }
             }
             if (sb.charAt(sb.length() - 1) == ',') sb.deleteCharAt(sb.length() - 1); // remove ',' at end
@@ -412,14 +427,7 @@ public String toPropertyForm(boolean includeBraces) {
         }
 
         public String toString() {
-            StringBuffer b = new StringBuffer();
-            b.append('{');
-            for (int i = 0; i < columns(); i++) {
-                b.append(getColString(i, null));
-                if (i < columns() - 1) b.append(", ");
-            }
-            b.append('}');
-            return new String(b);
+            return toPropertyForm(true, true);
         }
     }
 

source/de/anomic/kelondro/kelondroRowBufferedSet.java

@@ -112,23 +112,6 @@ public String toString() {
         }
     }
 
-    public byte[] toByteArray() {
-        synchronized (buffer) {
-            flush();
-            return super.toByteArray();
-        }
-    }
-
-    /*
-    public void add(byte[] a) {
-        this.add(super.rowdef.newEntry(a));
-    }
-    
-    public void add(kelondroRow.Entry a) {
-        this.put(a);
-    }
-    */
-
     public kelondroRow.Entry get(byte[] key) {
         long handle = profile.startRead();
         kelondroRow.Entry entry = null;

source/de/anomic/kelondro/kelondroRowCollection.java

@@ -35,7 +35,15 @@ public class kelondroRowCollection {
     protected int           sortBound;
     protected kelondroOrder sortOrder;
     protected int           sortColumn;
-
+
+    private static final int exp_chunkcount  = 0;
+    private static final int exp_last_read   = 1;
+    private static final int exp_last_wrote  = 2;
+    private static final int exp_order_type  = 3;
+    private static final int exp_order_col   = 4;
+    private static final int exp_order_bound = 5;
+    private static final int exp_collection  = 6;
+
     public kelondroRowCollection(kelondroRow rowdef) {
         this(rowdef, 0);
     }
@@ -51,17 +59,72 @@ public kelondroRowCollection(kelondroRow rowdef, int objectCount) {
         this.lastTimeWrote = System.currentTimeMillis();
     }
 
-    public kelondroRowCollection(kelondroRow rowdef, int objectCount, byte[] cache) {
+    public kelondroRowCollection(kelondroRow rowdef, int objectCount, byte[] cache, kelondroOrder sortOrder, int sortColumn, int sortBound) {
         this.rowdef = rowdef;
         this.chunkcache = cache;
         this.chunkcount = objectCount;
-        this.sortColumn = 0;
-        this.sortOrder = null;
-        this.sortBound = 0;
+        this.sortColumn = sortColumn;
+        this.sortOrder = sortOrder;
+        this.sortBound = sortBound;
         this.lastTimeRead = System.currentTimeMillis();
         this.lastTimeWrote = System.currentTimeMillis();
     }
 
+    public kelondroRowCollection(kelondroRow rowdef, byte[] exportedCollectionRowinstance) {
+        this.rowdef = rowdef;
+        kelondroRow.Entry exportedCollection = exportRow(exportedCollectionRowinstance.length - exportOverheadSize).newEntry(exportedCollectionRowinstance);
+        this.chunkcount = (int) exportedCollection.getColLongB256(exp_chunkcount);
+        this.lastTimeRead = (exportedCollection.getColLongB256(exp_last_read) + 10957) * day;
+        this.lastTimeWrote = (exportedCollection.getColLongB256(exp_last_wrote) + 10957) * day;
+        String sortOrderKey = exportedCollection.getColString(exp_order_type, null);
+        if (sortOrderKey.equals("__")) {
+            this.sortOrder = null;
+        } else {
+            this.sortOrder = kelondroNaturalOrder.bySignature(sortOrderKey);
+            if (this.sortOrder == null) this.sortOrder = kelondroBase64Order.bySignature(sortOrderKey);
+        }
+        this.sortColumn = (int) exportedCollection.getColLongB256(exp_order_col);
+        this.sortBound = (int) exportedCollection.getColLongB256(exp_order_bound);
+        this.chunkcache = exportedCollection.getColBytes(exp_collection);        
+    }
+
+    private static final long day = 1000 * 60 * 60 * 24;
+
+    public static int daysSince2000(long time) {
+        return (int) (time / day) - 10957;
+    }
+
+    private kelondroRow exportRow(int chunkcachelength) {
+        // find out the size of this collection
+        return new kelondroRow(
+                "int size-4 {b256}," +
+                "short lastread-2 {b256}," + // as daysSince2000
+                "short lastwrote-2 {b256}," + // as daysSince2000
+                "byte[] orderkey-2," +
+                "short ordercol-2 {b256}," +
+                "short orderbound-2 {b256}," +
+                "byte[] collection-" + chunkcachelength
+                );
+    }
+
+    public static final int exportOverheadSize = 14;
+
+    public byte[] exportCollection() {
+        // returns null if the collection is empty
+        if (size() == 0) return null;
+        trim();
+        kelondroRow row = exportRow(chunkcache.length);
+        kelondroRow.Entry entry = row.newEntry();
+        entry.setColLongB256(exp_chunkcount, size());
+        entry.setColLongB256(exp_last_read, daysSince2000(this.lastTimeRead));
+        entry.setColLongB256(exp_last_wrote, daysSince2000(this.lastTimeWrote));
+        entry.setCol(exp_order_type, (this.sortOrder == null) ? "__".getBytes() : this.sortOrder.signature().getBytes());
+        entry.setColLongB256(exp_order_col, this.sortColumn);
+        entry.setColLongB256(exp_order_bound, this.sortBound);
+        entry.setCol(exp_collection, chunkcache);
+        return entry.bytes();
+    }
+
     public kelondroRow row() {
         return this.rowdef;
     }
@@ -357,10 +420,6 @@ public String toString() {
         while (i.hasNext()) s.append(", " + ((kelondroRow.Entry) i.next()).toString());
         return new String(s);
     }
-
-    public byte[] toByteArray() {
-        return this.chunkcache;
-    }
 
     private final int compare(int i, int j) {
         assert (i < chunkcount);
@@ -385,4 +444,9 @@ private final int compare(int i, int j) {
         return c;
     }
 
+    public static void main(String[] args) {
+        System.out.println(new java.util.Date(10957 * day));
+        System.out.println(new java.util.Date(0));
+        System.out.println(daysSince2000(System.currentTimeMillis()));
+    }
 }