added test for bitsetimpl; fixed clone bug in memorygraphsafe; implem…

…ented memory efficient delete for memorygraphsafe; moved several memorygraph stuff to memorygraphsafe

README.md

@@ -29,13 +29,12 @@ If you want to import a bigger OSM (Germany) then run:
  * For Germany it takes approx 25 minutes for the import and roughly 1 minute for the ugly&slow UI to pop up. Probably you'll need to tune the memory settings - send me a mail if this fails!
  * At the moment the UI is a bit rough and simple so, don't click or drag too much ;)
 
-QuadTree Usage
----------------
-
-See the performacen comparison subproject and the articles:
-
-http://karussell.wordpress.com/2012/06/17/failed-experiment-memory-efficient-spatial-hashtable/
 
-http://karussell.wordpress.com/2012/05/29/tricks-to-speed-up-neighbor-searches-of-quadtrees-geo-spatial-java/
-
-http://karussell.wordpress.com/2012/05/23/spatial-keys-memory-efficient-geohashes/
+Further Links
+---------------
+ * [Spatial Key](http://karussell.wordpress.com/2012/05/23/spatial-keys-memory-efficient-geohashes/)
+ * [Speed up your Quad-tree](http://karussell.wordpress.com/2012/05/29/tricks-to-speed-up-neighbor-searches-of-quadtrees-geo-spatial-java/)
+ * [Spatial Hashtable](http://karussell.wordpress.com/2012/06/17/failed-experiment-memory-efficient-spatial-hashtable/)
+ * [Author@Twitter](https://twitter.com/timetabling)
+ * [Lumeo - Implementing a Graph API via Lucene](https://github.com/karussell/lumeo)
+ * [Cassovary - A production ready (?) in-memory & memory efficient Graph DB](https://github.com/twitter/cassovary)

core/src/main/java/de/jetsli/graph/coll/MyBitSet.java

@@ -17,7 +17,7 @@
 
 /**
  * Wrapper interface for different implementations like OpenBitset, BitSet, ...
- * 
+ *
  * Supports only integer value indices.
  *
  * @author Peter Karich, info@jetsli.de
@@ -29,6 +29,10 @@ public interface MyBitSet {
     void add(int index);
 
     int getCardinality();
-    
+
     void clear();
+
+    void ensureCapacity(int size);
+
+    int next(int index);
 }

core/src/main/java/de/jetsli/graph/coll/MyBitSetImpl.java

@@ -43,4 +43,13 @@ public void add(int index) {
     public int getCardinality() {
         return super.cardinality();
     }        
+
+    @Override
+    public void ensureCapacity(int size) {
+    }
+
+    @Override
+    public int next(int index) {
+        return super.nextSetBit(index);
+    }
 }

core/src/main/java/de/jetsli/graph/coll/MyOpenBitSet.java

@@ -24,10 +24,22 @@
  */
 public class MyOpenBitSet implements MyBitSet {
 
-    private final OpenBitSet bitSet;
+    private final ProtectedHack bitSet;
 
     public MyOpenBitSet(int no) {
-        bitSet = new OpenBitSet(no);
+        bitSet = new ProtectedHack(no);
+    }
+
+    static class ProtectedHack extends OpenBitSet {
+
+        public ProtectedHack(long numBits) {
+            super(numBits);
+        }
+
+        @Override
+        public int expandingWordNum(long index) {
+            return super.expandingWordNum(index);
+        }
     }
 
     public MyOpenBitSet() {
@@ -45,12 +57,14 @@ public MyOpenBitSet() {
     @Override public String toString() {
         try {
             StringBuilder sb = new StringBuilder();
+            sb.append("{");
             for (DocIdSetIterator iter = bitSet.iterator(); iter.nextDoc() != DocIdSetIterator.NO_MORE_DOCS;) {
-                if (sb.length() != 0)
+                if (sb.length() > 1)
                     sb.append(", ");
 
                 sb.append(iter.docID());
             }
+            sb.append("}");
             return sb.toString();
         } catch (Exception ex) {
             throw new RuntimeException("error constructing bitset string representation", ex);
@@ -66,4 +80,16 @@ public int getCardinality() {
     public void clear() {
         bitSet.clear(0, bitSet.length());
     }
+
+    @Override
+    public void ensureCapacity(int size) {
+        bitSet.expandingWordNum(size);
+        // this does not update the numBits variable and failse if assertationen are on!
+        // bitSet.ensureCapacity((size);
+    }
+
+    @Override
+    public int next(int index) {
+        return bitSet.nextSetBit(index);
+    }
 }

core/src/main/java/de/jetsli/graph/coll/MyTBitSet.java

@@ -54,4 +54,13 @@ public int getCardinality() {
     public void clear() {
         tHash.clear();
     }        
+
+    @Override
+    public void ensureCapacity(int size) {        
+    }
+
+    @Override
+    public int next(int index) {
+        throw new UnsupportedOperationException("Not supported yet.");
+    }
 }

core/src/main/java/de/jetsli/graph/routing/rideshare/DijkstraMultipleAtOnce.java

@@ -15,17 +15,17 @@
  */
 package de.jetsli.graph.routing.rideshare;
 
-import de.jetsli.graph.storage.MemoryGraph;
+import de.jetsli.graph.storage.Graph;
 
 /**
  *
  * @author Peter Karich, info@jetsli.de
  */
 public class DijkstraMultipleAtOnce {
 
-    private MemoryGraph graph;
+    private Graph graph;
 
-    public DijkstraMultipleAtOnce(MemoryGraph graph) {
+    public DijkstraMultipleAtOnce(Graph graph) {
         this.graph = graph;
     }
 

core/src/main/java/de/jetsli/graph/storage/MemoryGraph.java

@@ -64,6 +64,7 @@ private void ensureNodeIndex(int index) {
         if (size <= lons.length)
             return;
 
+        deletedNodes.ensureCapacity(size);
         refToEdges = growArray(refToEdges, size, FACTOR);
         lons = growArray(lons, size, FACTOR);
         lats = growArray(lats, size, FACTOR);

core/src/main/java/de/jetsli/graph/storage/MemoryGraphSafe.java

@@ -20,6 +20,7 @@
 import de.jetsli.graph.util.EdgeIdIterator;
 import de.jetsli.graph.util.Helper;
 import gnu.trove.list.array.TIntArrayList;
+import gnu.trove.map.hash.TIntIntHashMap;
 import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
@@ -79,7 +80,7 @@ public MemoryGraphSafe(String storageDir, int cap, int capEdge) {
             initNodes(cap);
             initEdges(capEdge);
         }
-    }    
+    }
 
     @Override public int getNodes() {
         // readLock.lock();
@@ -124,15 +125,15 @@ private void ensureEdgePointer(int pointer) {
 
     // Use ONLY within a writer lock area
     private void ensureNodeIndex(int index) {
-        if(index < size)
+        if (index < size)
             return;
-                
+
         size = index + 1;
-        if(size <= lats.length)
+        if (size <= lats.length)
             return;
-                
+
         int cap = Math.max(10, Math.round(size * FACTOR));
-        // TODO deletedNodes = copy(deletedNodes, cap);
+        deletedNodes.ensureCapacity(cap);
         lats = Arrays.copyOf(lats, cap);
         lons = Arrays.copyOf(lons, cap);
         // priorities = Arrays.copyOf(priorities, cap);
@@ -337,6 +338,7 @@ public Graph clone() {
         System.arraycopy(lons, 0, g.lons, 0, lons.length);
         System.arraycopy(refToEdges, 0, g.refToEdges, 0, refToEdges.length);
         System.arraycopy(edgesArea, 0, g.edgesArea, 0, edgesArea.length);
+        g.nextEdgePointer = nextEdgePointer;
         g.size = size;
         return g;
     }
@@ -371,14 +373,106 @@ public void optimize() {
         int deleted = deletedNodes.getCardinality();
         if (deleted == 0)
             return;
-
-        logger.info("lengthy operation: optimizing graph");
+
+
+        if (deleted < size / 5)
+            optimizeIfFewDeletes(deleted);
+        else
+            optimizeIfLotsOfDeletes(deleted);
+    }
+
+    /**
+     * This methods moves the last nodes into the deleted nodes, which is much more memory friendly
+     * but probably slower than optimizeIfLotsOfDeletes for many deletes.
+     */
+    void optimizeIfFewDeletes(int deleted) {
+        // Alternative to this method: use smaller segments for nodes and not one big fat java array?
+        //
+        // Prepare edge-update of nodes which are connected to deleted nodes and 
+        // create old- to new-index map.
+        int index = getNodes();
+        int counter = 0;
+        int newIndices[] = new int[deleted];
+        int oldIndices[] = new int[deleted];
+        TIntIntHashMap markedMap = new TIntIntHashMap(deleted, 1.5f, -1, -1);
+        for (int del = deletedNodes.next(0); del >= 0; del = deletedNodes.next(del + 1)) {
+            EdgeIdIterator delEdgesIter = getEdges(del);
+            while (delEdgesIter.next()) {
+                int currNode = delEdgesIter.nodeId();
+                if (deletedNodes.contains(currNode))
+                    continue;
+
+                // remove all edges to the deleted nodes
+                EdgeIdIterator nodesConnectedToDelIter = getEdges(currNode);
+                boolean firstNext = nodesConnectedToDelIter.next();
+                if (firstNext) {
+                    // this forces new edges to be created => TODO only update the edges. do not create new entries
+                    refToEdges[currNode] = -1;
+                    do {
+                        if (!deletedNodes.contains(nodesConnectedToDelIter.nodeId()))
+                            internalAdd(currNode, nodesConnectedToDelIter.nodeId(),
+                                    nodesConnectedToDelIter.distance(), nodesConnectedToDelIter.flags());
+                    } while (nodesConnectedToDelIter.next());
+                }
+            }
+
+            index--;
+            for (; index >= 0; index--) {
+                if (!deletedNodes.contains(index))
+                    break;
+            }
+
+            newIndices[counter] = del;
+            oldIndices[counter] = index;
+            markedMap.put(index, del);
+            counter++;
+        }
+
+        // now move marked nodes into deleted nodes
+        for (int i = 0; i < oldIndices.length; i++) {
+            int oldI = oldIndices[i];
+            int newI = newIndices[i];
+
+            EdgeIdIterator toMoveEdgeIter = getEdges(oldI);
+            while (toMoveEdgeIter.next()) {
+                int movedNewIndex = markedMap.get(toMoveEdgeIter.nodeId());
+                // if node not moved at all (<0) or node is not yet moved use the old index
+                if (movedNewIndex < 0 || movedNewIndex >= newI) {
+                    movedNewIndex = toMoveEdgeIter.nodeId();
+                }
+
+                // update edges of r to use newI instead of oldI
+                EdgeIdIterator updateIter = getEdges(movedNewIndex);
+                boolean firstNext = updateIter.next();
+                if (firstNext) {
+                    // this forces new edges to be created => TODO only update the edges. do not create new entries
+                    refToEdges[movedNewIndex] = -1;
+                    do {
+                        int connNode;
+                        if (updateIter.nodeId() == oldI)
+                            connNode = newI;
+                        else
+                            connNode = updateIter.nodeId();
+                        internalAdd(movedNewIndex, connNode, updateIter.distance(), updateIter.flags());
+                    } while (updateIter.next());
+                }
+            }
+            refToEdges[newI] = refToEdges[oldI];
+            lats[newI] = lats[oldI];
+            lons[newI] = lons[oldI];
+        }
+
+        deletedNodes = null;
+        size -= deleted;
+    }
+
+    /**
+     * This methods creates a new in-memory graph without the specified deleted nodes.
+     */
+    void optimizeIfLotsOfDeletes(int deleted) {
         MemoryGraphSafe inMemGraph = new MemoryGraphSafe(null, getNodes() - deleted, getMaxEdges());
 
-        /**
-         * This methods creates a new in-memory graph without the specified deleted nodes. see
-         * MMapGraph for a near duplicate
-         */
+        // see MMapGraph for a near duplicate         
         int locs = this.getNodes();
         int newNodeId = 0;
         int[] old2NewMap = new int[locs];

core/src/main/java/de/jetsli/graph/util/Helper.java

@@ -15,6 +15,7 @@
  */
 package de.jetsli.graph.util;
 
+import de.jetsli.graph.coll.MyBitSet;
 import java.io.*;
 import java.nio.MappedByteBuffer;
 import java.util.*;

core/src/main/java/de/jetsli/graph/util/TopologicalSorting.java

@@ -17,7 +17,6 @@
 
 import de.jetsli.graph.coll.MyBitSet;
 import de.jetsli.graph.coll.MyOpenBitSet;
-import de.jetsli.graph.storage.MemoryGraph;
 import de.jetsli.graph.storage.Graph;
 import gnu.trove.iterator.TIntIterator;
 import gnu.trove.list.array.TIntArrayList;
@@ -32,7 +31,7 @@ public class TopologicalSorting {
     /**
      * conditions: acyclicGraph and all reachable from 0
      */
-    public TIntArrayList sort(MemoryGraph g) {
+    public TIntArrayList sort(Graph g) {
         final TIntArrayList list = new TIntArrayList();
         if (g.getNodes() == 0)
             return list;

core/src/test/java/de/jetsli/graph/coll/AbstractMyBitSetTest.java

@@ -31,9 +31,37 @@ public void testToString() {
         MyBitSet bs = createBitSet(100);
         bs.add(12);
         bs.add(1);
-        assertEquals("1, 12", bs.toString());
+        assertEquals("{1, 12}", bs.toString());
     }
-
+
+    @Test
+    public void testNext() {
+        MyBitSet bs = createBitSet(100);
+        bs.add(7);
+        bs.add(90);
+        assertEquals(7, bs.next(0));
+        assertEquals(7, bs.next(7));
+        assertEquals(90, bs.next(8));
+        assertEquals(-1, bs.next(91));
+    }
+
+    @Test
+    public void testEnsureCapacity() {
+        MyBitSet bs = createBitSet(8);
+        bs.add(7);
+        try {
+            bs.add(8);
+            assertTrue(false);
+        } catch (Throwable ex) {
+        }
+        bs.ensureCapacity(16);
+        bs.add(8);
+        bs.add(9);
+        assertFalse(bs.contains(6));
+        assertTrue(bs.contains(7));
+        assertTrue(bs.contains(8));
+    }
+
     @Test
     public void testClear() {
         MyBitSet bs = createBitSet(100);