Permalink
Browse files

made creating and searching location2id index a lot faster (normed di…

…stance+different bitset for xsearch+different limit to search in fill empty); refactored calcdistance
  • Loading branch information...
1 parent e8f5401 commit 50f84a379ec7d754d56ac0d1d5509d4f40e53ff9 @karussell committed Jun 14, 2012
Showing with 286 additions and 151 deletions.
  1. +18 −0 core/check-swap-usage.sh
  2. +13 −3 core/nbactions.xml
  3. +1 −1 core/src/main/java/de/jetsli/graph/geohash/SpatialHashtable.java
  4. +3 −2 core/src/main/java/de/jetsli/graph/reader/LuceneStorage.java
  5. +34 −17 core/src/main/java/de/jetsli/graph/reader/MMapGraphStorage.java
  6. +3 −2 core/src/main/java/de/jetsli/graph/reader/Neo4JStorage.java
  7. +64 −35 core/src/main/java/de/jetsli/graph/reader/OSMReaderRouting.java
  8. +4 −2 core/src/main/java/de/jetsli/graph/reader/Storage.java
  9. +1 −1 core/src/main/java/de/jetsli/graph/storage/Location2IDFullIndex.java
  10. +34 −19 core/src/main/java/de/jetsli/graph/storage/Location2IDQuadtree.java
  11. +18 −12 core/src/main/java/de/jetsli/graph/storage/MMapGraph.java
  12. +1 −1 core/src/main/java/de/jetsli/graph/trees/QuadTreeSimple.java
  13. +40 −35 core/src/main/java/de/jetsli/graph/ui/MiniGraphUI.java
  14. +1 −1 core/src/main/java/de/jetsli/graph/{reader → util}/CalcDistance.java
  15. +5 −1 core/src/main/java/de/jetsli/graph/util/XFirstSearch.java
  16. +1 −1 core/src/main/java/de/jetsli/graph/util/shapes/Circle.java
  17. +1 −1 core/src/test/java/de/jetsli/graph/geohash/SpatialKeyAlgoTest.java
  18. +39 −11 core/src/test/java/de/jetsli/graph/reader/OSMReaderRoutingTest.java
  19. +1 −1 core/src/test/java/de/jetsli/graph/storage/Location2IDQuadtreeTest.java
  20. +1 −2 core/src/test/java/de/jetsli/graph/{reader → util}/CalcDistanceTest.java
  21. +1 −1 core/src/test/java/de/jetsli/graph/util/shapes/BBoxTest.java
  22. +1 −1 core/src/test/resources/de/jetsli/graph/reader/test1.xml
  23. +1 −1 perf-comparison/src/main/java/de/jetsli/quadtreecomparison/SimpleArray.java
@@ -0,0 +1,18 @@
+#!/bin/bash
+# Get current swap usage for all running processes
+# Erik Ljungstrom 27/05/2011
+SUM=0
+OVERALL=0
+for DIR in `find /proc/ -maxdepth 1 -type d | egrep "^/proc/[0-9]"` ; do
+PID=`echo $DIR | cut -d / -f 3`
+PROGNAME=`ps -p $PID -o comm --no-headers`
+for SWAP in `grep Swap $DIR/smaps 2>/dev/null| awk '{ print $2 }'`
+do
+let SUM=$SUM+$SWAP
+done
+echo "PID=$PID - Swap used: $SUM - ($PROGNAME )"
+let OVERALL=$OVERALL+$SUM
+SUM=0
+
+done
+echo "Overall swap used: $OVERALL"
View
@@ -18,7 +18,7 @@
</goals>
<properties>
<exec.classpathScope>runtime</exec.classpathScope>
- <exec.args>-Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.ui.MiniGraphUI /home/peterk/unterfranken.osm</exec.args>
+ <exec.args>-Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.ui.MiniGraphUI /media/SAMSUNG/germany.osm</exec.args>
<exec.executable>java</exec.executable>
</properties>
</action>
@@ -30,7 +30,7 @@
</goals>
<properties>
<exec.classpathScope>runtime</exec.classpathScope>
- <exec.args>-Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.ui.MiniGraphUI /home/peterk/unterfranken.osm</exec.args>
+ <exec.args>-Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.ui.MiniGraphUI /media/SAMSUNG/germany.osm</exec.args>
<jpda.listen>true</jpda.listen>
<exec.executable>java</exec.executable>
</properties>
@@ -42,9 +42,19 @@
<goal>org.codehaus.mojo:exec-maven-plugin:1.1.1:exec</goal>
</goals>
<properties>
- <exec.args>${profiler.args} -Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.ui.MiniGraphUI /home/peterk/unterfranken.osm</exec.args>
+ <exec.args>${profiler.args} -Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.ui.MiniGraphUI /media/SAMSUNG/germany.osm</exec.args>
<profiler.action>profile</profiler.action>
<exec.executable>${profiler.java}</exec.executable>
</properties>
</action>
+ <action>
+ <actionName>CUSTOM-assembly</actionName>
+ <displayName>assembly</displayName>
+ <goals>
+ <goal>assembly:single</goal>
+ </goals>
+ <properties>
+ <skipTests>true</skipTests>
+ </properties>
+ </action>
</actions>
@@ -18,7 +18,7 @@
import de.genvlin.core.data.*;
import de.genvlin.gui.plot.GPlotPanel;
import de.jetsli.graph.geohash.SpatialHashtable.BucketOverflowLoop;
-import de.jetsli.graph.reader.CalcDistance;
+import de.jetsli.graph.util.CalcDistance;
import de.jetsli.graph.reader.OSMReaderRouting;
import de.jetsli.graph.storage.Graph;
import de.jetsli.graph.trees.*;
@@ -15,6 +15,7 @@
*/
package de.jetsli.graph.reader;
+import de.jetsli.graph.util.CalcDistance;
import de.jetsli.graph.util.Helper;
import java.io.File;
import org.apache.lucene.analysis.KeywordAnalyzer;
@@ -108,12 +109,12 @@ public int getNodes() {
}
@Override
- public void setHasEdges(int osmId) {
+ public void setHasHighways(int osmId, boolean isHighway) {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
- public boolean hasEdges(int osmId) {
+ public boolean hasHighways(int osmId) {
throw new UnsupportedOperationException("Not supported yet.");
}
@@ -15,8 +15,10 @@
*/
package de.jetsli.graph.reader;
+import de.jetsli.graph.util.CalcDistance;
import de.jetsli.graph.storage.Graph;
import de.jetsli.graph.storage.MMapGraph;
+import de.jetsli.graph.util.StopWatch;
import gnu.trove.map.hash.TIntIntHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,25 +32,19 @@
private final Logger logger = LoggerFactory.getLogger(getClass());
private MMapGraph g;
private TIntIntHashMap osmIdToIndexMap;
- private final int size;
private final String file;
- public MMapGraphStorage(int size) {
- this(null, size);
- }
-
public MMapGraphStorage(String file, int size) {
- this.size = size;
this.file = file;
- osmIdToIndexMap = new TIntIntHashMap(size, 1.5f, -1, -1);
+ osmIdToIndexMap = new TIntIntHashMap(size, 1.4f, -1, -1);
}
-
+
@Override
public boolean loadExisting() {
g = new MMapGraph(file, -1);
return g.loadExisting();
}
-
+
@Override
public void createNew() {
g = new MMapGraph(file, osmIdToIndexMap.size());
@@ -62,28 +58,46 @@ public boolean addNode(int osmId, double lat, double lon) {
return true;
}
int counter = 0;
+ StopWatch sw1 = new StopWatch();
+ StopWatch sw2 = new StopWatch();
@Override
public boolean addEdge(int nodeIdFrom, int nodeIdTo, boolean reverse, CalcDistance callback) {
int fromIndex = osmIdToIndexMap.get(nodeIdFrom);
+ if (fromIndex == -10) {
+ logger.warn("fromIndex is unresolved:" + nodeIdFrom + " to was:" + nodeIdTo);
+ return false;
+ }
int toIndex = osmIdToIndexMap.get(nodeIdTo);
+ if (toIndex == -10) {
+ logger.warn("toIndex is unresolved:" + nodeIdTo + " from was:" + nodeIdFrom);
+ return false;
+ }
+
if (fromIndex == osmIdToIndexMap.getNoEntryValue() || toIndex == osmIdToIndexMap.getNoEntryValue())
return false;
try {
+ sw2.start();
double laf = g.getLatitude(fromIndex);
double lof = g.getLongitude(fromIndex);
double lat = g.getLatitude(toIndex);
double lot = g.getLongitude(toIndex);
+ sw2.stop();
+
+ sw1.start();
double dist = callback.calcDistKm(laf, lof, lat, lot);
+ sw1.stop();
if (dist <= 0) {
- counter++;
- if (counter % 10000 == 0)
- logger.info(counter + " - distances negative or zero. E.g. " + fromIndex + " (" + laf + ", " + lof + ")->"
- + toIndex + "(" + lat + ", " + lot + ") :" + dist);
+ logger.info(counter + " - distances negative or zero. " + fromIndex + " (" + laf + ", " + lof + ")->"
+ + toIndex + "(" + lat + ", " + lot + ") :" + dist);
return false;
}
+
g.edge(fromIndex, toIndex, dist, reverse);
+ counter++;
+ if (counter % 10000 == 0)
+ logger.info(counter + " g.edge:" + sw1.toString() + " sw2:" + sw2.toString());
return true;
} catch (Exception ex) {
logger.error("Problem with " + fromIndex + "->" + toIndex + " osm:" + nodeIdFrom + "->" + nodeIdTo, ex);
@@ -111,16 +125,19 @@ public void flush() {
@Override
public int getNodes() {
- return g.getLocations();
+ return osmIdToIndexMap.size();
}
@Override
- public void setHasEdges(int osmId) {
- osmIdToIndexMap.put(osmId, -10);
+ public void setHasHighways(int osmId, boolean isHighway) {
+ if (isHighway)
+ osmIdToIndexMap.put(osmId, -10);
+ else
+ osmIdToIndexMap.remove(osmId);
}
@Override
- public boolean hasEdges(int osmId) {
+ public boolean hasHighways(int osmId) {
return osmIdToIndexMap.get(osmId) == -10;
}
}
@@ -15,6 +15,7 @@
*/
package de.jetsli.graph.reader;
+import de.jetsli.graph.util.CalcDistance;
import de.jetsli.graph.storage.DistEntry;
import de.jetsli.graph.util.Helper;
import java.io.File;
@@ -41,12 +42,12 @@
private static final Logger logger = LoggerFactory.getLogger(Neo4JStorage.class);
@Override
- public void setHasEdges(int osmId) {
+ public void setHasHighways(int osmId, boolean isHighway) {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
- public boolean hasEdges(int osmId) {
+ public boolean hasHighways(int osmId) {
throw new UnsupportedOperationException("Not supported yet.");
}
Oops, something went wrong.

0 comments on commit 50f84a3

Please sign in to comment.