basic vanilla extract loader/server in Java

src/main/java/org/opentripplanner/osm/NodeTracker.java

@@ -12,6 +12,9 @@
  * Each block is 64 64-bit longs. At 512 bytes long, it tracks 4096 IDs.
  * Node numbers in OSM tend to be contiguous so maybe the blocks should be bigger.
  * Disabled debug statements have no measurable effect on speed (verified).
+ *
+ * MapDB TreeSets are much faster than MapDB HashSets, but in-memory NodeTrackers are
+ * much faster than MapDB TreeSets.
  */
 public class NodeTracker {
 

src/main/java/org/opentripplanner/osm/OSM.java

@@ -1,15 +1,16 @@
 package org.opentripplanner.osm;
 
-import java.io.File;
-import java.util.Map;
-
+import com.google.common.collect.Maps;
+import com.vividsolutions.jts.geom.Envelope;
 import org.mapdb.DB;
 import org.mapdb.DBMaker;
+import org.mapdb.Fun.Tuple3;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Maps;
-import com.vividsolutions.jts.geom.Envelope;
+import java.io.File;
+import java.util.Map;
+import java.util.NavigableSet;
 
 /**
  * OTP representation of a subset of OpenStreetMap. One or more PBF files can be loaded into this
@@ -22,46 +23,47 @@ public class OSM {
     public Map<Long, Node> nodes;
     public Map<Long, Way> ways;
     public Map<Long, Relation> relations;
+    public NavigableSet<Tuple3<Integer, Integer, Long>> index; // (x, y, wayId)
 
-    /** The nodes which are referenced more than once by ways in this OSM. */
-    public NodeTracker intersections;
-
     /** The MapDB backing this OSM, if any. */
     DB db = null; // db.close(); ?
-
-    public OSM(boolean diskBacked) {
+
+    /** If diskPath is null, OSM will be loaded into memory. */
+    public OSM(String diskPath) {
         // Using DB TreeMaps is observed not to be slower than memory.
         // HashMaps are both bigger and slower.
         // It lets you run in 400MB instead of a few GB.
-        if (diskBacked) {
-            LOG.info("OSM backed by temporary file.");
-            DB db = DBMaker.newFileDB(new File("/mnt/ssd2/vex/osmdb")) // use newMemoryDB for higher speed?
+        if (diskPath != null) {
+            LOG.info("OSM backed by file.");
+            DB db = DBMaker.newFileDB(new File(diskPath))
                     .transactionDisable()
                     .asyncWriteEnable()
                     .compressionEnable()
                     .make();
             nodes = db.getTreeMap("nodes");
             ways = db.getTreeMap("ways");
             relations = db.getTreeMap("relations");
+            index = db.getTreeSet("spatial_index");
         } else {
             // In-memory version
+            // use newMemoryDB for higher speed?
             nodes = Maps.newHashMap();
             ways = Maps.newHashMap();
-            relations = Maps.newHashMap();            
+            relations = Maps.newHashMap();
         }
     }
 
     // boolean filterTags
     public static OSM fromPBF(String pbfFile) {
         LOG.info("Reading entire PBF file '{}'", pbfFile);
-        FullParser fp = new FullParser();
-        fp.parse(pbfFile);
-        return fp.osm;
+        Parser parser = new Parser(null);
+        parser.parse(pbfFile);
+        return parser.osm;
     }
 
     public static OSM fromPBF(String pbfFile, Envelope env) {
         LOG.info("Reading PBF file '{}' filtering with envelope {}", pbfFile, env);
-        OSM osm = new OSM(true);
+        OSM osm = new OSM("/var/vex/osm");
         LOG.info("Finding nodes within the bounding geometry.");
         NodeGeomFilter ngf = new NodeGeomFilter(env);
         ngf.parse(pbfFile);
@@ -74,26 +76,7 @@ public static OSM fromPBF(String pbfFile, Envelope env) {
         LOG.info("Loading relations (which ones?)");
         return osm;
     }
-
-    /**
-     * Find nodes referenced by more than one way. NodeTracker intersections will be null until this
-     * is called. MapDB TreeSets are much faster than MapDB HashSets, but in-memory NodeTrackers are
-     * much faster than MapDB TreeSets.
-     */
-    public void findIntersections() {
-        LOG.info("Finding intersections.");
-        intersections = new NodeTracker();
-        NodeTracker referenced = new NodeTracker();
-        for (Way way : ways.values()) {
-            for (long nid : way.nodes) {
-                if (referenced.contains(nid)) {
-                    intersections.add(nid); // seen more than once
-                } else {
-                    referenced.add(nid); // seen for the first time
-                }
-            }
-        }
-        LOG.info("Done finding intersections.");        
-    }
+
+
 
 }

src/main/java/org/opentripplanner/osm/OSMMain.java

@@ -3,7 +3,6 @@
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
-import com.google.protobuf.ByteString;
 import com.vividsolutions.jts.geom.Envelope;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,9 +37,6 @@
 public class OSMMain {
 
     private static final Logger LOG = LoggerFactory.getLogger(OSMMain.class);
-    //static final String INPUT = "/var/otp/graphs/ny/new-york-latest.osm.pbf";
-    //static final String INPUT = "/var/otp/graphs/nl/netherlands-latest.osm.pbf";
-    //static final String INPUT = "/var/otp/graphs/trimet/portland.osm.pbf";
     static final Envelope ENV = new Envelope(4.4, 5.5, 52.2, 53.3);
 
     public static void main(String[] args) {
@@ -85,7 +81,7 @@ public static void analyzeTags(OSM osm) {
     }
 
     public static List<Edge> makeEdges(OSM osm) {
-        osm.findIntersections();
+//        osm.findIntersections();
         LOG.info("Making edges from Ways.");
         List<Edge> edges = Lists.newArrayList();
         for (Entry<Long, Way> e : osm.ways.entrySet()) {
@@ -98,7 +94,7 @@ public static List<Edge> makeEdges(OSM osm) {
                 if (n == (way.nodes.length - 1)) {
                     edge.to = node;
                     edges.add(edge);
-                } else if (osm.intersections.contains(node)) {
+//                } else if (osm.intersections.contains(node)) {
                     edge.to = node;
                     edges.add(edge);
                     edge = new Edge();

src/main/java/org/opentripplanner/osm/OSMTextOutput.java

@@ -0,0 +1,61 @@
+package org.opentripplanner.osm;
+
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.Locale;
+
+/**
+ * Write OSM ways and nodes out to a simple human-readable text format.
+ */
+public class OSMTextOutput {
+
+    OutputStream out;
+    PrintStream print;
+    OSM osm;
+    NodeTracker nodesSeen;
+
+    public OSMTextOutput (OutputStream out, OSM osm) {
+        this.osm = osm;
+        this.out = out;
+        this.print = new PrintStream(out);
+        nodesSeen = new NodeTracker();
+    }
+
+    public void printWay(long wayId) {
+        Way way = osm.ways.get(wayId);
+        print.print("W ");
+        print.print(wayId);
+        print.print(' ');
+        print.print(way.tags);
+        print.print('\n');
+        for (long nodeId : way.nodes) {
+            if (nodesSeen.contains(nodeId)) {
+                printNodeFull(nodeId);
+//                printNodeRef(nodeId);
+            } else {
+                printNodeFull(nodeId);
+                nodesSeen.add(nodeId);
+            }
+        }
+    }
+
+    public void printNodeRef(long nodeId) {
+        print.print("N ");
+        print.print(nodeId);
+        print.print('\n');
+    }
+
+    public void printNodeFull(long nodeId) {
+        print.print("N ");
+        print.print(nodeId);
+        print.print(' ');
+        Node node = osm.nodes.get(nodeId);
+        print.printf(Locale.US, "%2.6f", node.lat);
+        print.print(' ');
+        print.printf(Locale.US, "%3.6f", node.lon);
+        print.print(' ');
+        print.print(node.tags);
+        print.print('\n');
+    }
+
+}

src/main/java/org/opentripplanner/osm/Parser.java

@@ -13,19 +13,16 @@
  * not, see <http://www.gnu.org/licenses/>.
  */
 
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.List;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
+import crosby.binary.BinaryParser;
+import crosby.binary.Osmformat;
+import crosby.binary.file.BlockInputStream;
 import org.opentripplanner.osm.Relation.Type;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import crosby.binary.BinaryParser;
-import crosby.binary.Osmformat;
-import crosby.binary.file.BlockInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.List;
 
 /**
  * Parser for the OpenStreetMap PBF Format. Implements callbacks for the crosby.binary OSMPBF
@@ -38,7 +35,7 @@
  * parseDense, etc. rather than the corresponding handle* methods to avoid ever converting the
  * low-level PBF objects into objects using OTP's internal OSM model.
  */
-public abstract class Parser extends BinaryParser {
+public class Parser extends BinaryParser {
 
     protected static final Logger LOG = LoggerFactory.getLogger(Parser.class);
 
@@ -47,7 +44,15 @@ public abstract class Parser extends BinaryParser {
     // private Map<String, String> stringTable = new HashMap<String, String>();    
     long nodeCount = 0;
     long wayCount = 0;
-
+
+    public Parser () {
+        osm = new OSM(null);
+    }
+
+    public Parser (String diskPath) {
+        osm = new OSM(diskPath);
+    }
+
     private static final String[] retainKeys = new String[] {
         "highway", "parking", "bicycle"
     };
@@ -61,7 +66,7 @@ private boolean retainTag(String key) {
         // Not storing elements that lack interesting tags
         // reduces size by 80%.
         // return true;
-        return false;
+        return true;
     }
 
     // Load ways first, then skip loading all nodes which are not tracked.