Skip to content

Commit

Permalink
Merge pull request #555 from zod/pbfparser
Browse files Browse the repository at this point in the history 
Replace XML parser with pbfparser
  • Loading branch information
@afischerdev
afischerdev committed May 20, 2023
2 parents 11a9843 + c0245df commit 14b1ece
Show file tree
Hide file tree
Showing 13 changed files with 112 additions and 362 deletions.
5 changes: 3 additions & 2 deletions brouter-map-creator/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@ plugins {
id 'java-library'
}


dependencies {
implementation project(':brouter-codec')
implementation project(':brouter-util')
implementation project(':brouter-expressions')


implementation group: 'org.openstreetmap.osmosis', name: 'osmosis-osm-binary', version: '0.48.3'

testImplementation('junit:junit:4.13.1')
}
20 changes: 12 additions & 8 deletions misc/pbfparser/BPbfBlobDecoder.java → ...va/btools/mapcreator/BPbfBlobDecoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,18 @@
import org.openstreetmap.osmosis.osmbinary.Fileformat;
import org.openstreetmap.osmosis.osmbinary.Osmformat;

import btools.util.LongList;

import java.io.IOException;
import java.util.*;
import java.util.logging.Level;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;

import btools.util.LongList;

/**
* Converts PBF block data into decoded entities ready to be passed into an Osmosis pipeline. This
* class is designed to be passed into a pool of worker threads to allow multi-threaded decoding.
Expand Down Expand Up @@ -82,8 +86,8 @@ private void processOsmHeader(byte[] data) throws InvalidProtocolBufferException

// Build the list of active and unsupported features in the file.
List<String> supportedFeatures = Arrays.asList("OsmSchema-V0.6", "DenseNodes");
List<String> activeFeatures = new ArrayList<String>();
List<String> unsupportedFeatures = new ArrayList<String>();
List<String> activeFeatures = new ArrayList<>();
List<String> unsupportedFeatures = new ArrayList<>();
for (String feature : header.getRequiredFeaturesList()) {
if (supportedFeatures.contains(feature)) {
activeFeatures.add(feature);
Expand All @@ -106,7 +110,7 @@ private Map<String, String> buildTags(List<Integer> keys, List<Integer> values,
Iterator<Integer> keyIterator = keys.iterator();
Iterator<Integer> valueIterator = values.iterator();
if (keyIterator.hasNext()) {
Map<String, String> tags = new HashMap<String, String>();
Map<String, String> tags = new HashMap<>();
while (keyIterator.hasNext()) {
String key = fieldDecoder.decodeString(keyIterator.next());
String value = fieldDecoder.decodeString(valueIterator.next());
Expand Down Expand Up @@ -155,7 +159,7 @@ private void processNodes(Osmformat.DenseNodes nodes, BPbfFieldDecoder fieldDeco
int valueIndex = keysValuesIterator.next();

if (tags == null) {
tags = new HashMap<String, String>();
tags = new HashMap<>();
}

tags.put(fieldDecoder.decodeString(keyIndex), fieldDecoder.decodeString(valueIndex));
Expand Down
0 misc/pbfparser/BPbfFieldDecoder.java → ...a/btools/mapcreator/BPbfFieldDecoder.java
View file
File renamed without changes.
234 changes: 90 additions & 144 deletions brouter-map-creator/src/main/java/btools/mapcreator/OsmParser.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,17 @@
package btools.mapcreator;

import org.openstreetmap.osmosis.osmbinary.Fileformat;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.zip.GZIPInputStream;
import java.util.HashMap;
import java.util.Map;

import btools.util.LongList;

/**
* Parser for OSM data
Expand All @@ -22,179 +29,118 @@ public void readMap(File mapFile,
NodeListener nListener,
WayListener wListener,
RelationListener rListener) throws Exception {

this.nListener = nListener;
this.wListener = wListener;
this.rListener = rListener;

if (mapFile == null) {
_br = new BufferedReader(new InputStreamReader(System.in));
} else {
if (mapFile.getName().endsWith(".gz")) {
_br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(mapFile))));
} else {
_br = new BufferedReader(new InputStreamReader(new FileInputStream(mapFile)));
}
}
System.out.println("*** PBF Parsing: " + mapFile);

for (; ; ) {
String line = _br.readLine();
if (line == null) break;
// once more for testing
int rawBlobCount = 0;

if (checkNode(line)) continue;
if (checkWay(line)) continue;
if (checkRelation(line)) continue;
if (checkChangeset(line)) continue;
}
long bytesRead = 0L;
Boolean avoidMapPolling = Boolean.getBoolean("avoidMapPolling");

if (mapFile != null) {
_br.close();
if (!avoidMapPolling) {
// wait for file to become available
while (!mapFile.exists()) {
System.out.println("--- waiting for " + mapFile + " to become available");
Thread.sleep(10000);
}
}
}

long currentSize = mapFile.length();
long currentSizeTime = System.currentTimeMillis();

DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(mapFile)));


private boolean checkNode(String line) throws Exception {
int idx0 = line.indexOf("<node id=\"");
if (idx0 < 0) return false;
idx0 += 10;
int idx1 = line.indexOf('"', idx0);

long nodeId = Long.parseLong(line.substring(idx0, idx1));

int idx2 = line.indexOf(" lat=\"");
if (idx2 < 0) return false;
idx2 += 6;
int idx3 = line.indexOf('"', idx2);
double lat = Double.parseDouble(line.substring(idx2, idx3));
int idx4 = line.indexOf(" lon=\"");
if (idx4 < 0) return false;
idx4 += 6;
int idx5 = line.indexOf('"', idx4);
double lon = Double.parseDouble(line.substring(idx4, idx5));

NodeData n = new NodeData(nodeId, lon, lat);

if (!line.endsWith("/>")) {
// read additional tags
for (; ; ) {
String l2 = _br.readLine();
if (l2 == null) return false;

int i2;
if ((i2 = l2.indexOf("<tag k=\"")) >= 0) { // property-tag
i2 += 8;
int ri2 = l2.indexOf('"', i2);
String key = l2.substring(i2, ri2);
i2 = l2.indexOf(" v=\"", ri2);
if (i2 >= 0) {
i2 += 4;
int ri3 = l2.indexOf('"', i2);
String value = l2.substring(i2, ri3);

n.putTag(key, value);
for (; ; ) {
if (!avoidMapPolling) {
// continue reading if either more then a 100 MB unread, or the current-size is known for more than 2 Minutes
while (currentSize - bytesRead < 100000000L) {
long newSize = mapFile.length();
if (newSize != currentSize) {
currentSize = newSize;
currentSizeTime = System.currentTimeMillis();
} else if (System.currentTimeMillis() - currentSizeTime > 120000) {
break;
}
if (currentSize - bytesRead < 100000000L) {
System.out.println("--- waiting for more data, currentSize=" + currentSize + " bytesRead=" + bytesRead);
Thread.sleep(10000);
}
} else if (l2.indexOf("</node>") >= 0) { // end-tag
break;
}
}
}
nListener.nextNode(n);
return true;
}

int headerLength;
try {
headerLength = dis.readInt();
bytesRead += 4;
} catch (EOFException e) {
break;
}

private boolean checkWay(String line) throws Exception {
int idx0 = line.indexOf("<way id=\"");
if (idx0 < 0) return false;
byte[] headerBuffer = new byte[headerLength];
dis.readFully(headerBuffer);
bytesRead += headerLength;
Fileformat.BlobHeader blobHeader = Fileformat.BlobHeader.parseFrom(headerBuffer);

idx0 += 9;
int idx1 = line.indexOf('"', idx0);
long id = Long.parseLong(line.substring(idx0, idx1));
byte[] blobData = new byte[blobHeader.getDatasize()];
dis.readFully(blobData);
bytesRead += blobData.length;

WayData w = new WayData(id);
new BPbfBlobDecoder(blobHeader.getType(), blobData, this).process();

// read the nodes
for (; ; ) {
String l2 = _br.readLine();
if (l2 == null) return false;

int i2;
if ((i2 = l2.indexOf("<nd ref=\"")) >= 0) { // node reference
i2 += 9;
int ri2 = l2.indexOf('"', i2);
long nid = Long.parseLong(l2.substring(i2, ri2));
w.nodes.add(nid);
} else if ((i2 = l2.indexOf("<tag k=\"")) >= 0) { // property-tag
i2 += 8;
int ri2 = l2.indexOf('"', i2);
String key = l2.substring(i2, ri2);
i2 = l2.indexOf(" v=\"", ri2);
if (i2 >= 0) {
i2 += 4;
int ri3 = l2.indexOf('"', i2);
String value = l2.substring(i2, ri3);
w.putTag(key, value);
}
} else if (l2.indexOf("</way>") >= 0) { // end-tag
break;
}
rawBlobCount++;
}
wListener.nextWay(w);
return true;
dis.close();
System.out.println("read raw blobs: " + rawBlobCount);
}

private boolean checkChangeset(String line) throws Exception {
int idx0 = line.indexOf("<changeset id=\"");
if (idx0 < 0) return false;

if (!line.endsWith("/>")) {
int loopcheck = 0;
for (; ; ) {
String l2 = _br.readLine();
if (l2.indexOf("</changeset>") >= 0 || ++loopcheck > 10000) break;
}
public void addNode(long nid, Map<String, String> tags, double lat, double lon) {
NodeData n = new NodeData(nid, lon, lat);
n.setTags((HashMap<String, String>) tags);
try {
nListener.nextNode(n);
} catch (Exception e) {
throw new RuntimeException("error writing node: " + e);
}
return true;
}

private boolean checkRelation(String line) throws Exception {
int idx0 = line.indexOf("<relation id=\"");
if (idx0 < 0) return false;
public void addWay(long wid, Map<String, String> tags, LongList nodes) {
WayData w = new WayData(wid, nodes);
w.setTags((HashMap<String, String>) tags);

idx0 += 14;
int idx1 = line.indexOf('"', idx0);
long rid = Long.parseLong(line.substring(idx0, idx1));
try {
wListener.nextWay(w);
} catch (Exception e) {
throw new RuntimeException("error writing way: " + e);
}
}

RelationData r = new RelationData(rid);
public void addRelation(long rid, Map<String, String> tags, LongList wayIds, LongList fromWid, LongList toWid, LongList viaNid) {
RelationData r = new RelationData(rid, wayIds);
r.setTags((HashMap<String, String>) tags);

// read the nodes
for (; ; ) {
String l2 = _br.readLine();
if (l2 == null) return false;

int i2;
if ((i2 = l2.indexOf("<member type=\"way\" ref=\"")) >= 0) { // node reference
i2 += 24;
int ri2 = l2.indexOf('"', i2);
long wid = Long.parseLong(l2.substring(i2, ri2));
r.ways.add(wid);
} else if ((i2 = l2.indexOf("<tag k=\"")) >= 0) { // property-tag
i2 += 8;
int ri2 = l2.indexOf('"', i2);
String key = l2.substring(i2, ri2);
i2 = l2.indexOf(" v=\"", ri2);
if (i2 >= 0) {
i2 += 4;
int ri3 = l2.indexOf('"', i2);
String value = l2.substring(i2, ri3);
r.putTag(key, value);
try {
rListener.nextRelation(r);
if (fromWid == null || toWid == null || viaNid == null || viaNid.size() != 1) {
// dummy-TR for each viaNid
for (int vi = 0; vi < (viaNid == null ? 0 : viaNid.size()); vi++) {
rListener.nextRestriction(r, 0L, 0L, viaNid.get(vi));
}
return;
}
for (int fi = 0; fi < fromWid.size(); fi++) {
for (int ti = 0; ti < toWid.size(); ti++) {
rListener.nextRestriction(r, fromWid.get(fi), toWid.get(ti), viaNid.get(0));
}
} else if (l2.indexOf("</relation>") >= 0) { // end-tag
break;
}
} catch (Exception e) {
throw new RuntimeException("error writing relation", e);
}
rListener.nextRelation(r);
return true;
}

}
6 changes: 4 additions & 2 deletions brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@
public class MapcreatorTest {
@Test
public void mapcreatorTest() throws Exception {
URL mapurl = this.getClass().getResource("/dreieich.osm.gz");
Assert.assertNotNull("test-osm-map dreieich.osm not found", mapurl);
System.setProperty("avoidMapPolling", "true");

URL mapurl = this.getClass().getResource("/dreieich.pbf");
Assert.assertNotNull("test-osm-map dreieich.pbf not found", mapurl);
File mapFile = new File(mapurl.getFile());
File workingDir = mapFile.getParentFile();
File profileDir = new File(workingDir, "/../../../../misc/profiles2");
Expand Down
Binary file added brouter-map-creator/src/test/resources/dreieich.pbf
View file
Binary file not shown.
Loading

0 comments on commit 14b1ece

Please sign in to comment.