-
Notifications
You must be signed in to change notification settings - Fork 12
/
VexFormatCodec.java
342 lines (304 loc) · 11.4 KB
/
VexFormatCodec.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
package com.conveyal.osmlib;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
/**
* Decode (or encode) a stream of VEX data into an OTP OSM data store.
* This is a sort of data pump between an OTP OSM store and and InputStream / OutputStream.
* Neither threadsafe nor reentrant!
*/
public class VexFormatCodec {
public static final String HEADER = "VEXFMT";
public static final int VEX_NODE = 0;
public static final int VEX_WAY = 1;
public static final int VEX_RELATION = 2;
public static final int VEX_NONE = 3;
/* The input stream providing decompressed VEX format. */
private CodedInputStream vin;
/* The output sink for uncompressed VEX format. */
private CodedOutputStream vout;
/* Persistent values for delta coding. */
private long id, ref, prevId, prevRef, prevFixedLat, prevFixedLon;
private double lat, lon;
/* The OSM data store to stuff the elements into. */
private OSM osm;
/* Writers demonstrate format, readers consume it. */
public void writeVex(OSM osm, OutputStream gzVexStream) throws IOException {
this.vout = CodedOutputStream.newInstance(new GZIPOutputStream(gzVexStream));
this.osm = osm;
vout.writeRawBytes(HEADER.getBytes());
long nEntities = 0;
nEntities += writeNodeBlock();
nEntities += writeWayBlock();
nEntities += writeRelationBlock();
// Empty block of type NONE indicates end of blocks
beginWriteBlock(VEX_NONE);
endWriteBlock(0);
// Total number of blocks written as a "checksum"
vout.writeUInt64NoTag(nEntities);
}
public void readVex(InputStream gzVexStream, OSM osm) throws IOException {
this.vin = CodedInputStream.newInstance(new GZIPInputStream(gzVexStream));
this.osm = osm;
byte[] header = vin.readRawBytes(HEADER.length());
if ( ! Arrays.equals(header, HEADER.getBytes())) {
throw new IOException("Corrupt header.");
}
boolean done = false;
long nBlocks = 0;
while ( ! done) {
done = readBlock();
nBlocks += 1;
}
long expectedBlocks = vin.readUInt64();
if (expectedBlocks != nBlocks) {
throw new IOException("Did not read the expected number of blocks.");
}
}
private void beginWriteBlock(int etype) throws IOException {
prevId = prevRef = 0;
prevFixedLat = prevFixedLon = 0;
vout.writeUInt32NoTag(etype);
}
/* @param n - the number of entities that were written in this block. */
private void endWriteBlock(int n) throws IOException {
vout.writeSInt64NoTag(0L);
vout.writeUInt32NoTag(n);
}
/** Note that the MapDB TreeMap is ordered, so we are writing out the nodes in ID order! */
private int writeNodeBlock() throws IOException {
beginWriteBlock(VEX_NODE);
int n = 0;
for (Map.Entry<Long, Node> entry : osm.nodes.entrySet()) {
writeNode(entry.getKey(), entry.getValue());
n++;
}
endWriteBlock(n);
return n;
}
private int writeWayBlock() throws IOException {
beginWriteBlock(VEX_WAY);
int n = 0;
for (Map.Entry<Long, Way> entry : osm.ways.entrySet()) {
writeWay(entry.getKey(), entry.getValue());
n++;
}
endWriteBlock(n);
return n;
}
private int writeRelationBlock() throws IOException {
beginWriteBlock(VEX_RELATION);
int n = 0;
for (Map.Entry<Long, Relation> entry : osm.relations.entrySet()) {
writeRelation(entry.getKey(), entry.getValue());
n++;
}
endWriteBlock(n);
return n;
}
/** Write the first elements common to all OSM entities: ID and tags. */
private void writeTagged(long id, OSMEntity tagged) throws IOException {
vout.writeSInt64NoTag(id - prevId);
prevId = id;
writeTags(tagged);
}
private void writeNode(long id, Node node) throws IOException {
writeTagged(id, node);
// plain ints should be fine rather than longs:
// 2**31 = 2147483648
// 180e7 = 1800000000.0
long fixedLat = (long) (node.fixedLat);
long fixedLon = (long) (node.fixedLon);
vout.writeSInt64NoTag(prevFixedLat - fixedLat);
vout.writeSInt64NoTag(prevFixedLon - fixedLon);
prevFixedLat = fixedLat;
prevFixedLon = fixedLon;
}
private void writeWay(long id, Way way) throws IOException {
writeTagged(id, way);
vout.writeUInt32NoTag(way.nodes.length);
for (long ref : way.nodes) {
vout.writeSInt64NoTag(prevRef - ref);
prevRef = ref;
}
}
private void writeRelation(long id, Relation relation) throws IOException {
writeTagged(id, relation);
vout.writeUInt32NoTag(relation.members.size());
for (Relation.Member member : relation.members) {
vout.writeSInt64NoTag(member.id);
vout.writeUInt32NoTag(member.type.ordinal()); // FIXME bad, assign specific numbers
vout.writeStringNoTag(member.role);
}
}
public void writeTags (OSMEntity tagged) throws IOException {
List<OSMEntity.Tag> tags = tagged.tags;
// TODO This could stand a little more abstraction, like List<Tag> getTags()
if (tagged.tags == null) {
vout.writeUInt32NoTag(0);
} else {
vout.writeUInt32NoTag(tags.size());
for (OSMEntity.Tag tag : tagged.tags) {
if (tag.value == null) tag.value = "";
vout.writeStringNoTag(tag.key);
vout.writeStringNoTag(tag.value);
}
}
}
public boolean readBlock() throws IOException {
// Reset delta coding fields
lat = lon = id = ref = 0;
int blockType = vin.readUInt32();
if (blockType == VEX_NONE) return true; // NONE block indicates end of file
boolean blockEnd = false;
int nRead = 0;
while ( ! blockEnd) {
switch (blockType) {
case VEX_NODE:
blockEnd = readNode();
break;
case VEX_WAY:
blockEnd = readWay();
break;
case VEX_RELATION:
blockEnd = readRelation();
break;
}
nRead += 1;
}
if (vin.readUInt32() != nRead) {
throw new IOException("Block length mismatch.");
}
return false;
}
public List<OSMEntity.Tag> readTags() throws IOException {
OSMEntity tagged = new Node();
int nTags = vin.readUInt32();
for (int i = 0; i < nTags; i++) {
String key = vin.readString();
String val = vin.readString();
tagged.addTag(key, val);
}
return tagged.tags;
}
public boolean readNode() throws IOException {
/* Create a new instance each time because we don't know if this is going in a MapDB or a normal Map. */
Node node = new Node();
long idDelta = vin.readSInt64();
id += idDelta;
if (idDelta == 0) return true;
node.tags = readTags();
lat += vin.readSInt64() * 1000000d;
lon += vin.readSInt64() * 1000000d;
node.setLatLon(lat, lon);
osm.nodes.put(id, node);
return false;
}
public boolean readWay() throws IOException {
/* Create a new instance each time because we don't know if this is going in a MapDB or a normal Map. */
Way way = new Way();
long idDelta = vin.readSInt64();
id += idDelta;
if (idDelta == 0) return true;
way.tags = readTags();
int nNodes = vin.readUInt32();
way.nodes = new long[nNodes];
for (int i = 0; i < nNodes; i++) {
ref += vin.readSInt64();
way.nodes[i] = ref;
}
osm.ways.put(id, way);
return false;
}
public boolean readRelation() throws IOException {
/* Create a new instance each time because we don't know if this is going in a MapDB or a normal Map. */
Relation relation = new Relation();
long idDelta = vin.readSInt64();
if (idDelta == 0) return true;
id += idDelta;
osm.relations.put(id, relation);
return false;
}
/* This should wrap a VexFormatCodec reference rather than being an inner class of it. */
private class Converter extends Parser {
private int curr_etype;
private int ecount;
// This could be done in the constructor.
private void streamBegin() throws IOException {
vout = CodedOutputStream.newInstance(new GZIPOutputStream(new FileOutputStream("/home/abyrd/test.vex")));
vout.writeRawBytes(HEADER.getBytes());
curr_etype = VEX_NONE;
ecount = 0;
}
private void streamEnd() throws IOException {
checkBlockTransition(VEX_NONE);
vout.flush();
}
private void checkBlockTransition(int toEtype) throws IOException {
if (curr_etype != toEtype) {
if (curr_etype != VEX_NONE) {
endWriteBlock(ecount);
if (ecount < 1000) {
LOG.warn("Wrote very small block of length {}", ecount);
}
ecount = 0;
}
beginWriteBlock(toEtype);
curr_etype = toEtype;
}
}
@Override
public void handleNode(long id, Node node) {
try {
checkBlockTransition(VEX_NODE);
writeNode(id, node);
ecount++;
} catch (IOException ex) {
throw new RuntimeException();
}
}
@Override
public void handleWay(long id, Way way) {
try {
checkBlockTransition(VEX_WAY);
writeWay(id, way);
ecount++;
} catch (IOException ex) {
throw new RuntimeException();
}
}
@Override
public void handleRelation(long id, Relation relation) {
try {
checkBlockTransition(VEX_RELATION);
writeRelation(id, relation);
ecount++;
} catch (IOException ex) {
throw new RuntimeException();
}
}
}
// TODO Need Etype for class, int for etype, etc.
/** This main method will convert a PBF file to VEX in a streaming manner, without an intermediate datastore. */
public static void main (String[] args) {
// final String INPUT = "/var/otp/graphs/ny/new-york-latest.osm.pbf";
final String INPUT = "/var/otp/graphs/nl/netherlands-latest.osm.pbf";
// final String INPUT = "/var/otp/graphs/trimet/portland.osm.pbf";
Converter converter = new VexFormatCodec().new Converter();
try {
converter.streamBegin();
converter.parse(INPUT);
converter.streamEnd();
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
}