Skip to content

Commit

Permalink
Make use of node tags like 'highway' or 'crossing' (#2705)
Browse files Browse the repository at this point in the history 
* Pass all node tags to edge handlers

* Remove node tag count (was only used for analysis)

* crossing EV (#2706)

* add tests; fix traffic_signalS; remove zebra as same as uncontrolled; include proposed tags https://wiki.openstreetmap.org/wiki/Proposed_features/Highway_crossing_cleanup

* fix

* fix tests

* minor fix

* minor typo

* node tag whitelist; use get not put in removeTag

* include node tags handling for road_access and fords (road_environment)

* mark only node tags of barrier edge as 'split_node'

* avoid stream and collect

* minor further perf tune

* make barrier edge explicit via tag

* simplify handleNodeTags

* Remove artificial gh:split_node tag and use long hash set instead

* minor rename

* use EdgeKVStorage for node tags storage

* rename EdgeKVStorage to KVStorage as also used to temporarily store node tags for import

* log more infos about node tags; no compact necessary due to KVStorage; limit pointer of KVStorage to positive numbers for now

* rename method to handleBarrierEdge

* fix taggednodecount method

* node tags for the barrier edge should be identical, i.e. use the ref instead of a copy

---------

Co-authored-by: Peter <graphhopper@gmx.de>
  • Loading branch information
@easbar @karussell
easbar and karussell committed Mar 16, 2023
1 parent 33859a0 commit d781707
Show file tree
Hide file tree
Showing 41 changed files with 567 additions and 258 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

### 7.0 [14 Mar 2023]

- access node tags via List instead of Map: List<Map<String, Object>> nodeTags = way.getTag("node_tags", emptyList()), see #2705
- remove StringEncodedValue support from custom model due to insufficient usage/testing
- handle also node_tags in handleWayTags, when extending AbstractAccessParser call handleNodeTags, #2738
- Format of 'areas' in CustomModel changed to 'FeatureCollection'. The old format is deprecated and will be removed in a later version, #2734
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/java/com/graphhopper/GraphHopper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -796,7 +796,7 @@ private String getVersionsString() {
",edges:" + Constants.VERSION_EDGE +
",geometry:" + Constants.VERSION_GEOMETRY +
",location_index:" + Constants.VERSION_LOCATION_IDX +
",string_index:" + Constants.VERSION_EDGEKV_STORAGE +
",string_index:" + Constants.VERSION_KV_STORAGE +
",nodesCH:" + Constants.VERSION_NODE_CH +
",shortcuts:" + Constants.VERSION_SHORTCUT;
}
Expand Down Expand Up @@ -1176,7 +1176,7 @@ protected void postProcessing(boolean closeEarly) {
if (closeEarly) {
boolean includesCustomProfiles = profilesByName.values().stream().anyMatch(p -> p instanceof CustomProfile);
if (!includesCustomProfiles)
// when there are custom profiles we must not close way geometry or EdgeKVStorage, because
// when there are custom profiles we must not close way geometry or KVStorage, because
// they might be needed to evaluate the custom weightings for the following preparations
baseGraph.flushAndCloseGeometryAndNameStorage();
}
Expand Down
69 changes: 44 additions & 25 deletions core/src/main/java/com/graphhopper/reader/osm/OSMNodeData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,23 @@

package com.graphhopper.reader.osm;

import com.carrotsearch.hppc.LongScatterSet;
import com.carrotsearch.hppc.LongSet;
import com.graphhopper.coll.GHLongIntBTree;
import com.graphhopper.coll.LongIntMap;
import com.graphhopper.reader.ReaderNode;
import com.graphhopper.search.KVStorage;
import com.graphhopper.storage.Directory;
import com.graphhopper.util.PointAccess;
import com.graphhopper.util.PointList;
import com.graphhopper.util.shapes.GHPoint3D;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.HashMap;
import java.util.Map;
import java.util.function.DoubleSupplier;
import java.util.function.IntUnaryOperator;

import static java.util.Collections.emptyMap;
import java.util.stream.Collectors;

/**
* This class stores OSM node data while reading an OSM file in {@link WaySegmentParser}. It is not trivial to do this
Expand Down Expand Up @@ -67,28 +68,30 @@ class OSMNodeData {
private final PillarInfo pillarNodes;
private final PointAccess towerNodes;

// this map stores an index for each OSM node we keep the node tags of. a value of -1 means there is no entry
// yet and a value of -2 means there was an entry but it was removed again
// this map stores an index for each OSM node we keep the node tags of. a value of -1 means there is no entry yet.
private final LongIntMap nodeTagIndicesByOsmNodeIds;

// stores node tags
private final List<Map<String, Object>> nodeTags;
private final KVStorage nodeKVStorage;
// collect all nodes that should be split and a barrier edge should be created between them.
private final LongSet nodesToBeSplit;

private int nextTowerId = 0;
private int nextPillarId = 0;
// we use negative ids to create artificial OSM node ids
private long nextArtificialOSMNodeId = -Long.MAX_VALUE;

public OSMNodeData(PointAccess nodeAccess, Directory directory) {
// we use GHLongIntBTree, because it is based on a tree, not an array, so it can store as many entries as there
// are longs. this also makes it memory efficient, because there is no need to pre-allocate memory for empty
// entries.
// We use GHLongIntBTree, because it is based on a tree, not an array, so it can store as many entries as there
// are longs. This also makes it memory efficient, because there is no need to pre-allocate memory for empty
// entries, and it also avoids allocating a new array and copying into it when increasing the size.
idsByOsmNodeIds = new GHLongIntBTree(200);
towerNodes = nodeAccess;
pillarNodes = new PillarInfo(towerNodes.is3D(), directory);

nodeTagIndicesByOsmNodeIds = new GHLongIntBTree(200);
nodeTags = new ArrayList<>();
nodesToBeSplit = new LongScatterSet();
nodeKVStorage = new KVStorage(directory, false);
}

public boolean is3D() {
Expand Down Expand Up @@ -132,11 +135,15 @@ public long getNodeCount() {
return idsByOsmNodeIds.getSize();
}

public long getTaggedNodeCount() {
return nodeTagIndicesByOsmNodeIds.getSize();
}

/**
* @return the number of nodes for which we store tags
*/
public long getTaggedNodeCount() {
return nodeTags.size();
public long getNodeTagCapacity() {
return nodeKVStorage.getCapacity();
}

/**
Expand Down Expand Up @@ -187,7 +194,7 @@ SegmentNode addCopyOfNode(SegmentNode node) {
if (idsByOsmNodeIds.put(newOsmId, INTERMEDIATE_NODE) != EMPTY_NODE)
throw new IllegalStateException("Artificial osm node id already exists: " + newOsmId);
int id = addPillarNode(newOsmId, point.getLat(), point.getLon(), point.getEle());
return new SegmentNode(newOsmId, id);
return new SegmentNode(newOsmId, id, node.tags);
}

int convertPillarToTowerNode(int id, long osmNodeId) {
Expand Down Expand Up @@ -241,11 +248,13 @@ public void addCoordinatesToPointList(int id, PointList pointList) {

public void setTags(ReaderNode node) {
int tagIndex = nodeTagIndicesByOsmNodeIds.get(node.getId());
if (tagIndex == -2)
throw new IllegalStateException("Cannot add tags after they were removed");
else if (tagIndex == -1) {
nodeTagIndicesByOsmNodeIds.put(node.getId(), nodeTags.size());
nodeTags.add(node.getTags());
if (tagIndex == -1) {
long pointer = nodeKVStorage.add(node.getTags().entrySet().stream().map(m -> new KVStorage.KeyValue(m.getKey(),
m.getValue() instanceof String ? KVStorage.cutString((String) m.getValue()) : m.getValue())).
collect(Collectors.toList()));
if (pointer > Integer.MAX_VALUE)
throw new IllegalStateException("Too many key value pairs are stored in node tags, was " + pointer);
nodeTagIndicesByOsmNodeIds.put(node.getId(), (int) pointer);
} else {
throw new IllegalStateException("Cannot add tags twice, duplicate node OSM ID: " + node.getId());
}
Expand All @@ -255,16 +264,12 @@ public Map<String, Object> getTags(long osmNodeId) {
int tagIndex = nodeTagIndicesByOsmNodeIds.get(osmNodeId);
if (tagIndex < 0)
return Collections.emptyMap();
return nodeTags.get(tagIndex);
}

public void removeTags(long osmNodeId) {
int prev = nodeTagIndicesByOsmNodeIds.put(osmNodeId, -2);
nodeTags.set(prev, emptyMap());
return nodeKVStorage.getMap(tagIndex);
}

public void release() {
pillarNodes.clear();
nodeKVStorage.clear();
}

public int towerNodeToId(int towerId) {
Expand All @@ -282,4 +287,18 @@ public int pillarNodeToId(int pillarId) {
public int idToPillarNode(int id) {
return id - 3;
}

public boolean setSplitNode(long osmNodeId) {
return nodesToBeSplit.add(osmNodeId);
}

public void unsetSplitNode(long osmNodeId) {
int removed = nodesToBeSplit.removeAll(osmNodeId);
if (removed == 0)
throw new IllegalStateException("Node " + osmNodeId + " was not a split node");
}

public boolean isSplitNode(long osmNodeId) {
return nodesToBeSplit.contains(osmNodeId);
}
}
36 changes: 19 additions & 17 deletions core/src/main/java/com/graphhopper/reader/osm/OSMReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
import com.graphhopper.routing.util.countryrules.CountryRule;
import com.graphhopper.routing.util.countryrules.CountryRuleFactory;
import com.graphhopper.routing.util.parsers.RestrictionSetter;
import com.graphhopper.search.EdgeKVStorage;
import com.graphhopper.search.KVStorage;
import com.graphhopper.storage.BaseGraph;
import com.graphhopper.storage.IntsRef;
import com.graphhopper.storage.NodeAccess;
Expand All @@ -57,7 +57,7 @@
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static com.graphhopper.search.EdgeKVStorage.KeyValue.*;
import static com.graphhopper.search.KVStorage.KeyValue.*;
import static com.graphhopper.util.Helper.nf;
import static java.util.Collections.emptyList;

Expand Down Expand Up @@ -228,7 +228,7 @@ private boolean isFerry(ReaderWay way) {
* This method is called during the second pass of {@link WaySegmentParser} and provides an entry point to enrich
* the given OSM way with additional tags before it is passed on to the tag parsers.
*/
protected void setArtificialWayTags(PointList pointList, ReaderWay way, double distance, Map<String, Object> nodeTags) {
protected void setArtificialWayTags(PointList pointList, ReaderWay way, double distance, List<Map<String, Object>> nodeTags) {
way.setTag("node_tags", nodeTags);
way.setTag("edge_distance", distance);
way.setTag("point_list", pointList);
Expand Down Expand Up @@ -293,14 +293,16 @@ protected void setArtificialWayTags(PointList pointList, ReaderWay way, double d
* @param toIndex a unique integer id for the last node of this segment
* @param pointList coordinates of this segment
* @param way the OSM way this segment was taken from
* @param nodeTags node tags of this segment if it is an artificial edge, empty otherwise
* @param nodeTags node tags of this segment. there is one map of tags for each point.
*/
protected void addEdge(int fromIndex, int toIndex, PointList pointList, ReaderWay way, Map<String, Object> nodeTags) {
protected void addEdge(int fromIndex, int toIndex, PointList pointList, ReaderWay way, List<Map<String, Object>> nodeTags) {
// sanity checks
if (fromIndex < 0 || toIndex < 0)
throw new AssertionError("to or from index is invalid for this edge " + fromIndex + "->" + toIndex + ", points:" + pointList);
if (pointList.getDimension() != nodeAccess.getDimension())
throw new AssertionError("Dimension does not match for pointList vs. nodeAccess " + pointList.getDimension() + " <-> " + nodeAccess.getDimension());
if (pointList.size() != nodeTags.size())
throw new AssertionError("there should be as many maps of node tags as there are points. node tags: " + nodeTags.size() + ", points: " + pointList.size());

// todo: in principle it should be possible to delay elevation calculation so we do not need to store
// elevations during import (saves memory in pillar info during import). also note that we already need to
Expand Down Expand Up @@ -350,7 +352,7 @@ else if (config.getElevationSmoothing().equals("moving_average"))
IntsRef edgeFlags = encodingManager.createEdgeFlags();
osmParsers.handleWayTags(edgeFlags, way, relationFlags);
EdgeIteratorState edge = baseGraph.edge(fromIndex, toIndex).setDistance(distance).setFlags(edgeFlags);
List<EdgeKVStorage.KeyValue> list = way.getTag("key_values", Collections.emptyList());
List<KVStorage.KeyValue> list = way.getTag("key_values", Collections.emptyList());
if (!list.isEmpty())
edge.setKeyValues(list);

Expand Down Expand Up @@ -394,7 +396,7 @@ else if (Math.abs(edgeDistance - geometryDistance) > tolerance)
*/
protected void preprocessWay(ReaderWay way, WaySegmentParser.CoordinateSupplier coordinateSupplier) {
// storing the road name does not yet depend on the flagEncoder so manage it directly
List<EdgeKVStorage.KeyValue> list = new ArrayList<>();
List<KVStorage.KeyValue> list = new ArrayList<>();
if (config.isParseWayNames()) {
// http://wiki.openstreetmap.org/wiki/Key:name
String name = "";
Expand All @@ -403,28 +405,28 @@ protected void preprocessWay(ReaderWay way, WaySegmentParser.CoordinateSupplier
if (name.isEmpty())
name = fixWayName(way.getTag("name"));
if (!name.isEmpty())
list.add(new EdgeKVStorage.KeyValue(STREET_NAME, name));
list.add(new KVStorage.KeyValue(STREET_NAME, name));

// http://wiki.openstreetmap.org/wiki/Key:ref
String refName = fixWayName(way.getTag("ref"));
if (!refName.isEmpty())
list.add(new EdgeKVStorage.KeyValue(STREET_REF, refName));
list.add(new KVStorage.KeyValue(STREET_REF, refName));

if (way.hasTag("destination:ref")) {
list.add(new EdgeKVStorage.KeyValue(STREET_DESTINATION_REF, fixWayName(way.getTag("destination:ref"))));
list.add(new KVStorage.KeyValue(STREET_DESTINATION_REF, fixWayName(way.getTag("destination:ref"))));
} else {
if (way.hasTag("destination:ref:forward"))
list.add(new EdgeKVStorage.KeyValue(STREET_DESTINATION_REF, fixWayName(way.getTag("destination:ref:forward")), true, false));
list.add(new KVStorage.KeyValue(STREET_DESTINATION_REF, fixWayName(way.getTag("destination:ref:forward")), true, false));
if (way.hasTag("destination:ref:backward"))
list.add(new EdgeKVStorage.KeyValue(STREET_DESTINATION_REF, fixWayName(way.getTag("destination:ref:backward")), false, true));
list.add(new KVStorage.KeyValue(STREET_DESTINATION_REF, fixWayName(way.getTag("destination:ref:backward")), false, true));
}
if (way.hasTag("destination")) {
list.add(new EdgeKVStorage.KeyValue(STREET_DESTINATION, fixWayName(way.getTag("destination"))));
list.add(new KVStorage.KeyValue(STREET_DESTINATION, fixWayName(way.getTag("destination"))));
} else {
if (way.hasTag("destination:forward"))
list.add(new EdgeKVStorage.KeyValue(STREET_DESTINATION, fixWayName(way.getTag("destination:forward")), true, false));
list.add(new KVStorage.KeyValue(STREET_DESTINATION, fixWayName(way.getTag("destination:forward")), true, false));
if (way.hasTag("destination:backward"))
list.add(new EdgeKVStorage.KeyValue(STREET_DESTINATION, fixWayName(way.getTag("destination:backward")), false, true));
list.add(new KVStorage.KeyValue(STREET_DESTINATION, fixWayName(way.getTag("destination:backward")), false, true));
}
}
way.setTag("key_values", list);
Expand Down Expand Up @@ -478,8 +480,8 @@ protected void preprocessWay(ReaderWay way, WaySegmentParser.CoordinateSupplier
static String fixWayName(String str) {
if (str == null)
return "";
// the EdgeKVStorage does not accept too long strings -> Helper.cutStringForKV
return EdgeKVStorage.cutString(WAY_NAME_PATTERN.matcher(str).replaceAll(", "));
// the KVStorage does not accept too long strings -> Helper.cutStringForKV
return KVStorage.cutString(WAY_NAME_PATTERN.matcher(str).replaceAll(", "));
}

/**
Expand Down
6 changes: 5 additions & 1 deletion core/src/main/java/com/graphhopper/reader/osm/SegmentNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,16 @@

package com.graphhopper.reader.osm;

import java.util.Map;

class SegmentNode {
long osmNodeId;
int id;
Map<String, Object> tags;

public SegmentNode(long osmNodeId, int id) {
public SegmentNode(long osmNodeId, int id, Map<String, Object> tags) {
this.osmNodeId = osmNodeId;
this.id = id;
this.tags = tags;
}
}

0 comments on commit d781707

Please sign in to comment.