Add landuse encoded value based on closed-ring ways tagged with landuse=*

core/pom.xml

@@ -96,6 +96,12 @@
             <artifactId>logback-classic</artifactId>
             <scope>test</scope>
         </dependency>
+        <!-- todonow: remove later -->
+        <dependency>
+            <groupId>org.openjdk.jol</groupId>
+            <artifactId>jol-core</artifactId>
+            <version>0.13</version>
+        </dependency>
     </dependencies>
 
     <build>
@@ -165,4 +171,4 @@
             </resource>
         </resources>
     </build>
-</project>
+</project>

core/src/main/java/com/graphhopper/GraphHopper.java

@@ -922,7 +922,7 @@ protected void importOSM() {
         }
 
         logger.info("start creating graph from " + osmFile);
-        OSMReader reader = new OSMReader(baseGraph.getBaseGraph(), osmParsers, osmReaderConfig).setFile(_getOSMFile()).
+        OSMReader reader = new OSMReader(baseGraph.getBaseGraph(), encodingManager, osmParsers, osmReaderConfig).setFile(_getOSMFile()).
                 setAreaIndex(areaIndex).
                 setElevationProvider(eleProvider).
                 setCountryRuleFactory(countryRuleFactory);

core/src/main/java/com/graphhopper/reader/osm/OSMArea.java

@@ -0,0 +1,73 @@
+/*
+ *  Licensed to GraphHopper GmbH under one or more contributor
+ *  license agreements. See the NOTICE file distributed with this work for
+ *  additional information regarding copyright ownership.
+ *
+ *  GraphHopper GmbH licenses this file to you under the Apache License,
+ *  Version 2.0 (the "License"); you may not use this file except in
+ *  compliance with the License. You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.graphhopper.reader.osm;
+
+import com.graphhopper.routing.util.AreaIndex;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.geom.Polygon;
+import org.locationtech.jts.geom.impl.PackedCoordinateSequence;
+
+import java.util.Arrays;
+import java.util.Map;
+
+public class OSMArea implements AreaIndex.Area {
+    private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory();
+    final PackedCoordinateSequence.Float border;
+    private final long tagPointer;
+    private double area = -1;
+
+    public OSMArea(int numNodes, long tagPointer) {
+        float[] coords = new float[numNodes * 2];
+        Arrays.fill(coords, Float.NaN);
+        border = new PackedCoordinateSequence.Float(coords, 2, 0);
+        this.tagPointer = tagPointer;
+    }
+
+    public void setCoordinate(int index, double lat, double lon) {
+        if (area >= 0)
+            throw new IllegalStateException("Cannot modify coordinates after getBorder() was called");
+        // todonow: precision/double->float cast?
+        border.setX(index, (float) lon);
+        border.setY(index, (float) lat);
+    }
+
+    public boolean isValid() {
+        float[] coords = border.getRawCoordinates();
+        for (float f : coords)
+            if (Float.isNaN(f))
+                return false;
+        // todonow: print warnings for non-closed geometries and missing coordinates?
+        return coords.length >= 8 && coords[0] == coords[coords.length - 2] && coords[1] == coords[coords.length - 1];
+    }
+
+    public long getTagPointer() {
+        return tagPointer;
+    }
+
+    public double getArea() {
+        return area;
+    }
+
+    @Override
+    public Polygon getBorder() {
+        Polygon border = GEOMETRY_FACTORY.createPolygon(this.border);
+        area = border.getArea();
+        return border;
+    }
+}

core/src/main/java/com/graphhopper/reader/osm/OSMAreaData.java

@@ -0,0 +1,121 @@
+/*
+ *  Licensed to GraphHopper GmbH under one or more contributor
+ *  license agreements. See the NOTICE file distributed with this work for
+ *  additional information regarding copyright ownership.
+ *
+ *  GraphHopper GmbH licenses this file to you under the Apache License,
+ *  Version 2.0 (the "License"); you may not use this file except in
+ *  compliance with the License. You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.graphhopper.reader.osm;
+
+import com.carrotsearch.hppc.IntArrayList;
+import com.carrotsearch.hppc.LongArrayList;
+import com.carrotsearch.hppc.cursors.LongCursor;
+import com.graphhopper.coll.GHLongIntBTree;
+import com.graphhopper.coll.LongIntMap;
+import com.graphhopper.reader.ReaderNode;
+import com.graphhopper.reader.ReaderWay;
+import com.graphhopper.search.KVStorage;
+import com.graphhopper.storage.Directory;
+import com.graphhopper.storage.RAMDirectory;
+import com.graphhopper.util.BitUtil;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class OSMAreaData {
+    // todonow: private
+    final List<OSMArea> osmAreas;
+    final LongToTwoIntsMap osmAreaNodeIndicesByOSMNodeIds;
+    final KVStorage osmAreaTagStorage;
+
+    public OSMAreaData(Directory directory) {
+        osmAreas = new ArrayList<>();
+        osmAreaNodeIndicesByOSMNodeIds = new LongToTwoIntsMap();
+        // todonow: using a separate ram directory, because otherwise it is harder to measure memory (bc the kv storage keeps a reference to the directory)
+        osmAreaTagStorage = new KVStorage(new RAMDirectory(), "osm_area_tags_");
+    }
+
+    public void addOSMAreaWithoutCoordinates(LongArrayList osmNodeIds, Map<String, Object> tags) {
+        long tagPointer = osmAreaTagStorage.add(tags.entrySet().stream().map(m -> new KVStorage.KeyValue(m.getKey(),
+                        m.getValue() instanceof String ? KVStorage.cutString((String) m.getValue()) : m.getValue()))
+                .collect(Collectors.toList()));
+        if (tagPointer > Integer.MAX_VALUE)
+            throw new IllegalStateException("Too many tags are stored for OSM areas " + tagPointer);
+        osmAreas.add(new OSMArea(osmNodeIds.size(), tagPointer));
+        for (LongCursor node : osmNodeIds)
+            osmAreaNodeIndicesByOSMNodeIds.put(node.value, osmAreas.size() - 1, node.index);
+    }
+
+    public void fillOSMAreaNodeCoordinates(ReaderNode node) {
+        long index = osmAreaNodeIndicesByOSMNodeIds.get(node.getId());
+        if (index >= 0) {
+            int osmAreaIndex = BitUtil.LITTLE.getIntLow(index);
+            int nodeIndex = BitUtil.LITTLE.getIntHigh(index);
+            OSMArea osmArea = osmAreas.get(osmAreaIndex);
+            // Note that we set the coordinates only for one particular node for one particular
+            // osm area, even though the same osm node might be used in multiple such areas. We will
+            // fix the next time we get to see the osm area ways.
+            osmArea.setCoordinate(nodeIndex, node.getLat(), node.getLon());
+        }
+    }
+
+    public void fixOSMArea(int osmAreaWayIndex, ReaderWay way) {
+        // The problem we solve here is that some osm nodes are used by multiple landuse/area ways.
+        // At the very least this is the case for the first and last nodes of the closed-ring ways,
+        // but there are also many areas that really share common nodes. Since we only store one
+        // index into the coordinate array of the area polygons, the coordinates for some polygon
+        // nodes won't be set. Therefore we need to make up for this here where we get to see the
+        // ways a second time. If this wasn't the case we could read the area ways&nodes in pass1
+        // instead of pass0 which would allow us to skip nodes and ways in pass0 (faster import).
+        OSMArea actual = osmAreas.get(osmAreaWayIndex);
+        for (LongCursor node : way.getNodes()) {
+            long index = osmAreaNodeIndicesByOSMNodeIds.get(node.value);
+            int osmAreaIndex = BitUtil.LITTLE.getIntLow(index);
+            int nodeIndex = BitUtil.LITTLE.getIntHigh(index);
+            OSMArea osmArea = osmAreas.get(osmAreaIndex);
+            actual.border.setX(node.index, osmArea.border.getX(nodeIndex));
+            actual.border.setY(node.index, osmArea.border.getY(nodeIndex));
+        }
+    }
+
+    public List<OSMArea> getOSMAreas() {
+        return osmAreas;
+    }
+
+    public static class LongToTwoIntsMap {
+        private final LongIntMap internalIdsByKey = new GHLongIntBTree(200);
+        private final IntArrayList vals1 = new IntArrayList();
+        private final IntArrayList vals2 = new IntArrayList();
+
+        public void put(long key, int val1, int val2) {
+            vals1.add(val1);
+            vals2.add(val2);
+            internalIdsByKey.put(key, vals1.size() - 1);
+        }
+
+        public long get(long key) {
+            int id = internalIdsByKey.get(key);
+            if (id < 0) return -1;
+            return BitUtil.LITTLE.combineIntsToLong(vals1.get(id), vals2.get(id));
+        }
+
+        public void clear() {
+            internalIdsByKey.clear();
+            vals1.release();
+            vals2.release();
+        }
+    }
+}

core/src/main/java/com/graphhopper/reader/osm/OSMNodeData.java

@@ -86,11 +86,11 @@ public OSMNodeData(PointAccess nodeAccess, Directory directory) {
         // entries, and it also avoids allocating a new array and copying into it when increasing the size.
         idsByOsmNodeIds = new GHLongIntBTree(200);
         towerNodes = nodeAccess;
-        pillarNodes = new PillarInfo(towerNodes.is3D(), directory);
+        pillarNodes = new PillarInfo(towerNodes.is3D(), directory, "");
 
         nodeTagIndicesByOsmNodeIds = new GHLongIntBTree(200);
         nodesToBeSplit = new LongScatterSet();
-        nodeKVStorage = new KVStorage(directory, false);
+        nodeKVStorage = new KVStorage(directory, "nodekv_");
     }
 
     public boolean is3D() {

core/src/main/java/com/graphhopper/reader/osm/OSMReader.java

@@ -33,8 +33,10 @@
 import com.graphhopper.routing.OSMReaderConfig;
 import com.graphhopper.routing.ev.Country;
 import com.graphhopper.routing.ev.EdgeIntAccess;
+import com.graphhopper.routing.ev.Landuse;
 import com.graphhopper.routing.util.AreaIndex;
 import com.graphhopper.routing.util.CustomArea;
+import com.graphhopper.routing.util.EncodingManager;
 import com.graphhopper.routing.util.OSMParsers;
 import com.graphhopper.routing.util.countryrules.CountryRule;
 import com.graphhopper.routing.util.countryrules.CountryRuleFactory;
@@ -47,6 +49,7 @@
 import com.graphhopper.util.*;
 import com.graphhopper.util.shapes.GHPoint;
 import com.graphhopper.util.shapes.GHPoint3D;
+import org.openjdk.jol.info.GraphLayout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -98,8 +101,13 @@ public class OSMReader {
     private WayToEdgesMap restrictedWaysToEdgesMap = new WayToEdgesMap();
     private List<ReaderRelation> restrictionRelations = new ArrayList<>();
 
-    public OSMReader(BaseGraph baseGraph, OSMParsers osmParsers, OSMReaderConfig config) {
+    private final OSMAreaData osmAreaData;
+    private AreaIndex<OSMArea> osmAreaIndex;
+    private final EncodingManager encodingManager;
+
+    public OSMReader(BaseGraph baseGraph, EncodingManager encodingManager, OSMParsers osmParsers, OSMReaderConfig config) {
         this.baseGraph = baseGraph;
+        this.encodingManager = encodingManager;
         this.edgeIntAccess = baseGraph.createEdgeIntAccess();
         this.config = config;
         this.nodeAccess = baseGraph.getNodeAccess();
@@ -114,6 +122,8 @@ public OSMReader(BaseGraph baseGraph, OSMParsers osmParsers, OSMReaderConfig con
         if (tempRelFlags.length != 2)
             // we use a long to store relation flags currently, so the relation flags ints ref must have length 2
             throw new IllegalArgumentException("OSMReader cannot use relation flags with != 2 integers");
+
+        osmAreaData = new OSMAreaData(baseGraph.getDirectory());
     }
 
     /**
@@ -161,16 +171,29 @@ public void readGraph() throws IOException {
         if (!baseGraph.isInitialized())
             throw new IllegalStateException("BaseGraph must be initialize before we can read OSM");
 
-        WaySegmentParser waySegmentParser = new WaySegmentParser.Builder(baseGraph.getNodeAccess(), baseGraph.getDirectory())
+        WaySegmentParser.Builder waySegmentParserBuilder = new WaySegmentParser.Builder(baseGraph.getNodeAccess(), baseGraph.getDirectory())
                 .setElevationProvider(eleProvider)
                 .setWayFilter(this::acceptWay)
                 .setSplitNodeFilter(this::isBarrierNode)
                 .setWayPreprocessor(this::preprocessWay)
                 .setRelationPreprocessor(this::preprocessRelations)
                 .setRelationProcessor(this::processRelation)
                 .setEdgeHandler(this::addEdge)
-                .setWorkerThreads(config.getWorkerThreads())
-                .build();
+                .setWorkerThreads(config.getWorkerThreads());
+        if (encodingManager.hasEncodedValue(Landuse.KEY)) {
+            waySegmentParserBuilder
+                    // todonow: we could move the pass1 way handling and the pass2 node handling of the standard way segment parser to pass0/1,
+                    //          then we could skip nodes in pass2 to speed up the import
+                    .setPass0WayPreHook(this::handleOSMArea)
+                    .setPass1NodePreHook(osmAreaData::fillOSMAreaNodeCoordinates)
+                    .setPass1WayPreHook(this::handleOSMAreaAgain)
+                    .setPass1FinishHook(() -> {
+//                        System.out.println(GraphLayout.parseInstance(osmAreaData.osmAreaNodeIndicesByOSMNodeIds).toFootprint());
+                        osmAreaData.osmAreaNodeIndicesByOSMNodeIds.clear();
+                    })
+                    .setPass2AfterNodesHook(this::buildOSMAreaIndex);
+        }
+        WaySegmentParser waySegmentParser = waySegmentParserBuilder.build();
         waySegmentParser.readOSM(osmFile);
         osmDataDate = waySegmentParser.getTimeStamp();
         if (baseGraph.getNodes() == 0)
@@ -189,6 +212,39 @@ public Date getDataDate() {
         return osmDataDate;
     }
 
+    void handleOSMArea(ReaderWay way) {
+        if (way.hasTag("landuse"))
+            osmAreaData.addOSMAreaWithoutCoordinates(way.getNodes(), way.getTags());
+    }
+
+
+    int osmAreaWayIndex = -1;
+
+    void handleOSMAreaAgain(ReaderWay way) {
+        if (way.hasTag("landuse")) {
+            osmAreaWayIndex++;
+            osmAreaData.fixOSMArea(osmAreaWayIndex, way);
+        }
+    }
+
+    void buildOSMAreaIndex() {
+//        System.out.println(GraphLayout.parseInstance(osmAreaData).toFootprint());
+        List<OSMArea> validAreas = osmAreaData.getOSMAreas().stream().filter(a -> {
+            if (!a.isValid()) {
+                OSM_WARNING_LOGGER.warn("invalid OSM area: " + a.border);
+                return false;
+            }
+            return true;
+        }).collect(Collectors.toList());
+//        System.out.println(GraphLayout.parseInstance(osmAreaData, validAreas).toFootprint());
+        osmAreaData.getOSMAreas().clear();
+        LOGGER.info("Building area index for {} OSM areas (invalid: {})", validAreas.size(), (osmAreaData.getOSMAreas().size() - validAreas.size()));
+        osmAreaIndex = new AreaIndex<>(validAreas);
+//        System.out.println(GraphLayout.parseInstance(osmAreaIndex).toFootprint());
+        // they partly overlap
+//        System.out.println(GraphLayout.parseInstance(osmAreaData, osmAreaIndex, validAreas).toFootprint());
+    }
+
     /**
      * This method is called for each way during the first and second pass of the {@link WaySegmentParser}. All OSM
      * ways that are not accepted here and all nodes that are not referenced by any such way will be ignored.
@@ -239,9 +295,11 @@ protected void setArtificialWayTags(PointList pointList, ReaderWay way, double d
         way.removeTag("country");
         way.removeTag("country_rule");
         way.removeTag("custom_areas");
+        way.removeTag("gh:osm_areas");
 
-        List<CustomArea> customAreas;
-        if (areaIndex != null) {
+        List<CustomArea> customAreas = emptyList();
+        List<OSMArea> osmAreas = emptyList();
+        if (areaIndex != null || osmAreaIndex != null) {
             double middleLat;
             double middleLon;
             if (pointList.size() > 2) {
@@ -253,10 +311,10 @@ protected void setArtificialWayTags(PointList pointList, ReaderWay way, double d
                 middleLat = (firstLat + lastLat) / 2;
                 middleLon = (firstLon + lastLon) / 2;
             }
-            customAreas = areaIndex.query(middleLat, middleLon);
-        } else {
-            customAreas = emptyList();
+            if (areaIndex != null) customAreas = areaIndex.query(middleLat, middleLon);
+            if (osmAreaIndex != null) osmAreas = osmAreaIndex.query(middleLat, middleLon);
         }
+        osmAreas.sort(Comparator.comparing(OSMArea::getArea));
 
         // special handling for countries: since they are built-in with GraphHopper they are always fed to the EncodingManager
         Country country = Country.MISSING;
@@ -284,6 +342,12 @@ protected void setArtificialWayTags(PointList pointList, ReaderWay way, double d
 
         // also add all custom areas as artificial tag
         way.setTag("custom_areas", customAreas);
+
+        List<Map<String, Object>> osmAreaTags = osmAreas.stream()
+                .map(a -> osmAreaData.osmAreaTagStorage.getAll(a.getTagPointer())
+                        .stream().collect(Collectors.toMap(kv -> kv.key, kv -> kv.value)))
+                .collect(Collectors.toList());
+        way.setTag("gh:osm_areas", osmAreaTags);
     }
 
     /**