(WIP) EncodingManager refactoring

NOTICE.md

@@ -7,7 +7,7 @@ The core module includes the following software:
  * slf4j.org - SLF4J distributed under the MIT license. 
  * com.carrotsearch:hppc (Apache license)
  * SparseArray from the Android project (Apache license)
- * Snippets regarding mmap, vint/vlong and compression from Lucene (Apache license)
+ * Code regarding mmap, IntsRef, vint/vlong and compression from Lucene (Apache license)
  * XMLGraphics-Commons for CGIAR elevation files (Apache License)
  * Apache Commons Lang - we copied the implementation of the Levenshtein Distance (Apache License)
  * Apache Commons Collections - we copied parts of the BinaryHeap (Apache License)

core/pom.xml

@@ -36,6 +36,11 @@
         </license>
     </licenses>
     <dependencies>
+        <dependency>
+            <groupId>com.graphhopper</groupId>
+            <artifactId>graphhopper-json</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.carrotsearch</groupId>
             <artifactId>hppc</artifactId>
@@ -54,25 +59,26 @@
             <version>2.1</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
+        <!--
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
             <version>${jackson.version}</version>
         </dependency>
-
+        -->
+
         <dependency>
             <groupId>com.vividsolutions</groupId>
             <artifactId>jts</artifactId>
             <version>1.13</version>
         </dependency>
 
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>${jackson.version}</version>
-            <scope>test</scope>
-        </dependency>
-
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-log4j12</artifactId>
@@ -85,7 +91,7 @@
             <version>${log4j.version}</version>
             <scope>test</scope>
         </dependency>
-        
+
     </dependencies>
 
     <build>

core/src/main/java/com/graphhopper/GraphHopper.java

@@ -17,13 +17,16 @@
  */
 package com.graphhopper;
 
+import com.graphhopper.json.GHJson;
 import com.graphhopper.json.geo.JsonFeature;
 import com.graphhopper.reader.DataReader;
+import com.graphhopper.reader.ReaderWay;
 import com.graphhopper.reader.dem.*;
 import com.graphhopper.routing.*;
 import com.graphhopper.routing.ch.CHAlgoFactoryDecorator;
 import com.graphhopper.routing.ch.PrepareContractionHierarchies;
 import com.graphhopper.routing.lm.LMAlgoFactoryDecorator;
+import com.graphhopper.routing.profiles.*;
 import com.graphhopper.routing.subnetwork.PrepareRoutingSubnetworks;
 import com.graphhopper.routing.template.AlternativeRoutingTemplate;
 import com.graphhopper.routing.template.RoundTripRoutingTemplate;
@@ -49,6 +52,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.text.DateFormat;
 import java.util.*;
@@ -113,8 +117,14 @@ public class GraphHopper implements GraphHopperAPI {
     private ElevationProvider eleProvider = ElevationProvider.NOOP;
     private FlagEncoderFactory flagEncoderFactory = FlagEncoderFactory.DEFAULT;
     private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+    private final GHJson json;
 
     public GraphHopper() {
+        this(GHJson.EMPTY);
+    }
+
+    public GraphHopper(GHJson json) {
+        this.json = json;
         chFactoryDecorator.setEnabled(true);
         lmFactoryDecorator.setEnabled(false);
 
@@ -148,8 +158,7 @@ public EncodingManager getEncodingManager() {
     }
 
     /**
-     * Specify which vehicles can be read by this GraphHopper instance. An encoding manager defines
-     * how data from every vehicle is written (und read) into edges of the graph.
+     * Specify which data can be read by this GraphHopper instance.
      */
     public GraphHopper setEncodingManager(EncodingManager em) {
         ensureNotLoaded();
@@ -533,8 +542,26 @@ public GraphHopper init(CmdArgs args) {
         removeZipped = args.getBool("graph.remove_zipped", removeZipped);
         int bytesForFlags = args.getInt("graph.bytes_for_flags", 4);
         String flagEncodersStr = args.get("graph.flag_encoders", "");
-        if (!flagEncodersStr.isEmpty())
-            setEncodingManager(new EncodingManager(flagEncoderFactory, flagEncodersStr, bytesForFlags));
+        if (!flagEncodersStr.isEmpty()) {
+            if (bytesForFlags > 8) {
+                final Map<String, Double> speedMap = TagParserFactory.Car.createSpeedMap();
+                ReaderWayFilter filter = new ReaderWayFilter() {
+                    @Override
+                    public boolean accept(ReaderWay way) {
+                        return speedMap.containsKey(way.getTag("highway"));
+                    }
+                };
+                setEncodingManager(new EncodingManager.Builder(bytesForFlags).
+                        addGlobalEncodedValues().
+                        add(TagParserFactory.Car.createMaxSpeed(new DecimalEncodedValue(TagParserFactory.CAR_MAX_SPEED, 5, 0, 5, false), filter)).
+                        add(TagParserFactory.Car.createAverageSpeed(new DecimalEncodedValue(TagParserFactory.CAR_AVERAGE_SPEED, 5, 0, 5, false), speedMap)).
+                        add(TagParserFactory.Car.createAccess(new BooleanEncodedValue(TagParserFactory.CAR_ACCESS, true), filter)).
+                        build());
+            } else {
+                setEncodingManager(new EncodingManager.Builder(bytesForFlags).addGlobalEncodedValues().
+                        addAll(flagEncoderFactory, flagEncodersStr, bytesForFlags, true).build());
+            }
+        }
 
         if (args.get("graph.locktype", "native").equals("simple"))
             lockFactory = new SimpleFSLockFactory();
@@ -722,7 +749,7 @@ public boolean load(String graphHopperFolder) {
         setGraphHopperLocation(graphHopperFolder);
 
         if (encodingManager == null)
-            setEncodingManager(EncodingManager.create(flagEncoderFactory, ghLocation));
+            setEncodingManager(EncodingManager.create(flagEncoderFactory, json, ghLocation));
 
         if (!allowWrites && dataAccessType.isMMap())
             dataAccessType = DAType.MMAP_RO;
@@ -736,9 +763,9 @@ public boolean load(String graphHopperFolder) {
 
         if (chFactoryDecorator.isEnabled()) {
             initCHAlgoFactoryDecorator();
-            ghStorage = new GraphHopperStorage(chFactoryDecorator.getWeightings(), dir, encodingManager, hasElevation(), ext);
+            ghStorage = new GraphHopperStorage(chFactoryDecorator.getWeightings(), dir, encodingManager, json, hasElevation(), ext);
         } else {
-            ghStorage = new GraphHopperStorage(dir, encodingManager, hasElevation(), ext);
+            ghStorage = new GraphHopperStorage(dir, encodingManager, json, hasElevation(), ext);
         }
 
         ghStorage.setSegmentSize(defaultSegmentSize);
@@ -748,7 +775,7 @@ public boolean load(String graphHopperFolder) {
 
         GHLock lock = null;
         try {
-            // create locks only if writes are allowed, if they are not allowed a lock cannot be created 
+            // create locks only if writes are allowed, if they are not allowed a lock cannot be created
             // (e.g. on a read only filesystem locks would fail)
             if (ghStorage.getDirectory().getDefaultType().isStoring() && isAllowWrites()) {
                 lockFactory.setLockDir(new File(ghLocation));
@@ -829,7 +856,7 @@ public void postProcessing() {
             if (ghStorage.isCHPossible() && isCHPrepared())
                 throw new IllegalArgumentException("Sorting a prepared CHGraph is not possible yet. See #12");
 
-            GraphHopperStorage newGraph = GHUtility.newStorage(ghStorage);
+            GraphHopperStorage newGraph = GHUtility.newStorage(ghStorage, json);
             GHUtility.sortDFS(ghStorage, newGraph);
             logger.info("graph sorted (" + Helper.getMemInfo() + ")");
             ghStorage = newGraph;
@@ -852,20 +879,14 @@ public void postProcessing() {
     }
 
     private void interpolateBridgesAndOrTunnels() {
-        if (ghStorage.getEncodingManager().supports("generic")) {
-            final FlagEncoder genericFlagEncoder = ghStorage.getEncodingManager()
-                    .getEncoder("generic");
-            if (!(genericFlagEncoder instanceof DataFlagEncoder)) {
-                throw new IllegalStateException("'generic' flag encoder for elevation interpolation of "
-                        + "bridges and tunnels is enabled but does not have the expected type "
-                        + DataFlagEncoder.class.getName() + ".");
-            }
-            final DataFlagEncoder dataFlagEncoder = (DataFlagEncoder) genericFlagEncoder;
+        EncodingManager em = ghStorage.getEncodingManager();
+        if (em.supports(TagParserFactory.ROAD_ENVIRONMENT)) {
+            StringEncodedValue roadEnvironmentEnc = em.getStringEncodedValue(TagParserFactory.ROAD_ENVIRONMENT);
             StopWatch sw = new StopWatch().start();
-            new TunnelElevationInterpolator(ghStorage, dataFlagEncoder).execute();
+            new TunnelElevationInterpolator(ghStorage, roadEnvironmentEnc).execute();
             float tunnel = sw.stop().getSeconds();
             sw = new StopWatch().start();
-            new BridgeElevationInterpolator(ghStorage, dataFlagEncoder).execute();
+            new BridgeElevationInterpolator(ghStorage, roadEnvironmentEnc).execute();
             logger.info("Bridge interpolation " + (int) sw.stop().getSeconds() + "s, "
                     + "tunnel interpolation " + (int) tunnel + "s");
         }
@@ -887,7 +908,9 @@ public Weighting createWeighting(HintsMap hintsMap, FlagEncoder encoder, Graph g
         String weightingStr = hintsMap.getWeighting().toLowerCase();
         Weighting weighting = null;
 
-        if (encoder.supports(GenericWeighting.class)) {
+        if (weightingStr.equals("fastest2")) {
+            weighting = new FastestCarWeighting(encodingManager, "fastest2");
+        } else if (encoder.supports(GenericWeighting.class)) {
             weighting = new GenericWeighting((DataFlagEncoder) encoder, hintsMap);
         } else if ("shortest".equalsIgnoreCase(weightingStr)) {
             weighting = new ShortestWeighting(encoder);
@@ -910,7 +933,7 @@ public Weighting createWeighting(HintsMap hintsMap, FlagEncoder encoder, Graph g
         if (hintsMap.has(Routing.BLOCK_AREA)) {
             String blockAreaStr = hintsMap.get(Parameters.Routing.BLOCK_AREA, "");
             GraphEdgeIdFinder.BlockArea blockArea = new GraphEdgeIdFinder(graph, locationIndex).
-                    parseBlockArea(blockAreaStr, new DefaultEdgeFilter(encoder));
+                    parseBlockArea(blockAreaStr, new DefaultEdgeFilter(encoder.getAccessEncodedValue()));
             return new BlockAreaWeighting(weighting, blockArea);
         }
 
@@ -922,6 +945,10 @@ public Weighting createWeighting(HintsMap hintsMap, FlagEncoder encoder, Graph g
      */
     public Weighting createTurnWeighting(Graph graph, Weighting weighting, TraversalMode tMode) {
         FlagEncoder encoder = weighting.getFlagEncoder();
+        // TODO NOW For new encoding approach we need different turn cost support?
+        if (encoder == null)
+            return weighting;
+
         if (encoder.supports(TurnWeighting.class) && !tMode.equals(TraversalMode.NODE_BASED))
             return new TurnWeighting(weighting, (TurnCostExtension) graph.getExtension());
         return weighting;
@@ -986,19 +1013,22 @@ public List<Path> calcPaths(GHRequest request, GHResponse ghRsp) {
 
             RoutingTemplate routingTemplate;
             if (ROUND_TRIP.equalsIgnoreCase(algoStr))
-                routingTemplate = new RoundTripRoutingTemplate(request, ghRsp, locationIndex, maxRoundTripRetries);
+                routingTemplate = new RoundTripRoutingTemplate(request, ghRsp, encodingManager, locationIndex, maxRoundTripRetries);
             else if (ALT_ROUTE.equalsIgnoreCase(algoStr))
-                routingTemplate = new AlternativeRoutingTemplate(request, ghRsp, locationIndex);
+                routingTemplate = new AlternativeRoutingTemplate(request, ghRsp, encodingManager, locationIndex);
             else
-                routingTemplate = new ViaRoutingTemplate(request, ghRsp, locationIndex);
+                routingTemplate = new ViaRoutingTemplate(request, ghRsp, encodingManager, locationIndex);
 
+            // TODO NOW how can we know the weighting (that needs the QueryGraph) before the lookup?
+            // then we can easily call weighting.createEdgeFilter(true, true) instead of:
+            EdgeFilter edgeFilter = vehicle.equals(EncodingManager.ENCODER_NAME) ? EdgeFilter.ALL_EDGES : new DefaultEdgeFilter(encoder.getAccessEncodedValue());
             List<Path> altPaths = null;
             int maxRetries = routingTemplate.getMaxRetries();
             Locale locale = request.getLocale();
             Translation tr = trMap.getWithFallBack(locale);
             for (int i = 0; i < maxRetries; i++) {
                 StopWatch sw = new StopWatch().start();
-                List<QueryResult> qResults = routingTemplate.lookup(points, encoder);
+                List<QueryResult> qResults = routingTemplate.lookup(points, edgeFilter);
                 ghRsp.addDebugInfo("idLookup:" + sw.stop().getSeconds() + "s");
                 if (ghRsp.hasErrors())
                     return Collections.emptyList();
@@ -1202,6 +1232,12 @@ protected void cleanUp() {
     protected void flush() {
         logger.info("flushing graph " + ghStorage.toString() + ", details:" + ghStorage.toDetailsString() + ", "
                 + Helper.getMemInfo() + ")");
+        try {
+            if (json != GHJson.EMPTY)
+                json.toJson(encodingManager, new FileWriter(ghLocation + "/storage.json"));
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
         ghStorage.flush();
         logger.info("flushed graph " + Helper.getMemInfo() + ")");
         fullyLoaded = true;

core/src/main/java/com/graphhopper/coll/GHTreeMapComposed.java

@@ -24,25 +24,24 @@
 /**
  * A priority queue implemented by a TreeMap. As the tree map does not allow duplicated we compose
  * the key via priority | nodeId.
- * <p>
  *
  * @author Peter Karich
  */
 public class GHTreeMapComposed {
     private static final Integer NOT_EMPTY = new Integer(-3);
-    private final BitUtil bitUtil = BitUtil.BIG;
+    private final BitUtil bitUtil = BitUtil.LITTLE;
     private final TreeMap<Long, Integer> map;
 
     public GHTreeMapComposed() {
-        map = new TreeMap<Long, Integer>();
+        map = new TreeMap<>();
     }
 
     public void clear() {
         map.clear();
     }
 
     void remove(int key, int value) {
-        long v = bitUtil.toLong(value, key);
+        long v = bitUtil.toLong(key, value);
         if (!map.remove(v).equals(NOT_EMPTY)) {
             throw new IllegalStateException("cannot remove key " + key + " with value " + value
                     + " - did you insert " + key + "," + value + " before?");
@@ -55,7 +54,7 @@ public void update(int key, int oldValue, int value) {
     }
 
     public void insert(int key, int value) {
-        long v = bitUtil.toLong(value, key);
+        long v = bitUtil.toLong(key, value);
         map.put(v, NOT_EMPTY);
     }
 

core/src/main/java/com/graphhopper/json/EncodingManagerDeserializer.java

@@ -0,0 +1,53 @@
+package com.graphhopper.json;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.graphhopper.routing.profiles.*;
+import com.graphhopper.routing.util.EncodingManager;
+import com.graphhopper.routing.util.FlagEncoderFactory;
+
+import java.io.IOException;
+
+class EncodingManagerDeserializer extends JsonDeserializer<EncodingManager> {
+    @Override
+    public EncodingManager deserialize(JsonParser parser, DeserializationContext ctxt) throws IOException {
+        final JsonNode node = parser.getCodec().readTree(parser);
+        final ObjectMapper mapper = (ObjectMapper) parser.getCodec();
+        int extendedDataSize = node.get("extended_data_size").asInt();
+        EncodingManager.Builder emBuilder = new EncodingManager.Builder(extendedDataSize);
+
+        // 1. loop over encoded values
+        for (JsonNode encNode : node.get("encoded_values")) {
+            String classType = encNode.get("class_type").asText();
+            switch (classType) {
+                case "BooleanEncodedValue":
+                    emBuilder.addEncodedValue(mapper.treeToValue(encNode, BooleanEncodedValue.class));
+                    break;
+                case "IntEncodedValue":
+                    emBuilder.addEncodedValue(mapper.treeToValue(encNode, IntEncodedValue.class));
+                    break;
+                case "DecimalEncodedValue":
+                    emBuilder.addEncodedValue(mapper.treeToValue(encNode, DecimalEncodedValue.class));
+                    break;
+                case "MappedEncodedValue":
+                    emBuilder.addEncodedValue(mapper.treeToValue(encNode, MappedDecimalEncodedValue.class));
+                    break;
+                case "StringEncodedValue":
+                    emBuilder.addEncodedValue(mapper.treeToValue(encNode, StringEncodedValue.class));
+                    break;
+                default:
+                    throw new IllegalStateException("Unknown EncodedValue type " + classType);
+            }
+        }
+
+        // 2. init old flag encoders; custom FlagEncoderFactory is only possible when configuring; do not add EncodedValues, just check for there existence
+        // TODO NOW check config vs. serialized encoded-values.json somehow
+        int bytesForEdgeFlags = node.get("bits_for_edge_flags").asInt() / 8;
+        String fesString = node.get("flag_encoder_details_list").asText();
+        emBuilder.addAll(FlagEncoderFactory.DEFAULT, fesString, bytesForEdgeFlags, false);
+        return emBuilder.build();
+    }
+}

core/src/main/java/com/graphhopper/json/GHModule.java

@@ -0,0 +1,10 @@
+package com.graphhopper.json;
+
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.graphhopper.routing.util.EncodingManager;
+
+public class GHModule extends SimpleModule {
+    public GHModule() {
+        addDeserializer(EncodingManager.class, new EncodingManagerDeserializer());
+    }
+}

core/src/main/java/com/graphhopper/reader/DataReader.java

@@ -18,7 +18,6 @@
 package com.graphhopper.reader;
 
 import com.graphhopper.reader.dem.ElevationProvider;
-import com.graphhopper.routing.util.EncodingManager;
 
 import java.io.File;
 import java.io.IOException;