Move SpatialRuleLookup benchmark into Measurement class (#1894)

* Remove duplicate tags

* Add `jackson-datatype-jts` as a dependency

* Remove junit performance test

* Use fitting units in the report

* Measure SpatialRuleLookup performance

Co-authored-by: Thomas Butz <thomas.butz@optitool.de>

benchmark/benchmark.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 # usage:
-# benchmark/benchmark.sh <data_dir> <results_dir> <small_osm_map_path> <big_osm_map_path> <use_measurement_time_as_ref_time>
+# benchmark/benchmark.sh <data_dir> <results_dir> <small_osm_map_path> <big_osm_map_path> <use_measurement_time_as_ref_time> <spatial_rule_borders_dir>
 #
 # where:
 # <data_dir> = base directory used to store results
 # <results_dir> = name of directory where results of this run are stored (inside <data_dir>)
 # <small_osm_map_path> = path to osm map the measurement is run on for slow measurements
 # <big_osm_map_path> = path to osm map the measurement is run on for fast measurements
 # <use_measurement_time_as_ref_time> = true/false, false by default, meaning the git commit time will be used as reference
+# <spatial_rule_borders_dir> = base directory to load borders of spatial rules from
 
 # make this script exit if a command fails, a variable is missing etc.
 set -euo pipefail
@@ -17,6 +18,7 @@ defaultSingleResultsDir=measurements/results/$(date '+%d-%m-%Y-%s%N')/
 defaultSmallMap=core/files/andorra.osm.pbf
 defaultBigMap=core/files/andorra.osm.pbf
 defaultUseMeasurementTimeAsRefTime=false
+defaultBordersDirectory=core/files/spatialrules
 
 # this is the directory where we read/write data from/to
 DATA_DIR=${1:-$defaultDataDir}
@@ -26,6 +28,7 @@ SINGLE_RESULTS_DIR=${2:-$defaultSingleResultsDir}
 SMALL_OSM_MAP=${3:-$defaultSmallMap}
 BIG_OSM_MAP=${4:-$defaultBigMap}
 USE_MEASUREMENT_TIME_AS_REF_TIME=${5:-$defaultUseMeasurementTimeAsRefTime}
+BORDERS_DIRECTORY=${6:-$defaultBordersDirectory}
 
 # create directories
 mkdir -p ${DATA_DIR}
@@ -51,6 +54,7 @@ measurement.lm=false \
 graph.location=${TMP_DIR}measurement-small-gh \
 prepare.min_network_size=10000 \
 prepare.min_oneway_network_size=10000 \
+spatial_rules.borders_directory=${BORDERS_DIRECTORY} \
 measurement.json=true \
 measurement.count=5000 \
 measurement.use_measurement_time_as_ref_time=${USE_MEASUREMENT_TIME_AS_REF_TIME}
@@ -72,6 +76,7 @@ measurement.lm=true \
 graph.location=${TMP_DIR}measurement-big-gh \
 prepare.min_network_size=10000 \
 prepare.min_oneway_network_size=10000 \
+spatial_rules.borders_directory=${BORDERS_DIRECTORY} \
 measurement.json=true \
 measurement.count=5000 \
 measurement.use_measurement_time_as_ref_time=${USE_MEASUREMENT_TIME_AS_REF_TIME}

core/src/main/java/com/graphhopper/util/MiniPerfTest.java

@@ -28,11 +28,16 @@
  * @author Peter Karich
  */
 public abstract class MiniPerfTest {
+
+    private static final double NS_PER_S  = 1e9;
+    private static final double NS_PER_MS = 1e6;
+    private static final double NS_PER_US = 1e3;
+
     protected Logger logger = LoggerFactory.getLogger(getClass());
     private int counts = 100;
-    private double fullTime = 0;
-    private double max;
-    private double min = Double.MAX_VALUE;
+    private long fullTime = 0;
+    private long max;
+    private long min = Long.MAX_VALUE;
     private int dummySum;
 
     public MiniPerfTest start() {
@@ -65,21 +70,21 @@ public MiniPerfTest setIterations(int counts) {
      * @return minimum time of every call, in ms
      */
     public double getMin() {
-        return min / 1e6;
+        return min / NS_PER_MS;
     }
 
     /**
      * @return maximum time of every calls, in ms
      */
     public double getMax() {
-        return max / 1e6;
+        return max / NS_PER_MS;
     }
 
     /**
      * @return time for all calls accumulated, in ms
      */
     public double getSum() {
-        return fullTime / 1e6;
+        return fullTime / NS_PER_MS;
     }
 
     /**
@@ -88,16 +93,33 @@ public double getSum() {
     public double getMean() {
         return getSum() / counts;
     }
+
+    private String formatDuration(double durationNs) {
+        double divisor;
+        String unit;
+        if (durationNs > 1e7d) {
+            divisor = NS_PER_S;
+            unit = "s";
+        } else if (durationNs > 1e4d) {
+            divisor = NS_PER_MS;
+            unit = "ms";
+        } else {
+            divisor = NS_PER_US;
+            unit = "us";
+        }
+        return nf(durationNs / divisor) + unit;
+    }
 
     public String getReport() {
-        return "sum:" + nf(getSum() / 1000f) + "s, time/call:" + nf(getMean() / 1000f) + "s";
+        double meanNs = ((double) fullTime) / counts;
+        return "sum:" + formatDuration(fullTime) + ", time/call:" + formatDuration(meanNs);
     }
 
     public int getDummySum() {
         return dummySum;
     }
 
-    public String nf(Number num) {
+    private String nf(Number num) {
         return new DecimalFormat("#.###", DecimalFormatSymbols.getInstance(Locale.ROOT)).format(num);
     }
 

core/src/test/java/com/graphhopper/routing/util/spatialrules/SpatialRuleLookupBuilderTest.java

@@ -24,7 +24,6 @@
 import com.graphhopper.routing.util.CarFlagEncoder;
 import com.graphhopper.routing.util.EncodingManager;
 import com.graphhopper.routing.util.parsers.SpatialRuleParser;
-import com.graphhopper.routing.util.spatialrules.SpatialRuleLookupBuilder.SpatialRuleFactory;
 import com.graphhopper.routing.util.spatialrules.countries.GermanySpatialRule;
 import com.graphhopper.storage.Graph;
 import com.graphhopper.storage.GraphBuilder;
@@ -41,8 +40,6 @@
 import java.io.FileReader;
 import java.io.IOException;
 import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 import static com.graphhopper.util.GHUtility.updateDistancesFor;
 import static junit.framework.TestCase.assertFalse;
@@ -210,50 +207,4 @@ public Envelope getBounds() {
         e4.setFlags(em.handleWayTags(livingStreet2, map, relFlags));
         assertEquals(MaxSpeed.UNSET_SPEED, e4.get(tmpCarMaxSpeedEnc), .1);
     }
-
-    @Test
-    public void testSpeed() throws IOException {
-        final FileReader reader = new FileReader(COUNTRIES_FILE);
-        SpatialRuleFactory rulePerCountryFactory = new SpatialRuleFactory() {
-
-            @Override
-            public SpatialRule createSpatialRule(final String id, final List<Polygon> borders) {
-                return new AbstractSpatialRule(borders) {
-
-                    @Override
-                    public double getMaxSpeed(RoadClass roadClass, TransportationMode transport, double currentMaxSpeed) {
-                        return 100;
-                    }
-
-                    @Override
-                    public RoadAccess getAccess(RoadClass roadClass, TransportationMode transport, RoadAccess currentRoadAccess) {
-                        return RoadAccess.YES;
-                    }
-
-                    @Override
-                    public String getId() {
-                        return id;
-                    }
-                };
-            }
-        };
-        SpatialRuleLookup spatialRuleLookup = SpatialRuleLookupBuilder.buildIndex(Collections.singletonList(
-                Jackson.newObjectMapper().readValue(reader, JsonFeatureCollection.class)), "ISO_A3", rulePerCountryFactory);
-
-        // generate random points in central Europe
-        int randomPoints = 250_000;
-        long start = System.nanoTime();
-        for (int i = 0; i < randomPoints; i++) {
-            double lat = 46d + Math.random() * 7d;
-            double lon = 6d + Math.random() * 21d;
-            spatialRuleLookup.lookupRule(new GHPoint(lat, lon));
-        }
-
-        long duration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
-        // System.out.println("Lookup of " + randomPoints + " points took " + duration + "ms");
-        // System.out.println("Average lookup duration: " + ((double) duration) / randomPoints + "ms");
-        long maxBenchmarkRuntimeMs = 5_000L;
-        assertTrue("Benchmark must be finished in less than " + maxBenchmarkRuntimeMs + "ms",
-                duration < maxBenchmarkRuntimeMs);
-    }
 }

tools/pom.xml

@@ -3,9 +3,7 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>com.graphhopper</groupId>
     <artifactId>graphhopper-tools</artifactId>
-    <version>1.0-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>GraphHopper Tools</name>
 
@@ -38,6 +36,10 @@
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.graphhopper.external</groupId>
+            <artifactId>jackson-datatype-jts</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>

tools/src/main/java/com/graphhopper/tools/Measurement.java

@@ -17,30 +17,42 @@
  */
 package com.graphhopper.tools;
 
+import com.bedatadriven.jackson.datatype.jts.JtsModule;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.graphhopper.*;
 import com.graphhopper.coll.GHBitSet;
 import com.graphhopper.coll.GHBitSetImpl;
 import com.graphhopper.config.CHProfileConfig;
 import com.graphhopper.config.LMProfileConfig;
 import com.graphhopper.config.ProfileConfig;
+import com.graphhopper.json.geo.JsonFeatureCollection;
 import com.graphhopper.reader.DataReader;
 import com.graphhopper.reader.osm.GraphHopperOSM;
 import com.graphhopper.routing.util.*;
+import com.graphhopper.routing.util.spatialrules.*;
+import com.graphhopper.routing.util.spatialrules.SpatialRuleLookupBuilder.SpatialRuleFactory;
 import com.graphhopper.storage.*;
 import com.graphhopper.storage.index.LocationIndex;
 import com.graphhopper.util.*;
 import com.graphhopper.util.Parameters.Algorithms;
 import com.graphhopper.util.Parameters.CH;
 import com.graphhopper.util.Parameters.Landmark;
 import com.graphhopper.util.shapes.BBox;
+import com.graphhopper.util.shapes.GHPoint;
+
+import org.locationtech.jts.geom.Polygon;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.*;
@@ -74,6 +86,7 @@ public static void main(String[] strs) throws IOException {
     // Every value is one y-value in a separate diagram with an identical x-value for every Measurement.start call
     void start(PMap args) throws IOException {
         final String graphLocation = args.get("graph.location", "");
+        final String countryBordersDirectory = args.get("spatial_rules.borders_directory", "");
         final boolean useJson = args.getBool("measurement.json", false);
         boolean cleanGraph = args.getBool("measurement.clean", false);
         String summaryLocation = args.get("measurement.summaryfile", "");
@@ -232,6 +245,10 @@ protected DataReader importData() throws IOException {
                             edgeBased().withInstructions().withPointHints().simplify());
                 }
             }
+            if (!isEmpty(countryBordersDirectory)) {
+                printSpatialRuleLookupTest(countryBordersDirectory, count * 100);
+            }
+
         } catch (Exception ex) {
             logger.error("Problem while measuring " + graphLocation, ex);
             put("error", ex.toString());
@@ -456,6 +473,57 @@ public int doCalc(boolean warmup, int run) {
         }.setIterations(count).start();
         print("unit_tests" + description + ".get_edge_state", miniPerf);
     }
+
+    private void printSpatialRuleLookupTest(String countryBordersDirectory, int count) {
+        ObjectMapper objectMapper = new ObjectMapper();
+        objectMapper.registerModule(new JtsModule());
+        objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+
+        List<JsonFeatureCollection> jsonFeatureCollections = new ArrayList<>();
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(countryBordersDirectory), "*.{geojson,json}")) {
+            for (Path borderFile : stream) {
+                try (BufferedReader reader = Files.newBufferedReader(borderFile, StandardCharsets.UTF_8)) {
+                    JsonFeatureCollection jsonFeatureCollection = objectMapper.readValue(reader, JsonFeatureCollection.class);
+                    jsonFeatureCollections.add(jsonFeatureCollection);
+                }
+            }
+        } catch (IOException e) {
+            logger.error("Failed to load borders.", e);
+            return;
+        }
+
+        SpatialRuleFactory rulePerCountryFactory = new SpatialRuleFactory() {
+
+            @Override
+            public SpatialRule createSpatialRule(final String id, final List<Polygon> borders) {
+                return new AbstractSpatialRule(borders) {
+                    @Override
+                    public String getId() {
+                        return id;
+                    }
+                };
+            }
+        };
+
+        final SpatialRuleLookup spatialRuleLookup = SpatialRuleLookupBuilder.buildIndex(jsonFeatureCollections, "ISO_A3", rulePerCountryFactory);
+
+        // generate random points in central Europe
+        final List<GHPoint> randomPoints = new ArrayList<>(count);
+        for (int i = 0; i < count; i++) {
+            double lat = 46d + Math.random() * 7d;
+            double lon = 6d + Math.random() * 21d;
+            randomPoints.add(new GHPoint(lat, lon));
+        }
+
+        MiniPerfTest lookupPerfTest = new MiniPerfTest() {
+            @Override
+            public int doCalc(boolean warmup, int run) {
+                return spatialRuleLookup.lookupRule(randomPoints.get(run)).hashCode();
+            }
+        }.setIterations(count).start();
+
+        print("spatialrulelookup", lookupPerfTest);
+    }
 
     private void compareRouting(final GraphHopper hopper, String vehicle, int count) {
         logger.info("Comparing " + count + " routes. Differences will be printed to stderr.");