Merge branch 'extraction-tagtranslation' into 'master'

Optimize tag translation in data extraction requests

See merge request giscience/big-data/ohsome/ohsome-api!31

pom.xml

@@ -146,6 +146,12 @@
 			<artifactId>commons-csv</artifactId>
 			<version>${apachecommons-csv.version}</version>
 		</dependency>
+		<!-- hikari database connection pool -->
+		<dependency>
+			<groupId>com.zaxxer</groupId>
+			<artifactId>HikariCP</artifactId>
+			<version>3.4.2</version>
+		</dependency>
 
   </dependencies>
 

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/Application.java

@@ -1,7 +1,9 @@
 package org.heigit.bigspatialdata.ohsome.ohsomeapi;
 
+import com.zaxxer.hikari.HikariConfig;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import org.springframework.context.ApplicationContext;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.ProcessingData;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.DbConnData;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.RemoteTagTranslator;
@@ -31,6 +33,11 @@ public class Application implements ApplicationRunner {
   public static final int DEFAULT_NUMBER_OF_CLUSTER_NODES = 24;
   public static final int DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS = 40;
 
+  private static ApplicationContext context;
+  public static ApplicationContext getApplicationContext() {
+    return context;
+  }
+
   /** Main method to run this SpringBootApplication. */
   public static void main(String[] args) {
     if (args == null || args.length == 0) {
@@ -41,7 +48,7 @@ public static void main(String[] args) {
     }
     try {
       preRun(new DefaultApplicationArguments(args));
-      SpringApplication.run(Application.class, args);
+      context = SpringApplication.run(Application.class, args);
     } catch (Exception e) {
       e.printStackTrace();
       System.exit(1);
@@ -98,6 +105,12 @@ public static void preRun(ApplicationArguments args) throws Exception {
                 throw new RuntimeException(e);
               }
             });
+            HikariConfig hikariConfig = new HikariConfig();
+            hikariConfig.setJdbcUrl(jdbcParam[1]);
+            hikariConfig.setUsername(jdbcParam[2]);
+            hikariConfig.setPassword(jdbcParam[3]);
+            hikariConfig.setMaximumPoolSize(numberOfDataExtractionThreads);
+            DbConnData.keytablesDbPoolConfig = hikariConfig;
             break;
           case "database.multithreading":
             if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("false")) {

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java

@@ -33,7 +33,6 @@
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.DbConnData;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.ExtractMetadata;
-import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.RemoteTagTranslator;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.Description;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.dataaggregationresponse.Attribution;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.dataaggregationresponse.DefaultAggregationResponse;
@@ -120,7 +119,6 @@ public static void executeElements(ElementsGeometry elemGeom, HttpServletRequest
     }
     final MapReducer<Feature> preResult;
     ExecutionUtils exeUtils = new ExecutionUtils(processingData);
-    RemoteTagTranslator mapTagTranslator = DbConnData.mapTagTranslator;
     preResult = mapRed.map(snapshot -> {
       Map<String, Object> properties = new TreeMap<>();
       if (includeOSMMetadata) {
@@ -134,7 +132,7 @@ public static void executeElements(ElementsGeometry elemGeom, HttpServletRequest
         geom = snapshot.getGeometryUnclipped();
       }
       return exeUtils.createOSMFeature(snapshot.getEntity(), geom, properties, keysInt, includeTags,
-          includeOSMMetadata, elemGeom, mapTagTranslator.get());
+          includeOSMMetadata, elemGeom);
     }).filter(Objects::nonNull);
     Metadata metadata = null;
     if (processingData.isShowMetadata()) {
@@ -202,7 +200,6 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
     }
     MapReducer<Feature> contributionPreResult = null;
     ExecutionUtils exeUtils = new ExecutionUtils(processingData);
-    RemoteTagTranslator mapTagTranslator = DbConnData.mapTagTranslator;
     inputProcessor.processPropertiesParam();
     InputProcessingUtils utils = inputProcessor.getUtils();
     final boolean includeTags = inputProcessor.includeTags();
@@ -246,7 +243,7 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
             if (!processingData.containsSimpleFeatureTypes()
                 || utils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes)) {
               output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt,
-                  includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get()));
+                  includeTags, includeOSMMetadata, elemGeom));
             }
           }
         }
@@ -275,7 +272,7 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
           if (!processingData.containsSimpleFeatureTypes()
               || utils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes)) {
             output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt,
-                includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get()));
+                includeTags, includeOSMMetadata, elemGeom));
           }
         }
       }
@@ -304,7 +301,7 @@ public static void executeElementsFullHistory(ElementsGeometry elemGeom,
           if (!processingData.containsSimpleFeatureTypes()
               || utils.checkGeometryOnSimpleFeatures(geom, simpleFeatureTypes)) {
             return Collections.singletonList(exeUtils.createOSMFeature(entity, geom, properties,
-                keysInt, includeTags, includeOSMMetadata, elemGeom, mapTagTranslator.get()));
+                keysInt, includeTags, includeOSMMetadata, elemGeom));
           } else {
             return Collections.emptyList();
           }

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/executor/ExecutionUtils.java

@@ -9,10 +9,12 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.opencsv.CSVWriter;
+import com.zaxxer.hikari.HikariDataSource;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
+import java.sql.SQLException;
 import java.text.DecimalFormat;
 import java.text.DecimalFormatSymbols;
 import java.util.ArrayList;
@@ -46,6 +48,7 @@
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.InputProcessor;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.ProcessingData;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType;
+import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.DbConnData;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb.ExtractMetadata;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.Description;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.output.dataaggregationresponse.Attribution;
@@ -79,6 +82,7 @@
 import org.heigit.bigspatialdata.oshdb.util.OSHDBTag;
 import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
 import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
+import org.heigit.bigspatialdata.oshdb.util.exceptions.OSHDBKeytablesNotFoundException;
 import org.heigit.bigspatialdata.oshdb.util.geometry.Geo;
 import org.heigit.bigspatialdata.oshdb.util.geometry.OSHDBGeometryBuilder;
 import org.heigit.bigspatialdata.oshdb.util.tagtranslator.OSMTag;
@@ -219,7 +223,6 @@ public static <A, U extends Comparable<U> & Serializable, V extends Comparable<V
   public void streamElementsResponse(HttpServletResponse servletResponse, DataResponse osmData,
       boolean isFullHistory, Stream<org.wololo.geojson.Feature> snapshotStream,
       Stream<org.wololo.geojson.Feature> contributionStream) throws Exception {
-
     JsonFactory jsonFactory = new JsonFactory();
     ByteArrayOutputStream tempStream = new ByteArrayOutputStream();
 
@@ -245,12 +248,10 @@ public void writeIndentation(JsonGenerator g, int level) throws IOException {
             super.writeIndentation(g, level + 1);
           }
         };
-        DefaultPrettyPrinter printer = new DefaultPrettyPrinter("")
-            .withArrayIndenter(indenter)
-            .withObjectIndenter(indenter);
+        DefaultPrettyPrinter printer =
+            new DefaultPrettyPrinter("").withArrayIndenter(indenter).withObjectIndenter(indenter);
         return jsonFactory.createGenerator(outputBuffers.get(), JsonEncoding.UTF8)
-            .setCodec(objMapper)
-            .setPrettyPrinter(printer);
+            .setCodec(objMapper).setPrettyPrinter(printer);
       } catch (IOException e) {
         throw new RuntimeException(e);
       }
@@ -407,15 +408,14 @@ public List<String[]> createCsvTopComments(String url, String text, String apiVe
   /** Creates the <code>Feature</code> objects in the OSM data response. */
   public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry geometry,
       Map<String, Object> properties, int[] keysInt, boolean includeTags,
-      boolean includeOSMMetadata, ElementsGeometry elemGeom, TagTranslator tt) {
+      boolean includeOSMMetadata, ElementsGeometry elemGeom) {
     if (geometry.isEmpty()) {
       // skip invalid geometries (e.g. ways with 0 nodes)
       return null;
     }
     if (includeTags) {
       for (OSHDBTag oshdbTag : entity.getTags()) {
-        OSMTag tag = tt.getOSMTagOf(oshdbTag);
-        properties.put(tag.getKey(), tag.getValue());
+        properties.put(String.valueOf(oshdbTag.getKey()), oshdbTag);
       }
     } else if (keysInt.length != 0) {
       int[] tags = entity.getRawTags();
@@ -424,8 +424,8 @@ public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry ge
         int tagValueId = tags[i + 1];
         for (int key : keysInt) {
           if (tagKeyId == key) {
-            OSMTag tag = tt.getOSMTagOf(tagKeyId, tagValueId);
-            properties.put(tag.getKey(), tag.getValue());
+            properties.put(String.valueOf(tagKeyId), new OSHDBTag(tagKeyId, tagValueId));
+            break;
           }
         }
       }
@@ -1091,11 +1091,41 @@ private void writeStreamResponse(ThreadLocal<JsonGenerator> outputJsonGen,
       Stream<org.wololo.geojson.Feature> stream, ThreadLocal<ByteArrayOutputStream> outputBuffers,
       final ServletOutputStream outputStream)
       throws ExecutionException, InterruptedException, IOException {
+    ThreadLocal<TagTranslator> tts;
+    HikariDataSource keytablesConnectionPool;
+    if (DbConnData.keytablesDbPoolConfig != null) {
+      keytablesConnectionPool = new HikariDataSource(DbConnData.keytablesDbPoolConfig);
+      tts = ThreadLocal.withInitial(() -> {
+        try {
+          return new TagTranslator(keytablesConnectionPool.getConnection());
+        } catch (OSHDBKeytablesNotFoundException | SQLException e) {
+          throw new RuntimeException(e);
+        }
+      });
+    } else {
+      keytablesConnectionPool = null;
+      tts = ThreadLocal.withInitial(() -> DbConnData.tagTranslator);
+    }
     ReentrantLock lock = new ReentrantLock();
     AtomicBoolean errored = new AtomicBoolean(false);
     ForkJoinPool threadPool = new ForkJoinPool(ProcessingData.getNumberOfDataExtractionThreads());
     try {
       threadPool.submit(() -> stream.parallel().map(data -> {
+        // 0. resolve tags
+        Map<String, Object> tags = data.getProperties();
+        List<String> keysToDelete = new LinkedList<>();
+        List<OSMTag> tagsToAdd = new LinkedList<>();
+        for (Entry<String, Object> tag : tags.entrySet()) {
+          String key = tag.getKey();
+          if (key.charAt(0) != '@') {
+            keysToDelete.add(key);
+            tagsToAdd.add(tts.get().getOSMTagOf((OSHDBTag) tag.getValue()));
+          }
+        }
+        tags.keySet().removeAll(keysToDelete);
+        for (OSMTag tag : tagsToAdd) {
+          tags.put(tag.getKey(), tag.getValue());
+        }
         // 1. convert features to geojson
         try {
           outputBuffers.get().reset();
@@ -1133,6 +1163,9 @@ private void writeStreamResponse(ThreadLocal<JsonGenerator> outputJsonGen,
     } finally {
       threadPool.shutdown();
       outputStream.flush();
+      if (keytablesConnectionPool != null) {
+        keytablesConnectionPool.close();
+      }
     }
   }
 

src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/oshdb/DbConnData.java

@@ -1,5 +1,6 @@
 package org.heigit.bigspatialdata.ohsome.ohsomeapi.oshdb;
 
+import com.zaxxer.hikari.HikariConfig;
 import org.heigit.bigspatialdata.oshdb.api.db.OSHDBDatabase;
 import org.heigit.bigspatialdata.oshdb.api.db.OSHDBJdbc;
 import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
@@ -11,7 +12,8 @@ public class DbConnData {
   public static OSHDBJdbc keytables = null;
   public static TagTranslator tagTranslator = null;
   public static RemoteTagTranslator mapTagTranslator = null;
-
+  public static HikariConfig keytablesDbPoolConfig = null;
+
   private DbConnData() {
     throw new IllegalStateException("Utility class");
   }

src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/ElementsControllerTest.java

@@ -10,8 +10,10 @@
 import java.util.Spliterators;
 import java.util.stream.StreamSupport;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.Application;
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.springframework.boot.SpringApplication;
 import org.springframework.boot.test.web.client.TestRestTemplate;
 import org.springframework.http.ResponseEntity;
 import org.springframework.util.LinkedMultiValueMap;
@@ -36,6 +38,12 @@ public static void applicationMainStartup() {
     Application.main(params.toArray(new String[0]));
   }
 
+  /** Stops this application context. */
+  @AfterClass
+  public static void applicationMainShutdown() {
+    SpringApplication.exit(Application.getApplicationContext(), () -> 0);
+  }
+
   /*
    * ./elements/geometry tests
    */

src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/GetControllerTest.java

@@ -14,8 +14,10 @@
 import java.util.stream.StreamSupport;
 import org.apache.commons.csv.CSVRecord;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.Application;
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.springframework.boot.SpringApplication;
 import org.springframework.boot.test.web.client.TestRestTemplate;
 import org.springframework.http.ResponseEntity;
 
@@ -38,6 +40,12 @@ public static void applicationMainStartup() {
     Application.main(params.toArray(new String[0]));
   }
 
+  /** Stops this application context. */
+  @AfterClass
+  public static void applicationMainShutdown() {
+    SpringApplication.exit(Application.getApplicationContext(), () -> 0);
+  }
+
   /** Method to get response body as String */
   private String getResponseBody(String urlParams) {
     TestRestTemplate restTemplate = new TestRestTemplate();

src/test/java/org/heigit/bigspatialdata/ohsome/ohsomeapi/controller/PostControllerTest.java

@@ -13,8 +13,10 @@
 import java.util.stream.StreamSupport;
 import org.apache.commons.csv.CSVRecord;
 import org.heigit.bigspatialdata.ohsome.ohsomeapi.Application;
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.springframework.boot.SpringApplication;
 import org.springframework.boot.test.web.client.TestRestTemplate;
 import org.springframework.http.ResponseEntity;
 import org.springframework.util.LinkedMultiValueMap;
@@ -38,6 +40,12 @@ public static void applicationMainStartup() {
     Application.main(params.toArray(new String[0]));
   }
 
+  /** Stops this application context. */
+  @AfterClass
+  public static void applicationMainShutdown() {
+    SpringApplication.exit(Application.getApplicationContext(), () -> 0);
+  }
+
   /*
    * /elements/count tests
    */