Generifying SQL sources and time series

src/main/java/edu/ie3/datamodel/exceptions/InvalidColumnNameException.java

@@ -0,0 +1,26 @@
+/*
+ * © 2021. TU Dortmund University,
+ * Institute of Energy Systems, Energy Efficiency and Energy Economics,
+ * Research group Distribution grid planning and operation
+*/
+package edu.ie3.datamodel.exceptions;
+
+/**
+ * Exception that is thrown whenever data columns are not as expected.
+ *
+ * @version 0.1
+ * @since 10.12.20
+ */
+public class InvalidColumnNameException extends RuntimeException {
+  public InvalidColumnNameException(final String message, final Throwable cause) {
+    super(message, cause);
+  }
+
+  public InvalidColumnNameException(final Throwable cause) {
+    super(cause);
+  }
+
+  public InvalidColumnNameException(final String message) {
+    super(message);
+  }
+}

src/main/java/edu/ie3/datamodel/io/csv/timeseries/ColumnScheme.java

@@ -5,6 +5,7 @@
 */
 package edu.ie3.datamodel.io.csv.timeseries;
 
+import edu.ie3.datamodel.exceptions.SourceException;
 import edu.ie3.datamodel.models.value.*;
 import edu.ie3.util.StringUtils;
 import java.util.Arrays;
@@ -31,6 +32,27 @@ public String getScheme() {
     return scheme;
   }
 
+  public Class<? extends Value> getValueClass() throws SourceException {
+    switch (this) {
+      case ACTIVE_POWER:
+        return PValue.class;
+      case APPARENT_POWER:
+        return SValue.class;
+      case ENERGY_PRICE:
+        return EnergyPriceValue.class;
+      case APPARENT_POWER_AND_HEAT_DEMAND:
+        return HeatAndSValue.class;
+      case ACTIVE_POWER_AND_HEAT_DEMAND:
+        return HeatAndPValue.class;
+      case HEAT_DEMAND:
+        return HeatDemandValue.class;
+      case WEATHER:
+        return WeatherValue.class;
+      default:
+        throw new SourceException("Unknown column scheme '" + this + "'.");
+    }
+  }
+
   public static Optional<ColumnScheme> parse(String key) {
     String cleanString = StringUtils.cleanString(key).toLowerCase();
     return Arrays.stream(ColumnScheme.values())

src/main/java/edu/ie3/datamodel/io/source/TimeSeriesSource.java

@@ -5,10 +5,14 @@
 */
 package edu.ie3.datamodel.io.source;
 
+import static edu.ie3.datamodel.io.csv.timeseries.ColumnScheme.*;
+
+import edu.ie3.datamodel.io.csv.timeseries.ColumnScheme;
 import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries;
 import edu.ie3.datamodel.models.value.Value;
 import edu.ie3.util.interval.ClosedInterval;
 import java.time.ZonedDateTime;
+import java.util.EnumSet;
 import java.util.Optional;
 
 /**
@@ -17,6 +21,15 @@
  */
 public interface TimeSeriesSource<V extends Value> extends DataSource {
 
+  EnumSet<ColumnScheme> acceptableSchemes =
+      EnumSet.of(
+          ACTIVE_POWER,
+          APPARENT_POWER,
+          ENERGY_PRICE,
+          APPARENT_POWER_AND_HEAT_DEMAND,
+          ACTIVE_POWER_AND_HEAT_DEMAND,
+          HEAT_DEMAND);
+
   /**
    * Obtain the full time series
    *

src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java

@@ -44,77 +44,30 @@ public static CsvTimeSeriesSource<? extends Value> getSource(
       FileNamingStrategy fileNamingStrategy,
       CsvFileConnector.CsvIndividualTimeSeriesMetaInformation metaInformation)
       throws SourceException {
-    switch (metaInformation.getColumnScheme()) {
-      case ACTIVE_POWER:
-        TimeBasedSimpleValueFactory<PValue> pValueFactory =
-            new TimeBasedSimpleValueFactory<>(PValue.class);
-        return new CsvTimeSeriesSource<>(
-            csvSep,
-            folderPath,
-            fileNamingStrategy,
-            metaInformation.getUuid(),
-            metaInformation.getFullFilePath(),
-            PValue.class,
-            pValueFactory);
-      case APPARENT_POWER:
-        TimeBasedSimpleValueFactory<SValue> sValueFactory =
-            new TimeBasedSimpleValueFactory<>(SValue.class);
-        return new CsvTimeSeriesSource<>(
-            csvSep,
-            folderPath,
-            fileNamingStrategy,
-            metaInformation.getUuid(),
-            metaInformation.getFullFilePath(),
-            SValue.class,
-            sValueFactory);
-      case ENERGY_PRICE:
-        TimeBasedSimpleValueFactory<EnergyPriceValue> energyPriceFactory =
-            new TimeBasedSimpleValueFactory<>(EnergyPriceValue.class);
-        return new CsvTimeSeriesSource<>(
-            csvSep,
-            folderPath,
-            fileNamingStrategy,
-            metaInformation.getUuid(),
-            metaInformation.getFullFilePath(),
-            EnergyPriceValue.class,
-            energyPriceFactory);
-      case APPARENT_POWER_AND_HEAT_DEMAND:
-        TimeBasedSimpleValueFactory<HeatAndSValue> heatAndSValueFactory =
-            new TimeBasedSimpleValueFactory<>(HeatAndSValue.class);
-        return new CsvTimeSeriesSource<>(
-            csvSep,
-            folderPath,
-            fileNamingStrategy,
-            metaInformation.getUuid(),
-            metaInformation.getFullFilePath(),
-            HeatAndSValue.class,
-            heatAndSValueFactory);
-      case ACTIVE_POWER_AND_HEAT_DEMAND:
-        TimeBasedSimpleValueFactory<HeatAndPValue> heatAndPValueFactory =
-            new TimeBasedSimpleValueFactory<>(HeatAndPValue.class);
-        return new CsvTimeSeriesSource<>(
-            csvSep,
-            folderPath,
-            fileNamingStrategy,
-            metaInformation.getUuid(),
-            metaInformation.getFullFilePath(),
-            HeatAndPValue.class,
-            heatAndPValueFactory);
-      case HEAT_DEMAND:
-        TimeBasedSimpleValueFactory<HeatDemandValue> heatDemandValueFactory =
-            new TimeBasedSimpleValueFactory<>(HeatDemandValue.class);
-        return new CsvTimeSeriesSource<>(
-            csvSep,
-            folderPath,
-            fileNamingStrategy,
-            metaInformation.getUuid(),
-            metaInformation.getFullFilePath(),
-            HeatDemandValue.class,
-            heatDemandValueFactory);
-      default:
-        throw new SourceException(
-            "Unsupported column scheme '" + metaInformation.getColumnScheme() + "'.");
-    }
+    if (!acceptableSchemes.contains(metaInformation.getColumnScheme()))
+      throw new SourceException(
+          "Unsupported column scheme '" + metaInformation.getColumnScheme() + "'.");
+
+    Class<? extends Value> valClass = metaInformation.getColumnScheme().getValueClass();
+
+    return create(csvSep, folderPath, fileNamingStrategy, metaInformation, valClass);
+  }
+
+  private static <T extends Value> CsvTimeSeriesSource<T> create(
+      String csvSep,
+      String folderPath,
+      FileNamingStrategy fileNamingStrategy,
+      CsvFileConnector.CsvIndividualTimeSeriesMetaInformation metaInformation,
+      Class<T> valClass) {
+    TimeBasedSimpleValueFactory<T> valueFactory = new TimeBasedSimpleValueFactory<>(valClass);
+    return new CsvTimeSeriesSource<>(
+        csvSep,
+        folderPath,
+        fileNamingStrategy,
+        metaInformation.getUuid(),
+        metaInformation.getFullFilePath(),
+        valClass,
+        valueFactory);
   }
 
   /**

src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java

@@ -0,0 +1,115 @@
+/*
+ * © 2021. TU Dortmund University,
+ * Institute of Energy Systems, Energy Efficiency and Energy Economics,
+ * Research group Distribution grid planning and operation
+*/
+package edu.ie3.datamodel.io.source.sql;
+
+import edu.ie3.datamodel.exceptions.InvalidColumnNameException;
+import edu.ie3.datamodel.io.connectors.SqlConnector;
+import edu.ie3.datamodel.io.source.csv.CsvDataSource;
+import edu.ie3.util.StringUtils;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class SqlDataSource<T> {
+
+  protected static final Logger log = LoggerFactory.getLogger(CsvDataSource.class);
+
+  private final SqlConnector connector;
+
+  public SqlDataSource(SqlConnector connector) {
+    this.connector = connector;
+  }
+
+  /**
+   * Determine the corresponding database column name based on the provided factory field parameter
+   * name. Needed to support camel as well as snake case database column names.
+   *
+   * @param factoryColumnName the name of the field parameter set in the entity factory
+   * @param connector the sql connector of this source
+   * @param tableName the table name where the data is stored
+   * @return the column name that corresponds to the provided field parameter or an empty optional
+   *     if no matching column can be found
+   */
+  protected static String getDbColumnName(
+      String factoryColumnName, SqlConnector connector, String tableName) {
+    try {
+      ResultSet rs =
+          connector.getConnection().getMetaData().getColumns(null, null, tableName, null);
+
+      while (rs.next()) {
+        String databaseColumnName = rs.getString("COLUMN_NAME");
+        if (StringUtils.snakeCaseToCamelCase(databaseColumnName)
+            .equalsIgnoreCase(factoryColumnName)) {
+          return databaseColumnName;
+        }
+      }
+    } catch (SQLException ex) {
+      log.error(
+          "Cannot connect to database to retrieve db column name for factory column name '{}' in table '{}'",
+          factoryColumnName,
+          tableName,
+          ex);
+    }
+    throw new InvalidColumnNameException(
+        "Cannot find column for '"
+            + factoryColumnName
+            + "' in provided times series data configuration."
+            + "Please ensure that the database connection is working and the column names are correct!");
+  }
+
+  interface AddParams {
+    /**
+     * Enhance a PreparedStatement by inserting parameters for wildcards
+     *
+     * @param ps the PreparedStatement to enhance
+     * @throws SQLException if anything goes wrong during preparation of the query
+     */
+    void addParams(PreparedStatement ps) throws SQLException;
+  }
+
+  /**
+   * Executes the prepared statement after possibly adding parameters to the query using the given
+   * function. Finally, processes the results and creates a list of time based values via field map
+   * extraction.
+   *
+   * @param query the query to use
+   * @param addParams function that possibly adds parameters to query
+   * @return a list of resulting entities
+   */
+  protected List<T> executeQuery(String query, AddParams addParams) {
+    try (PreparedStatement ps = connector.getConnection().prepareStatement(query)) {
+      addParams.addParams(ps);
+
+      ResultSet resultSet = ps.executeQuery();
+      List<Map<String, String>> fieldMaps = connector.extractFieldMaps(resultSet);
+
+      return fieldMaps.stream()
+          .map(this::createEntity)
+          .flatMap(o -> o.map(Stream::of).orElseGet(Stream::empty))
+          .collect(Collectors.toList());
+    } catch (SQLException e) {
+      log.error("Error during execution of query " + query, e);
+    }
+
+    return Collections.emptyList();
+  }
+
+  /**
+   * Instantiates an entity produced by this source given the required field value map.
+   *
+   * @param fieldToValues map of fields to their respective values
+   * @return the entity if instantiation succeeds
+   */
+  protected abstract Optional<T> createEntity(Map<String, String> fieldToValues);
+}