Implementation of sources for csv files + several bugfixes

docs/uml/main/DataSourceClassDiagramm.puml

@@ -51,29 +51,31 @@ JDBCGridDataSource -->  JDBCDataConnector
 
 class CSVGridDataSource {
 }
-CSVGridDataSource --|> GridDataSource
-CSVGridDataSource -->  CSVDataConnector
+CSVGridDataSource ..|> GridDataSource
+CSVGridDataSource ..|> CSVDataConnector
 
 interface AssetDataSource {
-{abstract}Future<AssetData> fetchAssetData()
-{abstract}Future<List<EVModel>> fetchEvs()
-{abstract}Future<List<WECModel>> fetchWecs()
-{abstract}Future<List<...>> fetch...()
+{abstract}Set<EVModel> fetchEvs()
+{abstract}Set<WECModel> fetchWecPlants()
+{abstract}Set<...> fetch...()
 }
-AssetDataSource 	--|> DataSource
+AssetDataSource 	..|> DataSource
+
+interface ThermalSource
+ThermalSource ..|> DataSource
 
 interface TypeDataSource {
 {abstract}Future<TypeData> fetchTypeData()
 {abstract}Future<List<EVTypeModel>> fetchEvTypes()
 {abstract}Future<List<...>> fetch...()
 }
-TypeDataSource 	--|> DataSource
+TypeDataSource 	..|> DataSource
 
 interface GraphicDataSource {
 {abstract}Future<GraphicData> fetchGraphicData()
 {abstract}Future<List<...>> fetch...()
 }
-GraphicDataSource 	--|> DataSource
+GraphicDataSource 	..|> DataSource
 
 interface WeatherDataSource {
 {abstract}Future<WeatherData> fetchWeatherData()

docs/uml/main/InputDataDeployment.puml

@@ -47,6 +47,7 @@ weather
 interface grid_source
 interface assets_source
 interface types_source
+interface thermal_source
 interface graphics_source
 interface weather_source
 interface time_series_source
@@ -64,6 +65,9 @@ assets_source --> assets
 types_source --> psql
 types_source --> types
 
+thermal_source --> psql
+thermal_source --> types
+
 graphics_source --> psql
 graphics_source --> graphics
 
@@ -81,6 +85,7 @@ inputAccumulator --> types_source
 inputAccumulator --> graphics_source
 inputAccumulator --> weather_source
 inputAccumulator --> time_series_source
+inputAccumulator --> thermal_source
 
 projName_model.conf --> config_source
 

docs/uml/main/InputDatamodelConcept.puml

@@ -35,6 +35,7 @@ OperatorInput --|> InputEntity
 abstract Class ConnectorInput {
     + nodeA: NodeInput
     + nodeB: NodeInput
+    + parallelDevices: Integer
 }
 ConnectorInput --|> AssetInput
 ConnectorInput ..|> HasNodes

src/main/java/edu/ie3/datamodel/exceptions/SinkException.java

@@ -12,7 +12,7 @@
  * @version 0.1
  * @since 19.03.20
  */
-public class SinkException extends RuntimeException {
+public class SinkException extends Exception {
   public SinkException(final String message, final Throwable cause) {
     super(message, cause);
   }

src/main/java/edu/ie3/datamodel/exceptions/SourceException.java

@@ -0,0 +1,27 @@
+/*
+ * © 2020. TU Dortmund University,
+ * Institute of Energy Systems, Energy Efficiency and Energy Economics,
+ * Research group Distribution grid planning and operation
+*/
+package edu.ie3.datamodel.exceptions;
+
+/**
+ * Exception that should be used whenever an error occurs in a instance of a {@link
+ * edu.ie3.datamodel.io.source.DataSource}
+ *
+ * @version 0.1
+ * @since 19.03.20
+ */
+public class SourceException extends Exception {
+  public SourceException(final String message, final Throwable cause) {
+    super(message, cause);
+  }
+
+  public SourceException(final Throwable cause) {
+    super(cause);
+  }
+
+  public SourceException(final String message) {
+    super(message);
+  }
+}

src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java

@@ -13,8 +13,10 @@
 import edu.ie3.datamodel.models.timeseries.TimeSeries;
 import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry;
 import edu.ie3.datamodel.models.value.Value;
+import java.io.*;
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.stream.Stream;
 import org.apache.logging.log4j.LogManager;
@@ -36,6 +38,8 @@ public class CsvFileConnector implements DataConnector {
   private final FileNamingStrategy fileNamingStrategy;
   private final String baseFolderName;
 
+  private static final String FILE_ENDING = ".csv";
+
   public CsvFileConnector(String baseFolderName, FileNamingStrategy fileNamingStrategy) {
     this.baseFolderName = baseFolderName;
     this.fileNamingStrategy = fileNamingStrategy;
@@ -84,21 +88,68 @@ BufferedCsvWriter getOrInitWriter(T timeSeries, String[] headerElements, String
   /**
    * Initializes a writer with the given base folder and file definition
    *
-   * @param baseFolderName Base folder, where the file hierarchy should start
+   * @param baseFolder Base folder, where the file hierarchy should start
    * @param fileDefinition Definition of the files shape
    * @return an initialized buffered writer
    * @throws ConnectorException If the base folder is a file
    * @throws IOException If the writer cannot be initialized correctly
    */
-  private BufferedCsvWriter initWriter(String baseFolderName, CsvFileDefinition fileDefinition)
+  private BufferedCsvWriter initWriter(String baseFolder, CsvFileDefinition fileDefinition)
       throws ConnectorException, IOException {
-    File basePathDir = new File(baseFolderName);
+    File basePathDir = new File(baseFolder);
     if (basePathDir.isFile())
       throw new ConnectorException(
-          "Base path dir '" + baseFolderName + "' already exists and is a file!");
+          "Base path dir '" + baseFolder + "' already exists and is a file!");
     if (!basePathDir.exists()) basePathDir.mkdirs();
 
-    return new BufferedCsvWriter(baseFolderName, fileDefinition);
+    String fullPathToFile = baseFolder + File.separator + fileDefinition.getFilePath();
+
+    File pathFile = new File(fullPathToFile);
+    if (!pathFile.exists()) {
+      return new BufferedCsvWriter(baseFolder, fileDefinition, false, true);
+    }
+    log.warn(
+        "File '{}.csv' already exist. Will append new content WITHOUT new header! Full path: {}",
+        fileDefinition.getFileName(),
+        pathFile.getAbsolutePath());
+    return new BufferedCsvWriter(baseFolder, fileDefinition, false, false);
+  }
+
+  /**
+   * Initializes a file reader for the given class that should be read in. The expected file name is
+   * determined based on {@link FileNamingStrategy} of the this {@link CsvFileConnector} instance
+   *
+   * @param clz the class of the entity that should be read
+   * @return the reader that contains information about the file to be read in
+   * @throws FileNotFoundException
+   */
+  public BufferedReader initReader(Class<? extends UniqueEntity> clz) throws FileNotFoundException {
+
+    BufferedReader newReader;
+
+    String fileName = null;
+    try {
+      fileName =
+          fileNamingStrategy
+              .getFileName(clz)
+              .orElseThrow(
+                  () ->
+                      new ConnectorException(
+                          "Cannot find a naming strategy for class '"
+                              + clz.getSimpleName()
+                              + "'."));
+    } catch (ConnectorException e) {
+      log.error(
+          "Cannot get reader for entity '{}' as no file naming strategy for this file exists. Exception:{}",
+          clz::getSimpleName,
+          () -> e);
+    }
+    File filePath = new File(baseFolderName + File.separator + fileName + FILE_ENDING);
+    newReader =
+        new BufferedReader(
+            new InputStreamReader(new FileInputStream(filePath), StandardCharsets.UTF_8), 16384);
+
+    return newReader;
   }
 
   /**

src/main/java/edu/ie3/datamodel/io/csv/BufferedCsvWriter.java

@@ -31,15 +31,16 @@ public class BufferedCsvWriter extends BufferedWriter {
    * @param quoted True, if the entries may be quoted
    * @throws IOException If the FileOutputStream cannot be established.
    */
-  public BufferedCsvWriter(String baseFolder, CsvFileDefinition fileDefinition, boolean quoted)
+  public BufferedCsvWriter(
+      String baseFolder, CsvFileDefinition fileDefinition, boolean quoted, boolean writeHeader)
       throws IOException {
     super(
         new OutputStreamWriter(
             new FileOutputStream(baseFolder + File.separator + fileDefinition.getFilePath(), false),
             StandardCharsets.UTF_8));
     this.fileDefinition = fileDefinition;
     this.quoted = quoted;
-    writeFileHeader();
+    if (writeHeader) writeFileHeader(fileDefinition.headLineElements);
   }
 
   /**
@@ -49,16 +50,17 @@ public BufferedCsvWriter(String baseFolder, CsvFileDefinition fileDefinition, bo
    * @param fileDefinition The foreseen shape of the file
    * @throws IOException If the FileOutputStream cannot be established.
    */
-  public BufferedCsvWriter(String baseFolder, CsvFileDefinition fileDefinition) throws IOException {
-    this(baseFolder, fileDefinition, true);
+  public BufferedCsvWriter(String baseFolder, CsvFileDefinition fileDefinition, boolean writeHeader)
+      throws IOException {
+    this(baseFolder, fileDefinition, false, writeHeader);
   }
 
   /**
    * Actually persisting the provided entity field data
    *
    * @param entityFieldData a mapping of an entity instance fields to their values
    */
-  public void write(Map<String, String> entityFieldData) throws IOException {
+  public void write(Map<String, String> entityFieldData) throws IOException, SinkException {
     /* Check against eligible head line elements */
     String[] eligibleHeadLineElements = fileDefinition.getHeadLineElements();
     if (entityFieldData.size() != eligibleHeadLineElements.length
@@ -77,11 +79,8 @@ public void write(Map<String, String> entityFieldData) throws IOException {
    *
    * @throws IOException If something is messed up
    */
-  private void writeFileHeader() throws IOException {
-    writeOneLine(
-        quoted
-            ? StringUtils.quote(StringUtils.camelCaseToSnakeCase(fileDefinition.headLineElements))
-            : StringUtils.camelCaseToSnakeCase(fileDefinition.headLineElements));
+  private void writeFileHeader(String[] headLineElements) throws IOException {
+    writeOneLine(StringUtils.quote(StringUtils.camelCaseToSnakeCase(headLineElements)));
   }
 
   /**

src/main/java/edu/ie3/datamodel/io/extractor/Extractor.java

@@ -7,8 +7,11 @@
 
 import edu.ie3.datamodel.exceptions.ExtractorException;
 import edu.ie3.datamodel.models.Operable;
+import edu.ie3.datamodel.models.input.AssetTypeInput;
 import edu.ie3.datamodel.models.input.InputEntity;
+import edu.ie3.datamodel.models.input.OperatorInput;
 import java.util.*;
+import java.util.concurrent.CopyOnWriteArrayList;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -27,23 +30,24 @@ private Extractor() {
     throw new IllegalStateException("Utility classes cannot be instantiated");
   }
 
-  public static List<InputEntity> extractElements(NestedEntity nestedEntity)
+  public static Set<InputEntity> extractElements(NestedEntity nestedEntity)
       throws ExtractorException {
-    List<InputEntity> resultingList = new ArrayList<>();
+    CopyOnWriteArrayList<InputEntity> resultingList = new CopyOnWriteArrayList<>();
     if (nestedEntity instanceof HasNodes) {
       resultingList.addAll(((HasNodes) nestedEntity).allNodes());
     }
+    if (nestedEntity instanceof Operable) {
+      extractOperator((Operable) nestedEntity).ifPresent(resultingList::add);
+    }
     if (nestedEntity instanceof HasType) {
-      resultingList.add(((HasType) nestedEntity).getType());
+      resultingList.add(extractType((HasType) nestedEntity));
     }
-    if (nestedEntity instanceof Operable) {
-      resultingList.add(((Operable) nestedEntity).getOperator());
+    if (nestedEntity instanceof HasThermalBus) {
+      resultingList.add(((HasThermalBus) nestedEntity).getThermalBus());
     }
-
-    if (nestedEntity instanceof HasBus) {
-      resultingList.add(((HasBus) nestedEntity).getBus());
+    if (nestedEntity instanceof HasThermalStorage) {
+      resultingList.add(((HasThermalStorage) nestedEntity).getThermalStorage());
     }
-
     if (nestedEntity instanceof HasLine) {
       resultingList.add(((HasLine) nestedEntity).getLine());
     }
@@ -54,7 +58,7 @@ public static List<InputEntity> extractElements(NestedEntity nestedEntity)
           nestedEntity.getClass().getSimpleName());
     }
 
-    if (resultingList.isEmpty()) {
+    if (resultingList.isEmpty() && !(nestedEntity instanceof Operable)) {
       throw new ExtractorException(
           "Unable to extract entity of class '"
               + nestedEntity.getClass().getSimpleName()
@@ -64,6 +68,32 @@ public static List<InputEntity> extractElements(NestedEntity nestedEntity)
               + "sub-interfaces correctly?");
     }
 
-    return Collections.unmodifiableList(resultingList);
+    resultingList.stream()
+        .parallel()
+        .forEach(
+            element -> {
+              if (element instanceof NestedEntity) {
+                try {
+                  resultingList.addAll(extractElements((NestedEntity) element));
+                } catch (ExtractorException e) {
+                  log.error(
+                      "An error occurred during extraction of nested entity '{}':{}",
+                      () -> element.getClass().getSimpleName(),
+                      () -> e);
+                }
+              }
+            });
+
+    return Collections.unmodifiableSet(new HashSet<>(resultingList));
+  }
+
+  public static AssetTypeInput extractType(HasType entityWithType) {
+    return entityWithType.getType();
+  }
+
+  public static Optional<OperatorInput> extractOperator(Operable entityWithOperator) {
+    return entityWithOperator.getOperator().getId().equalsIgnoreCase("NO_OPERATOR_ASSIGNED")
+        ? Optional.empty()
+        : Optional.of(entityWithOperator.getOperator());
   }
 }

src/main/java/edu/ie3/datamodel/io/extractor/HasBus.java → src/main/java/edu/ie3/datamodel/io/extractor/HasThermalBus.java

@@ -14,7 +14,7 @@
  * @version 0.1
  * @since 31.03.20
  */
-public interface HasBus extends NestedEntity {
+public interface HasThermalBus extends NestedEntity {
 
-  ThermalBusInput getBus();
+  ThermalBusInput getThermalBus();
 }

src/main/java/edu/ie3/datamodel/io/extractor/HasThermalStorage.java

@@ -0,0 +1,20 @@
+/*
+ * © 2020. TU Dortmund University,
+ * Institute of Energy Systems, Energy Efficiency and Energy Economics,
+ * Research group Distribution grid planning and operation
+*/
+package edu.ie3.datamodel.io.extractor;
+
+import edu.ie3.datamodel.models.input.thermal.ThermalStorageInput;
+
+/**
+ * Interface that should be implemented by all elements holding a {@link ThermalStorageInput}
+ * elements and should be processable by the {@link Extractor}.
+ *
+ * @version 0.1
+ * @since 31.03.20
+ */
+public interface HasThermalStorage {
+
+  ThermalStorageInput getThermalStorage();
+}