From a63ded9d296a0d61be6fca60025f216766d47e24 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 29 Sep 2025 03:32:20 +0200 Subject: [PATCH 01/10] abstract --- .../io/connectors/CsvFileConnector.java | 64 +++++----- .../io/connectors/FileConnector.java | 62 ++++++++++ .../io/csv/CsvLoadProfileMetaInformation.java | 42 +------ .../FileLoadProfileMetaInformation.java | 60 ++++++++++ .../datamodel/io/source/FileDataSource.java | 95 +++++++++++++++ .../io/source/csv/CsvDataSource.java | 113 +++--------------- .../io/connectors/CsvFileConnectorTest.groovy | 6 + 7 files changed, 277 insertions(+), 165 deletions(-) create mode 100644 src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java create mode 100644 src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java create mode 100644 src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index b2d10f5dc..ac25d3e53 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -13,7 +13,12 @@ import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.value.Value; -import java.io.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.*; @@ -29,28 +34,25 @@ * @version 0.1 * @since 19.03.20 */ -public class CsvFileConnector implements DataConnector { +public class CsvFileConnector extends FileConnector { private static final Logger log = LoggerFactory.getLogger(CsvFileConnector.class); private final Map, BufferedCsvWriter> entityWriters = new HashMap<>(); private final Map timeSeriesWriters = new HashMap<>(); - private final Path baseDirectory; - private final Optional> customInputStream; private static final String FILE_ENDING = ".csv"; public CsvFileConnector(Path baseDirectory) { - this.baseDirectory = baseDirectory; - this.customInputStream = Optional.empty(); + super(baseDirectory); } public CsvFileConnector(Path baseDirectory, Function inputStreamSupplier) { - this.baseDirectory = baseDirectory; - this.customInputStream = Optional.ofNullable(inputStreamSupplier); + super(baseDirectory, inputStreamSupplier); } /** Returns the base directory of this connector. */ + @Override public Path getBaseDirectory() { - return baseDirectory; + return super.getBaseDirectory(); } public synchronized BufferedCsvWriter getOrInitWriter( @@ -61,7 +63,7 @@ public synchronized BufferedCsvWriter getOrInitWriter( /* If it is not available, build and register one */ try { - BufferedCsvWriter newWriter = initWriter(baseDirectory, fileDefinition); + BufferedCsvWriter newWriter = initWriter(getBaseDirectory(), fileDefinition); entityWriters.put(clz, newWriter); return newWriter; @@ -84,7 +86,7 @@ BufferedCsvWriter getOrInitWriter(T timeSeries, CsvFileDefinition fileDefinition /* If it is not available, build and register one */ try { - BufferedCsvWriter newWriter = initWriter(baseDirectory, fileDefinition); + BufferedCsvWriter newWriter = initWriter(getBaseDirectory(), fileDefinition); timeSeriesWriters.put(timeSeries.getUuid(), newWriter); return newWriter; @@ -94,6 +96,19 @@ BufferedCsvWriter getOrInitWriter(T timeSeries, CsvFileDefinition fileDefinition } } + /** + * Initializes a reader for the given file name. + * + * @param filePath path of file starting from base folder, including file name but not file + * extension + * @return the reader that contains information about the file to be read in + * @throws FileNotFoundException if no file with the provided file name can be found + */ + public BufferedReader initReader(Path filePath) throws FileNotFoundException { + InputStream inputStream = openInputStream(filePath); + return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), 16384); + } + /** * Initializes a writer with the given base folder and file definition * @@ -158,28 +173,6 @@ public synchronized void closeEntityWriter(Class clz) thro } } - /** - * Initializes a file reader for the given file name. - * - * @param filePath path of file starting from base folder, including file name but not file - * extension - * @return the reader that contains information about the file to be read in - * @throws FileNotFoundException if no file with the provided file name can be found - */ - public BufferedReader initReader(Path filePath) throws FileNotFoundException { - Path fullPath = baseDirectory.resolve(filePath.toString() + FILE_ENDING); - - InputStream inputStream; - - if (customInputStream.isPresent()) { - inputStream = customInputStream.get().apply(fullPath.toString()); - } else { - inputStream = new FileInputStream(fullPath.toFile()); - } - - return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), 16384); - } - @Override public void shutdown() { Stream.of(entityWriters.values(), timeSeriesWriters.values()) @@ -193,4 +186,9 @@ public void shutdown() { } }); } + + @Override + protected String getFileEnding() { + return FILE_ENDING; + } } diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java new file mode 100644 index 000000000..20d4bc6b8 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java @@ -0,0 +1,62 @@ +/* + * © 2024. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.connectors; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Optional; +import java.util.function.Function; + +/** Base connector for file-based sources and sinks. */ +public abstract class FileConnector implements DataConnector { + + private final Path baseDirectory; + private final Optional> customInputStream; + + protected FileConnector(Path baseDirectory) { + this(baseDirectory, null); + } + + protected FileConnector(Path baseDirectory, Function inputStreamSupplier) { + this.baseDirectory = baseDirectory; + this.customInputStream = Optional.ofNullable(inputStreamSupplier); + } + + /** Returns the base directory backing this connector. */ + public Path getBaseDirectory() { + return baseDirectory; + } + + /** + * Open an {@link InputStream} to the given file path (without file ending) relative to the base + * directory. + */ + protected InputStream openInputStream(Path filePath) throws FileNotFoundException { + Path fullPath = resolveFilePath(filePath); + if (customInputStream.isPresent()) { + return customInputStream.get().apply(fullPath.toString()); + } + return new FileInputStream(fullPath.toFile()); + } + + /** Resolve the path including the file ending relative to the base directory. */ + protected Path resolveFilePath(Path filePath) { + String relativePath = filePath.toString(); + if (!relativePath.endsWith(getFileEnding())) { + relativePath = relativePath + getFileEnding(); + } + return baseDirectory.resolve(relativePath); + } + + /** Returns the file ending (including the dot) handled by this connector. */ + protected abstract String getFileEnding(); + + @Override + public void shutdown() { + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java index 905f1ee3f..5d093a848 100644 --- a/src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java @@ -5,51 +5,17 @@ */ package edu.ie3.datamodel.io.csv; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; import java.nio.file.Path; -import java.util.Objects; - -public class CsvLoadProfileMetaInformation extends LoadProfileMetaInformation { - private final Path fullFilePath; +public class CsvLoadProfileMetaInformation extends FileLoadProfileMetaInformation { public CsvLoadProfileMetaInformation(String profile, Path fullFilePath) { - super(profile); - this.fullFilePath = fullFilePath; + super(profile, fullFilePath); } public CsvLoadProfileMetaInformation( LoadProfileMetaInformation metaInformation, Path fullFilePath) { - this(metaInformation.getProfile(), fullFilePath); - } - - public Path getFullFilePath() { - return fullFilePath; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof CsvLoadProfileMetaInformation that)) return false; - if (!super.equals(o)) return false; - return fullFilePath.equals(that.fullFilePath); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), fullFilePath); - } - - @Override - public String toString() { - return "CsvLoadProfileMetaInformation{" - + "uuid='" - + getUuid() - + '\'' - + ", profile='" - + getProfile() - + '\'' - + "fullFilePath=" - + fullFilePath - + '}'; + super(metaInformation, fullFilePath); } } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java new file mode 100644 index 000000000..08b293f4a --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java @@ -0,0 +1,60 @@ +/* + * © 2024. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.naming.timeseries; + +import java.nio.file.Path; +import java.util.Objects; +import java.util.UUID; + +public abstract class FileLoadProfileMetaInformation extends LoadProfileMetaInformation { + private final Path fullFilePath; + + protected FileLoadProfileMetaInformation(String profile, Path fullFilePath) { + super(profile); + this.fullFilePath = Objects.requireNonNull(fullFilePath, "fullFilePath"); + } + + protected FileLoadProfileMetaInformation(UUID uuid, String profile, Path fullFilePath) { + super(uuid, profile); + this.fullFilePath = Objects.requireNonNull(fullFilePath, "fullFilePath"); + } + + protected FileLoadProfileMetaInformation( + LoadProfileMetaInformation metaInformation, Path fullFilePath) { + this(metaInformation.getUuid(), metaInformation.getProfile(), fullFilePath); + } + + public Path getFullFilePath() { + return fullFilePath; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof FileLoadProfileMetaInformation that)) return false; + if (!super.equals(o)) return false; + return fullFilePath.equals(that.fullFilePath); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), fullFilePath); + } + + @Override + public String toString() { + return "FileLoadProfileMetaInformation{" + + "uuid='" + + getUuid() + + '\'' + + ", profile='" + + getProfile() + + '\'' + + "fullFilePath=" + + fullFilePath + + '}'; + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java new file mode 100644 index 000000000..1912feec8 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java @@ -0,0 +1,95 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source; + +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.models.Entity; +import edu.ie3.datamodel.utils.Try; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class FileDataSource implements DataSource { + + protected final Logger log = LoggerFactory.getLogger(getClass()); + private final Path baseDirectory; + private final FileNamingStrategy fileNamingStrategy; + + protected FileDataSource(Path directoryPath, FileNamingStrategy fileNamingStrategy) { + this.baseDirectory = Objects.requireNonNull(directoryPath, "directoryPath"); + this.fileNamingStrategy = Objects.requireNonNull(fileNamingStrategy, "fileNamingStrategy"); + } + + @Override + public Optional> getSourceFields(Class entityClass) + throws SourceException { + return getSourceFields(getFilePath(entityClass).getOrThrow()); + } + + public abstract Optional> getSourceFields(Path filePath) throws SourceException; + + @Override + public Stream> getSourceData(Class entityClass) + throws SourceException { + return buildStreamWithFieldsToAttributesMap(entityClass, true).getOrThrow(); + } + + public abstract Stream> getSourceData(Path filePath) throws SourceException; + + public FileNamingStrategy getNamingStrategy() { + return fileNamingStrategy; + } + + protected Path getBaseDirectory() { + return baseDirectory; + } + + protected Try>, SourceException> buildStreamWithFieldsToAttributesMap( + Class entityClass, boolean allowFileNotExisting) { + return getFilePath(entityClass) + .flatMap(path -> buildStreamWithFieldsToAttributesMap(path, allowFileNotExisting)); + } + + protected abstract Try>, SourceException> + buildStreamWithFieldsToAttributesMap(Path filePath, boolean allowFileNotExisting); + + protected Try getFilePath(Class entityClass) { + return Try.from( + fileNamingStrategy.getFilePath(entityClass), + () -> + new SourceException( + "Cannot find a naming strategy for class '" + entityClass.getSimpleName() + "'.")); + } + + protected Set getTimeSeriesFilePaths(Pattern pattern) { + Path baseDirectory = getBaseDirectory(); + try (Stream pathStream = Files.walk(baseDirectory)) { + return pathStream + .map(baseDirectory::relativize) + .filter( + path -> { + Path withoutEnding = + Path.of(FileNamingStrategy.removeFileNameEnding(path.toString())); + return pattern.matcher(withoutEnding.toString()).matches(); + }) + .collect(Collectors.toSet()); + } catch (IOException e) { + log.error("Unable to determine time series files readers for time series.", e); + return Collections.emptySet(); + } + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index 6879b57eb..e3ecd9a53 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -14,7 +14,7 @@ import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; -import edu.ie3.datamodel.io.source.DataSource; +import edu.ie3.datamodel.io.source.FileDataSource; import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.profile.LoadProfile; import edu.ie3.datamodel.utils.Try; @@ -24,16 +24,17 @@ import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; -import java.util.*; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.UUID; import java.util.function.Function; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Parent class of all .csv file related sources containing methods and fields consumed by almost @@ -42,41 +43,25 @@ * @version 0.1 * @since 05.04.20 */ -public class CsvDataSource implements DataSource { +public class CsvDataSource extends FileDataSource { - protected static final Logger log = LoggerFactory.getLogger(CsvDataSource.class); - - // general fields protected final String csvSep; protected final CsvFileConnector connector; - private final FileNamingStrategy fileNamingStrategy; - public CsvDataSource(String csvSep, Path directoryPath, FileNamingStrategy fileNamingStrategy) { + super(directoryPath, fileNamingStrategy); this.csvSep = csvSep; this.connector = new CsvFileConnector(directoryPath); - this.fileNamingStrategy = fileNamingStrategy; } public CsvDataSource( String csvSep, CsvFileConnector connector, FileNamingStrategy fileNamingStrategy) { + super(connector.getBaseDirectory(), fileNamingStrategy); this.csvSep = csvSep; this.connector = connector; - this.fileNamingStrategy = fileNamingStrategy; } @Override - public Optional> getSourceFields(Class entityClass) - throws SourceException { - return getSourceFields(getFilePath(entityClass).getOrThrow()); - } - - /** - * @param filePath path of file starting from base folder, including file name but not file - * extension - * @return The source field names as a set, if file exists - * @throws SourceException on error while reading the source file - */ public Optional> getSourceFields(Path filePath) throws SourceException { try (BufferedReader reader = connector.initReader(filePath)) { String line = reader.readLine(); @@ -102,27 +87,10 @@ public Optional> getSourceFields(Path filePath) throws SourceExcepti } @Override - public Stream> getSourceData(Class entityClass) - throws SourceException { - return buildStreamWithFieldsToAttributesMap(entityClass, true).getOrThrow(); - } - - /** - * @param filePath to the csv file - * @return a stream of maps that represent the rows in the csv file - * @throws SourceException on error while reading the source file - */ public Stream> getSourceData(Path filePath) throws SourceException { return buildStreamWithFieldsToAttributesMap(filePath, true).getOrThrow(); } - // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - - /** Returns the set {@link FileNamingStrategy}. */ - public FileNamingStrategy getNamingStrategy() { - return fileNamingStrategy; - } - /** * Receive the information for specific time series. They are given back filtered by the column * scheme in order to allow for accounting the different content types. @@ -133,13 +101,13 @@ public FileNamingStrategy getNamingStrategy() { */ public Map getCsvIndividualTimeSeriesMetaInformation(final ColumnScheme... columnSchemes) { - return getTimeSeriesFilePaths(fileNamingStrategy.getIndividualTimeSeriesPattern()) - .parallelStream() + FileNamingStrategy namingStrategy = getNamingStrategy(); + return getTimeSeriesFilePaths(namingStrategy.getIndividualTimeSeriesPattern()).parallelStream() .map( filePath -> { /* Extract meta information from file path and enhance it with the file path itself */ IndividualTimeSeriesMetaInformation metaInformation = - fileNamingStrategy.individualTimeSeriesMetaInformation(filePath.toString()); + namingStrategy.individualTimeSeriesMetaInformation(filePath.toString()); return new CsvIndividualTimeSeriesMetaInformation( metaInformation, FileNamingStrategy.removeFileNameEnding(filePath.getFileName())); }) @@ -160,13 +128,13 @@ public FileNamingStrategy getNamingStrategy() { */ public Map getCsvLoadProfileMetaInformation( LoadProfile... profiles) { - return getTimeSeriesFilePaths(fileNamingStrategy.getLoadProfileTimeSeriesPattern()) - .parallelStream() + FileNamingStrategy namingStrategy = getNamingStrategy(); + return getTimeSeriesFilePaths(namingStrategy.getLoadProfileTimeSeriesPattern()).parallelStream() .map( filePath -> { /* Extract meta information from file path and enhance it with the file path itself */ LoadProfileMetaInformation metaInformation = - fileNamingStrategy.loadProfileTimeSeriesMetaInformation(filePath.toString()); + namingStrategy.loadProfileTimeSeriesMetaInformation(filePath.toString()); return new CsvLoadProfileMetaInformation( metaInformation, FileNamingStrategy.removeFileNameEnding(filePath.getFileName())); }) @@ -179,33 +147,6 @@ public Map getCsvLoadProfileMetaInformati .collect(Collectors.toMap(LoadProfileMetaInformation::getProfile, Function.identity())); } - /** - * Returns a set of relative paths strings to time series files, with respect to the base folder - * path - * - * @param pattern for matching the time series - * @return A set of relative paths to time series files, with respect to the base folder path - */ - protected Set getTimeSeriesFilePaths(Pattern pattern) { - Path baseDirectory = connector.getBaseDirectory(); - try (Stream pathStream = Files.walk(baseDirectory)) { - return pathStream - .map(baseDirectory::relativize) - .filter( - path -> { - Path withoutEnding = - Path.of(FileNamingStrategy.removeFileNameEnding(path.toString())); - return pattern.matcher(withoutEnding.toString()).matches(); - }) - .collect(Collectors.toSet()); - } catch (IOException e) { - log.error("Unable to determine time series files readers for time series.", e); - return Collections.emptySet(); - } - } - - // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - /** * Takes a row string of a .csv file and a string array of the csv file headline, tries to split * the csv row string based and zip it together with the headline. This method does not contain @@ -280,19 +221,10 @@ protected String[] parseCsvRow(String csvRow, String csvSep) { .toArray(String[]::new); } - /** - * Tries to open a file reader based on the provided entity class and hands it over for further - * processing. - * - * @param entityClass the entity class that should be build and that is used to get the - * corresponding reader - * @return a parallel stream of maps, where each map represents one row of the csv file with the - * mapping (fieldName to fieldValue) - */ + @Override protected Try>, SourceException> buildStreamWithFieldsToAttributesMap( Class entityClass, boolean allowFileNotExisting) { - return getFilePath(entityClass) - .flatMap(path -> buildStreamWithFieldsToAttributesMap(path, allowFileNotExisting)); + return super.buildStreamWithFieldsToAttributesMap(entityClass, allowFileNotExisting); } /** @@ -304,6 +236,7 @@ protected Try>, SourceException> buildStreamWithField * @return a try containing either a parallel stream of maps, where each map represents one row of * the csv file with the mapping (fieldName to fieldValue) or an exception */ + @Override protected Try>, SourceException> buildStreamWithFieldsToAttributesMap( Path filePath, boolean allowFileNotExisting) { try (BufferedReader reader = connector.initReader(filePath)) { @@ -326,14 +259,6 @@ protected Try>, SourceException> buildStreamWithField } } - private Try getFilePath(Class entityClass) { - return Try.from( - fileNamingStrategy.getFilePath(entityClass), - () -> - new SourceException( - "Cannot find a naming strategy for class '" + entityClass.getSimpleName() + "'.")); - } - /** * Method to return a row to field value mapping from a csv file. * diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy index 273993e10..c5b2735d0 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy @@ -76,6 +76,9 @@ class CsvFileConnectorTest extends Specification { noExceptionThrown() nodeFile.exists() nodeFile.file // is it a file? + + cleanup: + connector.shutdown() } def "The csv file connector is able to init writers utilizing no directory hierarchy"() { @@ -95,6 +98,9 @@ class CsvFileConnectorTest extends Specification { noExceptionThrown() nodeFile.exists() nodeFile.file // is it a file? + + cleanup: + connector.shutdown() } def "Initialising a writer with incorrect base directory leads to ConnectorException"() { From 180f3ee3ca989227b5e49d707d2bcb86e9525b56 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 29 Sep 2025 03:46:13 +0200 Subject: [PATCH 02/10] spotless --- .../java/edu/ie3/datamodel/io/connectors/FileConnector.java | 5 ++--- .../io/naming/timeseries/FileLoadProfileMetaInformation.java | 2 +- .../java/edu/ie3/datamodel/io/source/FileDataSource.java | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java index 20d4bc6b8..0ece61aaa 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java @@ -1,5 +1,5 @@ /* - * © 2024. TU Dortmund University, + * © 2025. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ @@ -57,6 +57,5 @@ protected Path resolveFilePath(Path filePath) { protected abstract String getFileEnding(); @Override - public void shutdown() { - } + public void shutdown() {} } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java index 08b293f4a..0b109b6b5 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java @@ -1,5 +1,5 @@ /* - * © 2024. TU Dortmund University, + * © 2025. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ diff --git a/src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java index 1912feec8..abba53e26 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java @@ -1,5 +1,5 @@ /* - * © 2021. TU Dortmund University, + * © 2025. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ From b8988167984532eaf867250a3279c78f555d6f25 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 29 Sep 2025 03:50:57 +0200 Subject: [PATCH 03/10] fix --- .../edu/ie3/datamodel/io/connectors/FileConnector.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java index 0ece61aaa..ddb08d4d9 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java @@ -9,14 +9,13 @@ import java.io.FileNotFoundException; import java.io.InputStream; import java.nio.file.Path; -import java.util.Optional; import java.util.function.Function; /** Base connector for file-based sources and sinks. */ public abstract class FileConnector implements DataConnector { private final Path baseDirectory; - private final Optional> customInputStream; + private final Function customInputStream; protected FileConnector(Path baseDirectory) { this(baseDirectory, null); @@ -24,7 +23,7 @@ protected FileConnector(Path baseDirectory) { protected FileConnector(Path baseDirectory, Function inputStreamSupplier) { this.baseDirectory = baseDirectory; - this.customInputStream = Optional.ofNullable(inputStreamSupplier); + this.customInputStream = inputStreamSupplier; } /** Returns the base directory backing this connector. */ @@ -38,8 +37,8 @@ public Path getBaseDirectory() { */ protected InputStream openInputStream(Path filePath) throws FileNotFoundException { Path fullPath = resolveFilePath(filePath); - if (customInputStream.isPresent()) { - return customInputStream.get().apply(fullPath.toString()); + if (customInputStream != null) { + return customInputStream.apply(fullPath.toString()); } return new FileInputStream(fullPath.toFile()); } From 60252c08f5de4f064c945ac48d4fc4bc4269f574 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 29 Sep 2025 03:54:16 +0200 Subject: [PATCH 04/10] fix --- .../edu/ie3/datamodel/io/connectors/CsvFileConnector.java | 7 +------ .../edu/ie3/datamodel/io/source/csv/CsvDataSource.java | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index ac25d3e53..6b0c7c160 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -13,12 +13,7 @@ import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.value.Value; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.*; diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index e3ecd9a53..50fb9f730 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -25,12 +25,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Path; -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.TreeMap; -import java.util.UUID; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; From 19a91343452130132f40cdb7c6c99f0bc86678dd Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 1 Oct 2025 14:10:19 +0200 Subject: [PATCH 05/10] some more fittings and FileType enum --- .../io/connectors/CsvFileConnector.java | 14 +- .../io/connectors/FileConnector.java | 12 +- .../io/csv/CsvLoadProfileMetaInformation.java | 21 --- .../TimeSeriesMetaInformationFactory.java | 2 +- .../edu/ie3/datamodel/io/file/FileType.java | 36 +++++ .../io/naming/FileNamingStrategy.java | 1 + ...eIndividualTimeSeriesMetaInformation.java} | 31 ++-- .../FileLoadProfileMetaInformation.java | 45 +++--- .../IndividualTimeSeriesMetaInformation.java | 1 - .../LoadProfileMetaInformation.java | 1 - .../TimeSeriesMetaInformation.java | 2 +- .../datamodel/io/source/FileDataSource.java | 95 ------------ .../io/source/LoadProfileSource.java | 4 +- .../io/source/csv/CsvDataSource.java | 83 +++++------ .../io/source/csv/CsvLoadProfileSource.java | 6 +- .../csv/CsvTimeSeriesMappingSource.java | 1 + .../io/source/csv/CsvTimeSeriesSource.java | 6 +- .../io/source/csv/CsvWeatherSource.java | 8 +- .../io/source/file/FileDataSource.java | 141 ++++++++++++++++++ .../FileTimeSeriesMetaInformationSource.java} | 26 ++-- .../io/source/csv/CsvDataSourceTest.groovy | 19 ++- ...svTimeSeriesMetaInformationSourceIT.groovy | 77 ---------- .../source/csv/CsvTimeSeriesSourceTest.groovy | 7 +- ...leTimeSeriesMetaInformationSourceIT.groovy | 78 ++++++++++ 24 files changed, 385 insertions(+), 332 deletions(-) delete mode 100644 src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java create mode 100644 src/main/java/edu/ie3/datamodel/io/file/FileType.java rename src/main/java/edu/ie3/datamodel/io/{csv/CsvIndividualTimeSeriesMetaInformation.java => naming/timeseries/FileIndividualTimeSeriesMetaInformation.java} (56%) rename src/main/java/edu/ie3/datamodel/io/naming/{ => timeseries}/TimeSeriesMetaInformation.java (95%) delete mode 100644 src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java create mode 100644 src/main/java/edu/ie3/datamodel/io/source/file/FileDataSource.java rename src/main/java/edu/ie3/datamodel/io/source/{csv/CsvTimeSeriesMetaInformationSource.java => file/FileTimeSeriesMetaInformationSource.java} (70%) delete mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSourceIT.groovy create mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index 6b0c7c160..b086e3860 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -40,14 +40,8 @@ public CsvFileConnector(Path baseDirectory) { super(baseDirectory); } - public CsvFileConnector(Path baseDirectory, Function inputStreamSupplier) { - super(baseDirectory, inputStreamSupplier); - } - - /** Returns the base directory of this connector. */ - @Override - public Path getBaseDirectory() { - return super.getBaseDirectory(); + public CsvFileConnector(Path baseDirectory, Function inputStreamBuilder) { + super(baseDirectory, inputStreamBuilder); } public synchronized BufferedCsvWriter getOrInitWriter( @@ -58,7 +52,7 @@ public synchronized BufferedCsvWriter getOrInitWriter( /* If it is not available, build and register one */ try { - BufferedCsvWriter newWriter = initWriter(getBaseDirectory(), fileDefinition); + BufferedCsvWriter newWriter = initWriter(baseDirectory, fileDefinition); entityWriters.put(clz, newWriter); return newWriter; @@ -81,7 +75,7 @@ BufferedCsvWriter getOrInitWriter(T timeSeries, CsvFileDefinition fileDefinition /* If it is not available, build and register one */ try { - BufferedCsvWriter newWriter = initWriter(getBaseDirectory(), fileDefinition); + BufferedCsvWriter newWriter = initWriter(baseDirectory, fileDefinition); timeSeriesWriters.put(timeSeries.getUuid(), newWriter); return newWriter; diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java index ddb08d4d9..4fe470a76 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java @@ -14,16 +14,16 @@ /** Base connector for file-based sources and sinks. */ public abstract class FileConnector implements DataConnector { - private final Path baseDirectory; - private final Function customInputStream; + protected final Path baseDirectory; + private final Function inputStreamBuilder; protected FileConnector(Path baseDirectory) { this(baseDirectory, null); } - protected FileConnector(Path baseDirectory, Function inputStreamSupplier) { + protected FileConnector(Path baseDirectory, Function inputStreamBuilder) { this.baseDirectory = baseDirectory; - this.customInputStream = inputStreamSupplier; + this.inputStreamBuilder = inputStreamBuilder; } /** Returns the base directory backing this connector. */ @@ -37,8 +37,8 @@ public Path getBaseDirectory() { */ protected InputStream openInputStream(Path filePath) throws FileNotFoundException { Path fullPath = resolveFilePath(filePath); - if (customInputStream != null) { - return customInputStream.apply(fullPath.toString()); + if (inputStreamBuilder != null) { + return inputStreamBuilder.apply(fullPath.toString()); } return new FileInputStream(fullPath.toFile()); } diff --git a/src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java deleted file mode 100644 index 5d093a848..000000000 --- a/src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * © 2024. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation -*/ -package edu.ie3.datamodel.io.csv; - -import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; -import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; -import java.nio.file.Path; - -public class CsvLoadProfileMetaInformation extends FileLoadProfileMetaInformation { - public CsvLoadProfileMetaInformation(String profile, Path fullFilePath) { - super(profile, fullFilePath); - } - - public CsvLoadProfileMetaInformation( - LoadProfileMetaInformation metaInformation, Path fullFilePath) { - super(metaInformation, fullFilePath); - } -} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java index 71c64cbd8..2c18cdae0 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java @@ -7,10 +7,10 @@ import edu.ie3.datamodel.io.factory.EntityData; import edu.ie3.datamodel.io.factory.EntityFactory; -import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; +import edu.ie3.datamodel.io.naming.timeseries.TimeSeriesMetaInformation; import java.util.Collections; import java.util.List; import java.util.Set; diff --git a/src/main/java/edu/ie3/datamodel/io/file/FileType.java b/src/main/java/edu/ie3/datamodel/io/file/FileType.java new file mode 100644 index 000000000..0c6f995dc --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/file/FileType.java @@ -0,0 +1,36 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.file; + +import edu.ie3.datamodel.exceptions.ParsingException; +import java.util.Arrays; +import java.util.stream.Collectors; + +public enum FileType { + CSV(".csv"); + + public final String fileEnding; + + FileType(String fileEnding) { + this.fileEnding = fileEnding; + } + + public static FileType getFileType(String fileName) throws ParsingException { + FileType[] fileTypes = FileType.values(); + return Arrays.stream(fileTypes) + .filter(f -> f.fileEnding.equals(fileName)) + .findFirst() + .orElseThrow( + () -> + new ParsingException( + "No file ending found for file '" + + fileName + + "'. Only supports file types: " + + Arrays.stream(fileTypes) + .map(t -> t.fileEnding) + .collect(Collectors.joining(", ")))); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/naming/FileNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/FileNamingStrategy.java index fd7084a60..f3de5495e 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/FileNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/FileNamingStrategy.java @@ -8,6 +8,7 @@ import edu.ie3.datamodel.io.IoUtil; import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; +import edu.ie3.datamodel.io.naming.timeseries.TimeSeriesMetaInformation; import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; diff --git a/src/main/java/edu/ie3/datamodel/io/csv/CsvIndividualTimeSeriesMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileIndividualTimeSeriesMetaInformation.java similarity index 56% rename from src/main/java/edu/ie3/datamodel/io/csv/CsvIndividualTimeSeriesMetaInformation.java rename to src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileIndividualTimeSeriesMetaInformation.java index 49a5630d6..05a29d5eb 100644 --- a/src/main/java/edu/ie3/datamodel/io/csv/CsvIndividualTimeSeriesMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileIndividualTimeSeriesMetaInformation.java @@ -3,44 +3,49 @@ * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ -package edu.ie3.datamodel.io.csv; +package edu.ie3.datamodel.io.naming.timeseries; -import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; -import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; +import edu.ie3.datamodel.io.file.FileType; import java.nio.file.Path; import java.util.Objects; import java.util.UUID; /** Enhancing the {@link IndividualTimeSeriesMetaInformation} with the full path to csv file */ -public class CsvIndividualTimeSeriesMetaInformation extends IndividualTimeSeriesMetaInformation { +public class FileIndividualTimeSeriesMetaInformation extends IndividualTimeSeriesMetaInformation { private final Path fullFilePath; + private final FileType fileType; - public CsvIndividualTimeSeriesMetaInformation( - UUID uuid, ColumnScheme columnScheme, Path fullFilePath) { + public FileIndividualTimeSeriesMetaInformation( + UUID uuid, ColumnScheme columnScheme, Path fullFilePath, FileType fileType) { super(uuid, columnScheme); this.fullFilePath = fullFilePath; + this.fileType = fileType; } - public CsvIndividualTimeSeriesMetaInformation( - IndividualTimeSeriesMetaInformation metaInformation, Path fullFilePath) { - this(metaInformation.getUuid(), metaInformation.getColumnScheme(), fullFilePath); + public FileIndividualTimeSeriesMetaInformation( + IndividualTimeSeriesMetaInformation metaInformation, Path fullFilePath, FileType fileType) { + this(metaInformation.getUuid(), metaInformation.getColumnScheme(), fullFilePath, fileType); } public Path getFullFilePath() { return fullFilePath; } + public FileType getFileType() { + return fileType; + } + @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof CsvIndividualTimeSeriesMetaInformation that)) return false; + if (!(o instanceof FileIndividualTimeSeriesMetaInformation that)) return false; if (!super.equals(o)) return false; - return fullFilePath.equals(that.fullFilePath); + return fullFilePath.equals(that.fullFilePath) && fileType.equals(that.fileType); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), fullFilePath); + return Objects.hash(super.hashCode(), fullFilePath, fileType); } @Override @@ -53,6 +58,8 @@ public String toString() { + ", fullFilePath='" + fullFilePath + '\'' + + ", fileType=" + + fileType + '}'; } } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java index 0b109b6b5..7209b818b 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java @@ -1,60 +1,53 @@ /* - * © 2025. TU Dortmund University, + * © 2024. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ package edu.ie3.datamodel.io.naming.timeseries; +import edu.ie3.datamodel.io.file.FileType; import java.nio.file.Path; import java.util.Objects; -import java.util.UUID; -public abstract class FileLoadProfileMetaInformation extends LoadProfileMetaInformation { +public class FileLoadProfileMetaInformation extends LoadProfileMetaInformation { private final Path fullFilePath; + private final FileType fileType; - protected FileLoadProfileMetaInformation(String profile, Path fullFilePath) { + public FileLoadProfileMetaInformation(String profile, Path fullFilePath, FileType fileType) { super(profile); - this.fullFilePath = Objects.requireNonNull(fullFilePath, "fullFilePath"); - } - - protected FileLoadProfileMetaInformation(UUID uuid, String profile, Path fullFilePath) { - super(uuid, profile); - this.fullFilePath = Objects.requireNonNull(fullFilePath, "fullFilePath"); - } - - protected FileLoadProfileMetaInformation( - LoadProfileMetaInformation metaInformation, Path fullFilePath) { - this(metaInformation.getUuid(), metaInformation.getProfile(), fullFilePath); + this.fullFilePath = fullFilePath; + this.fileType = fileType; } public Path getFullFilePath() { return fullFilePath; } + public FileType getFileType() { + return fileType; + } + @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof FileLoadProfileMetaInformation that)) return false; + if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; - return fullFilePath.equals(that.fullFilePath); + FileLoadProfileMetaInformation that = (FileLoadProfileMetaInformation) o; + return Objects.equals(fullFilePath, that.fullFilePath) && fileType == that.fileType; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), fullFilePath); + return Objects.hash(super.hashCode(), fullFilePath, fileType); } @Override public String toString() { return "FileLoadProfileMetaInformation{" - + "uuid='" - + getUuid() - + '\'' - + ", profile='" - + getProfile() - + '\'' - + "fullFilePath=" + + ", fullFilePath='" + fullFilePath + + '\'' + + ", fileType=" + + fileType + '}'; } } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/IndividualTimeSeriesMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/IndividualTimeSeriesMetaInformation.java index dfd42685b..4414ece9b 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/IndividualTimeSeriesMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/IndividualTimeSeriesMetaInformation.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.naming.timeseries; -import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; import java.util.Objects; import java.util.UUID; diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java index 875c881e6..0de878118 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.naming.timeseries; -import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; import java.util.Objects; import java.util.UUID; diff --git a/src/main/java/edu/ie3/datamodel/io/naming/TimeSeriesMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/TimeSeriesMetaInformation.java similarity index 95% rename from src/main/java/edu/ie3/datamodel/io/naming/TimeSeriesMetaInformation.java rename to src/main/java/edu/ie3/datamodel/io/naming/timeseries/TimeSeriesMetaInformation.java index aa11f1ffd..7233e2718 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/TimeSeriesMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/TimeSeriesMetaInformation.java @@ -3,7 +3,7 @@ * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ -package edu.ie3.datamodel.io.naming; +package edu.ie3.datamodel.io.naming.timeseries; import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.Uniqueness; diff --git a/src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java deleted file mode 100644 index abba53e26..000000000 --- a/src/main/java/edu/ie3/datamodel/io/source/FileDataSource.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * © 2025. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation -*/ -package edu.ie3.datamodel.io.source; - -import edu.ie3.datamodel.exceptions.SourceException; -import edu.ie3.datamodel.io.naming.FileNamingStrategy; -import edu.ie3.datamodel.models.Entity; -import edu.ie3.datamodel.utils.Try; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class FileDataSource implements DataSource { - - protected final Logger log = LoggerFactory.getLogger(getClass()); - private final Path baseDirectory; - private final FileNamingStrategy fileNamingStrategy; - - protected FileDataSource(Path directoryPath, FileNamingStrategy fileNamingStrategy) { - this.baseDirectory = Objects.requireNonNull(directoryPath, "directoryPath"); - this.fileNamingStrategy = Objects.requireNonNull(fileNamingStrategy, "fileNamingStrategy"); - } - - @Override - public Optional> getSourceFields(Class entityClass) - throws SourceException { - return getSourceFields(getFilePath(entityClass).getOrThrow()); - } - - public abstract Optional> getSourceFields(Path filePath) throws SourceException; - - @Override - public Stream> getSourceData(Class entityClass) - throws SourceException { - return buildStreamWithFieldsToAttributesMap(entityClass, true).getOrThrow(); - } - - public abstract Stream> getSourceData(Path filePath) throws SourceException; - - public FileNamingStrategy getNamingStrategy() { - return fileNamingStrategy; - } - - protected Path getBaseDirectory() { - return baseDirectory; - } - - protected Try>, SourceException> buildStreamWithFieldsToAttributesMap( - Class entityClass, boolean allowFileNotExisting) { - return getFilePath(entityClass) - .flatMap(path -> buildStreamWithFieldsToAttributesMap(path, allowFileNotExisting)); - } - - protected abstract Try>, SourceException> - buildStreamWithFieldsToAttributesMap(Path filePath, boolean allowFileNotExisting); - - protected Try getFilePath(Class entityClass) { - return Try.from( - fileNamingStrategy.getFilePath(entityClass), - () -> - new SourceException( - "Cannot find a naming strategy for class '" + entityClass.getSimpleName() + "'.")); - } - - protected Set getTimeSeriesFilePaths(Pattern pattern) { - Path baseDirectory = getBaseDirectory(); - try (Stream pathStream = Files.walk(baseDirectory)) { - return pathStream - .map(baseDirectory::relativize) - .filter( - path -> { - Path withoutEnding = - Path.of(FileNamingStrategy.removeFileNameEnding(path.toString())); - return pattern.matcher(withoutEnding.toString()).matches(); - }) - .collect(Collectors.toSet()); - } catch (IOException e) { - log.error("Unable to determine time series files readers for time series.", e); - return Collections.emptySet(); - } - } -} diff --git a/src/main/java/edu/ie3/datamodel/io/source/LoadProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/LoadProfileSource.java index 238300c70..52ebc4ede 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/LoadProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/LoadProfileSource.java @@ -9,11 +9,11 @@ import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.SourceException; -import edu.ie3.datamodel.io.csv.CsvLoadProfileMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.BdewLoadProfileFactory; import edu.ie3.datamodel.io.factory.timeseries.LoadProfileData; import edu.ie3.datamodel.io.factory.timeseries.LoadProfileFactory; import edu.ie3.datamodel.io.factory.timeseries.RandomLoadProfileFactory; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; import edu.ie3.datamodel.io.source.csv.CsvDataSource; import edu.ie3.datamodel.io.source.csv.CsvLoadProfileSource; import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile; @@ -137,7 +137,7 @@ public static long getResolution(LoadProfile loadProfile) { getRandomLoadProfile() throws SourceException { CsvDataSource buildInSource = getBuildInSource(LoadProfileSource.class, "/load"); - CsvLoadProfileMetaInformation metaInformation = + FileLoadProfileMetaInformation metaInformation = buildInSource.getCsvLoadProfileMetaInformation(RANDOM_LOAD_PROFILE).values().stream() .findAny() .orElseThrow(); diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index 50fb9f730..030194c08 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -5,16 +5,17 @@ */ package edu.ie3.datamodel.io.source.csv; +import static edu.ie3.datamodel.io.file.FileType.CSV; + import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; -import edu.ie3.datamodel.io.csv.CsvLoadProfileMetaInformation; import edu.ie3.datamodel.io.naming.FileNamingStrategy; -import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; -import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; -import edu.ie3.datamodel.io.source.FileDataSource; +import edu.ie3.datamodel.io.naming.timeseries.TimeSeriesMetaInformation; +import edu.ie3.datamodel.io.source.file.FileDataSource; import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.profile.LoadProfile; import edu.ie3.datamodel.utils.Try; @@ -56,6 +57,18 @@ public CsvDataSource( this.connector = connector; } + @Override + public Optional> getSourceFields(Class entityClass) + throws SourceException { + return getSourceFields(getFilePath(entityClass).getOrThrow()); + } + + /** + * @param filePath path of file starting from base folder, including file name but not file + * extension + * @return The source field names as a set, if file exists + * @throws SourceException on error while reading the source file + */ @Override public Optional> getSourceFields(Path filePath) throws SourceException { try (BufferedReader reader = connector.initReader(filePath)) { @@ -81,6 +94,12 @@ public Optional> getSourceFields(Path filePath) throws SourceExcepti } } + @Override + public Stream> getSourceData(Class entityClass) + throws SourceException { + return buildStreamWithFieldsToAttributesMap(entityClass, true).getOrThrow(); + } + @Override public Stream> getSourceData(Path filePath) throws SourceException { return buildStreamWithFieldsToAttributesMap(filePath, true).getOrThrow(); @@ -94,24 +113,10 @@ public Stream> getSourceData(Path filePath) throws SourceExc * possible readers will be initialized. * @return A mapping from column scheme to the individual time series meta information */ - public Map + public Map getCsvIndividualTimeSeriesMetaInformation(final ColumnScheme... columnSchemes) { - FileNamingStrategy namingStrategy = getNamingStrategy(); - return getTimeSeriesFilePaths(namingStrategy.getIndividualTimeSeriesPattern()).parallelStream() - .map( - filePath -> { - /* Extract meta information from file path and enhance it with the file path itself */ - IndividualTimeSeriesMetaInformation metaInformation = - namingStrategy.individualTimeSeriesMetaInformation(filePath.toString()); - return new CsvIndividualTimeSeriesMetaInformation( - metaInformation, FileNamingStrategy.removeFileNameEnding(filePath.getFileName())); - }) - .filter( - metaInformation -> - columnSchemes == null - || columnSchemes.length == 0 - || Stream.of(columnSchemes) - .anyMatch(scheme -> scheme.equals(metaInformation.getColumnScheme()))) + return getIndividualTimeSeriesMetaInformation(columnSchemes) + .filter(metaInformation -> metaInformation.getFileType() == CSV) .collect(Collectors.toMap(TimeSeriesMetaInformation::getUuid, Function.identity())); } @@ -121,24 +126,10 @@ public Stream> getSourceData(Path filePath) throws SourceExc * * @return A mapping from profile to the load profile time series meta information */ - public Map getCsvLoadProfileMetaInformation( + public Map getCsvLoadProfileMetaInformation( LoadProfile... profiles) { - FileNamingStrategy namingStrategy = getNamingStrategy(); - return getTimeSeriesFilePaths(namingStrategy.getLoadProfileTimeSeriesPattern()).parallelStream() - .map( - filePath -> { - /* Extract meta information from file path and enhance it with the file path itself */ - LoadProfileMetaInformation metaInformation = - namingStrategy.loadProfileTimeSeriesMetaInformation(filePath.toString()); - return new CsvLoadProfileMetaInformation( - metaInformation, FileNamingStrategy.removeFileNameEnding(filePath.getFileName())); - }) - .filter( - metaInformation -> - profiles == null - || profiles.length == 0 - || Stream.of(profiles) - .anyMatch(profile -> profile.getKey().equals(metaInformation.getProfile()))) + return getLoadProfileMetaInformation(profiles) + .filter(metaInformation -> metaInformation.getFileType() == CSV) .collect(Collectors.toMap(LoadProfileMetaInformation::getProfile, Function.identity())); } @@ -216,10 +207,19 @@ protected String[] parseCsvRow(String csvRow, String csvSep) { .toArray(String[]::new); } - @Override + /** + * Tries to open a file reader based on the provided entity class and hands it over for further + * processing. + * + * @param entityClass the entity class that should be build and that is used to get the + * corresponding reader + * @return a parallel stream of maps, where each map represents one row of the csv file with the + * mapping (fieldName to fieldValue) + */ protected Try>, SourceException> buildStreamWithFieldsToAttributesMap( Class entityClass, boolean allowFileNotExisting) { - return super.buildStreamWithFieldsToAttributesMap(entityClass, allowFileNotExisting); + return getFilePath(entityClass) + .flatMap(path -> buildStreamWithFieldsToAttributesMap(path, allowFileNotExisting)); } /** @@ -231,7 +231,6 @@ protected Try>, SourceException> buildStreamWithField * @return a try containing either a parallel stream of maps, where each map represents one row of * the csv file with the mapping (fieldName to fieldValue) or an exception */ - @Override protected Try>, SourceException> buildStreamWithFieldsToAttributesMap( Path filePath, boolean allowFileNotExisting) { try (BufferedReader reader = connector.initReader(filePath)) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java index c785599b6..d9f85383b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java @@ -8,8 +8,8 @@ import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; -import edu.ie3.datamodel.io.csv.CsvLoadProfileMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.LoadProfileFactory; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; import edu.ie3.datamodel.io.source.LoadProfileSource; import edu.ie3.datamodel.models.profile.LoadProfile; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileEntry; @@ -37,7 +37,7 @@ public class CsvLoadProfileSource

public CsvLoadProfileSource( CsvDataSource source, - CsvLoadProfileMetaInformation metaInformation, + FileLoadProfileMetaInformation metaInformation, Class entryClass, LoadProfileFactory entryFactory) { super(entryClass, entryFactory); @@ -106,7 +106,7 @@ public Optional> getLoadProfileEnergyScaling() { * @return an individual time series */ protected LoadProfileTimeSeries buildLoadProfileTimeSeries( - CsvLoadProfileMetaInformation metaInformation, + FileLoadProfileMetaInformation metaInformation, Function, Try, FactoryException>> fieldToValueFunction) throws SourceException { diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java index c6de18c1e..62ccc0686 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java @@ -8,6 +8,7 @@ import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; + import java.nio.file.Path; import java.util.Map; import java.util.Optional; diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java index 05c9185c3..543f204bb 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java @@ -8,9 +8,9 @@ import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.*; import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.source.TimeSeriesSource; import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; @@ -44,7 +44,7 @@ public static CsvTimeSeriesSource getSource( String csvSep, Path folderPath, FileNamingStrategy fileNamingStrategy, - CsvIndividualTimeSeriesMetaInformation metaInformation) + FileIndividualTimeSeriesMetaInformation metaInformation) throws SourceException { if (!TimeSeriesUtils.isSchemeAccepted(metaInformation.getColumnScheme())) throw new SourceException( @@ -59,7 +59,7 @@ private static CsvTimeSeriesSource create( String csvSep, Path folderPath, FileNamingStrategy fileNamingStrategy, - CsvIndividualTimeSeriesMetaInformation metaInformation, + FileIndividualTimeSeriesMetaInformation metaInformation, Class valClass) { TimeBasedSimpleValueFactory valueFactory = new TimeBasedSimpleValueFactory<>(valClass); return new CsvTimeSeriesSource<>( diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 241442725..e3b2464d8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -11,11 +11,11 @@ import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.source.IdCoordinateSource; import edu.ie3.datamodel.io.source.WeatherSource; import edu.ie3.datamodel.models.Entity; @@ -157,7 +157,7 @@ protected IndividualTimeSeries mergeTimeSeries( private Map> getWeatherTimeSeries() throws SourceException { /* Get only weather time series meta information */ - Collection weatherCsvMetaInformation = + Collection weatherCsvMetaInformation = dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); return readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); } @@ -169,14 +169,14 @@ private Map> getWeatherTimeSeries() * @return time series mapped to the represented coordinate */ private Map> readWeatherTimeSeries( - Set weatherMetaInformation, + Set weatherMetaInformation, CsvFileConnector connector) throws SourceException { final Map> weatherTimeSeries = new HashMap<>(); Function, Optional>> fieldToValueFunction = this::buildWeatherValue; /* Reading in weather time series */ - for (CsvIndividualTimeSeriesMetaInformation data : weatherMetaInformation) { + for (FileIndividualTimeSeriesMetaInformation data : weatherMetaInformation) { Path path = data.getFullFilePath(); // we need a reader for each file diff --git a/src/main/java/edu/ie3/datamodel/io/source/file/FileDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/file/FileDataSource.java new file mode 100644 index 000000000..ac5456777 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/file/FileDataSource.java @@ -0,0 +1,141 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.file; + +import edu.ie3.datamodel.exceptions.ParsingException; +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.io.file.FileType; +import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.io.naming.timeseries.*; +import edu.ie3.datamodel.io.source.DataSource; +import edu.ie3.datamodel.models.Entity; +import edu.ie3.datamodel.models.profile.LoadProfile; +import edu.ie3.datamodel.utils.Try; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class FileDataSource implements DataSource { + + protected final Logger log = LoggerFactory.getLogger(getClass()); + protected final Path baseDirectory; + protected final FileNamingStrategy fileNamingStrategy; + + protected FileDataSource(Path directoryPath, FileNamingStrategy fileNamingStrategy) { + this.baseDirectory = Objects.requireNonNull(directoryPath, "directoryPath"); + this.fileNamingStrategy = Objects.requireNonNull(fileNamingStrategy, "fileNamingStrategy"); + } + + public abstract Optional> getSourceFields(Path filePath) throws SourceException; + + public abstract Stream> getSourceData(Path filePath) throws SourceException; + + public FileNamingStrategy getNamingStrategy() { + return fileNamingStrategy; + } + + protected Try getFilePath(Class entityClass) { + return Try.from( + fileNamingStrategy.getFilePath(entityClass), + () -> + new SourceException( + "Cannot find a naming strategy for class '" + entityClass.getSimpleName() + "'.")); + } + + protected Set getTimeSeriesFilePaths(Pattern pattern) { + try (Stream pathStream = Files.walk(baseDirectory)) { + return pathStream + .map(baseDirectory::relativize) + .filter( + path -> { + Path withoutEnding = + Path.of(FileNamingStrategy.removeFileNameEnding(path.toString())); + return pattern.matcher(withoutEnding.toString()).matches(); + }) + .collect(Collectors.toSet()); + } catch (IOException e) { + log.error("Unable to determine time series files readers for time series.", e); + return Collections.emptySet(); + } + } + + public Stream getIndividualTimeSeriesMetaInformation( + final ColumnScheme... columnSchemes) { + FileNamingStrategy namingStrategy = getNamingStrategy(); + return getTimeSeriesFilePaths(namingStrategy.getIndividualTimeSeriesPattern()).parallelStream() + .map( + filePath -> { + String fileName = filePath.getFileName().toString(); + FileType fileType; + try { + fileType = FileType.getFileType(fileName); + } catch (ParsingException e) { + log.warn("Unable to load load profile meta data for {}", fileName, e); + fileType = null; // will be filtered out + } + /* Extract meta information from file path and enhance it with the file path itself */ + IndividualTimeSeriesMetaInformation metaInformation = + namingStrategy.individualTimeSeriesMetaInformation(filePath.toString()); + return new FileIndividualTimeSeriesMetaInformation( + metaInformation, + Path.of(FileNamingStrategy.removeFileNameEnding(fileName)), + fileType); + }) + .filter(meta -> meta.getFileType() != null) + .filter( + metaInformation -> + columnSchemes == null + || columnSchemes.length == 0 + || Stream.of(columnSchemes) + .anyMatch(scheme -> scheme.equals(metaInformation.getColumnScheme()))); + } + + /** + * Receive the information for specific load profile time series. They are given back mapped to + * their uuid. + * + * @return A mapping from profile to the load profile time series meta information + */ + public Stream getLoadProfileMetaInformation( + LoadProfile... profiles) { + FileNamingStrategy namingStrategy = getNamingStrategy(); + + return getTimeSeriesFilePaths(namingStrategy.getLoadProfileTimeSeriesPattern()).parallelStream() + .map( + filePath -> { + String fileName = filePath.getFileName().toString(); + FileType fileType; + try { + fileType = FileType.getFileType(fileName); + } catch (ParsingException e) { + log.warn("Unable to load load profile meta data for {}", fileName, e); + fileType = null; // will be filtered out + } + + /* Extract meta information from file path and enhance it with the file path itself */ + LoadProfileMetaInformation metaInformation = + namingStrategy.loadProfileTimeSeriesMetaInformation(filePath.toString()); + return new FileLoadProfileMetaInformation( + metaInformation.getProfile(), + Path.of(FileNamingStrategy.removeFileNameEnding(fileName)), + fileType); + }) + .filter(meta -> meta.getFileType() != null) + .filter( + metaInformation -> + profiles == null + || profiles.length == 0 + || Stream.of(profiles) + .anyMatch( + profile -> profile.getKey().equals(metaInformation.getProfile()))); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSource.java b/src/main/java/edu/ie3/datamodel/io/source/file/FileTimeSeriesMetaInformationSource.java similarity index 70% rename from src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSource.java rename to src/main/java/edu/ie3/datamodel/io/source/file/FileTimeSeriesMetaInformationSource.java index 786474e87..abaa49432 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/file/FileTimeSeriesMetaInformationSource.java @@ -3,14 +3,15 @@ * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ -package edu.ie3.datamodel.io.source.csv; +package edu.ie3.datamodel.io.source.file; -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; import edu.ie3.datamodel.io.source.TimeSeriesMetaInformationSource; +import edu.ie3.datamodel.io.source.csv.CsvDataSource; import edu.ie3.datamodel.utils.TimeSeriesUtils; import java.nio.file.Path; import java.util.Collections; @@ -24,20 +25,19 @@ * CSV implementation for retrieving {@link TimeSeriesMetaInformationSource} from input directory * structures */ -public class CsvTimeSeriesMetaInformationSource extends TimeSeriesMetaInformationSource { +public class FileTimeSeriesMetaInformationSource extends TimeSeriesMetaInformationSource { - protected final CsvDataSource dataSource; - private final Map timeSeriesMetaInformation; + private final Map timeSeriesMetaInformation; /** - * Creates a time series type source + * Creates a time series type source from CSV * * @param csvSep the CSV separator * @param folderPath path that time series reside in * @param fileNamingStrategy the file naming strategy */ - public CsvTimeSeriesMetaInformationSource( + public FileTimeSeriesMetaInformationSource( String csvSep, Path folderPath, FileNamingStrategy fileNamingStrategy) { this(new CsvDataSource(csvSep, folderPath, fileNamingStrategy)); } @@ -45,17 +45,17 @@ public CsvTimeSeriesMetaInformationSource( /** * Creates a time series type source * - * @param dataSource a csv data source + * @param dataSource a file data source */ - public CsvTimeSeriesMetaInformationSource(CsvDataSource dataSource) { - this.dataSource = dataSource; + public FileTimeSeriesMetaInformationSource(FileDataSource dataSource) { // retrieve only the desired time series this.timeSeriesMetaInformation = - dataSource.getCsvIndividualTimeSeriesMetaInformation( - TimeSeriesUtils.getAcceptedColumnSchemes().toArray(new ColumnScheme[0])); + dataSource.getIndividualTimeSeriesMetaInformation( + TimeSeriesUtils.getAcceptedColumnSchemes().toArray(new ColumnScheme[0])) + .collect(Collectors.toMap(IndividualTimeSeriesMetaInformation::getUuid, Function.identity())); this.loadProfileMetaInformation = - dataSource.getCsvLoadProfileMetaInformation().values().stream() + dataSource.getLoadProfileMetaInformation() .collect(Collectors.toMap(LoadProfileMetaInformation::getProfile, Function.identity())); } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy index 754c62fa8..7287bdc1a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy @@ -7,10 +7,9 @@ package edu.ie3.datamodel.io.source.csv import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.connectors.CsvFileConnector -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation -import edu.ie3.datamodel.io.csv.CsvLoadProfileMetaInformation import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.models.input.system.LoadInput import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile import spock.lang.Shared @@ -384,11 +383,11 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta { def "The CsvDataSource is able to build correct uuid to meta information mapping"() { given: def expected = [ - (UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf"), ColumnScheme.APPARENT_POWER, Path.of("its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf")), - (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")), - (UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of("its_pqh_5022a70e-a58f-4bac-b8ec-1c62376c216b")), - (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), - (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")) + (UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf"), ColumnScheme.APPARENT_POWER, Path.of("its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf")), + (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")), + (UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of("its_pqh_5022a70e-a58f-4bac-b8ec-1c62376c216b")), + (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), + (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")) ] when: @@ -401,9 +400,9 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta { def "The CsvDataSource is able to build correct uuid to meta information mapping when restricting column schemes"() { given: def expected = [ - (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), - (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")), - (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")) + (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), + (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")), + (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")) ] when: diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSourceIT.groovy deleted file mode 100644 index 6623e76ca..000000000 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSourceIT.groovy +++ /dev/null @@ -1,77 +0,0 @@ -/* - * © 2021. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation - */ -package edu.ie3.datamodel.io.source.csv - -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation -import edu.ie3.datamodel.io.naming.FileNamingStrategy -import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme -import spock.lang.Shared -import spock.lang.Specification - -import java.nio.file.Path - -class CsvTimeSeriesMetaInformationSourceIT extends Specification implements CsvTestDataMeta { - @Shared - CsvTimeSeriesMetaInformationSource source - - def setupSpec() { - source = new CsvTimeSeriesMetaInformationSource(";", timeSeriesFolderPath, new FileNamingStrategy()) - } - - def "A CSV time series meta information source returns correct mapping of time series"() { - given: - def expectedTimeSeries = Set.of( - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("2fcb3e53-b94a-4b96-bea4-c469e499f1a1"), ColumnScheme.ENERGY_PRICE, Path.of('its_c_2fcb3e53-b94a-4b96-bea4-c469e499f1a1')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("76c9d846-797c-4f07-b7ec-2245f679f5c7"), ColumnScheme.ACTIVE_POWER_AND_HEAT_DEMAND, Path.of('its_ph_76c9d846-797c-4f07-b7ec-2245f679f5c7')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0"), ColumnScheme.HEAT_DEMAND, Path.of('its_h_c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("9185b8c1-86ba-4a16-8dea-5ac898e8caa5"), ColumnScheme.ACTIVE_POWER, Path.of('its_p_9185b8c1-86ba-4a16-8dea-5ac898e8caa5')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("3fbfaa97-cff4-46d4-95ba-a95665e87c26"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_3fbfaa97-cff4-46d4-95ba-a95665e87c26')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("46be1e57-e4ed-4ef7-95f1-b2b321cb2047"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of('its_pqh_46be1e57-e4ed-4ef7-95f1-b2b321cb2047')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("1061af70-1c03-46e1-b960-940b956c429f"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_1061af70-1c03-46e1-b960-940b956c429f')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("eeccbe3c-a47e-448e-8eca-1f369d3c24e6"), ColumnScheme.VOLTAGE, Path.of("its_v_eeccbe3c-a47e-448e-8eca-1f369d3c24e6")) - ) - - when: - def actual = source.timeSeriesMetaInformation - - then: - actual.size() == 8 - actual.every { - it.key == it.value.uuid && - expectedTimeSeries.contains(it.value) - } - } - - def "The CSV time series meta information source returns correct meta information for a given time series UUID"() { - when: - def timeSeriesUuid = UUID.fromString(uuid) - def result = source.getTimeSeriesMetaInformation(timeSeriesUuid) - - then: - result.present - result.get().columnScheme.scheme == columnScheme - - where: - uuid || columnScheme - "2fcb3e53-b94a-4b96-bea4-c469e499f1a1" || "c" - "76c9d846-797c-4f07-b7ec-2245f679f5c7" || "ph" - "c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0" || "h" - "9185b8c1-86ba-4a16-8dea-5ac898e8caa5" || "p" - "3fbfaa97-cff4-46d4-95ba-a95665e87c26" || "pq" - "46be1e57-e4ed-4ef7-95f1-b2b321cb2047" || "pqh" - "1061af70-1c03-46e1-b960-940b956c429f" || "pq" - "eeccbe3c-a47e-448e-8eca-1f369d3c24e6" || "v" - } - - def "The CSV time series meta information source returns an empty optional for an unknown time series UUID"() { - when: - def timeSeriesUuid = UUID.fromString("e9c13f5f-31da-44ea-abb7-59f616c3da16") - def result = source.getTimeSeriesMetaInformation(timeSeriesUuid) - - then: - result.empty - } -} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy index d6b6e48fb..033ffaa87 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy @@ -8,10 +8,10 @@ package edu.ie3.datamodel.io.source.csv import static edu.ie3.datamodel.models.StandardUnits.ENERGY_PRICE import edu.ie3.datamodel.exceptions.SourceException -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.io.factory.timeseries.TimeBasedSimpleValueFactory import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.* import edu.ie3.util.TimeUtil @@ -19,7 +19,6 @@ import spock.lang.Specification import tech.units.indriya.quantity.Quantities import java.nio.file.Path -import java.time.ZoneId class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { @@ -99,7 +98,7 @@ class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { def "The factory method in csv time series source refuses to build time series with unsupported column type"() { given: - def metaInformation = new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("8bc9120d-fb9b-4484-b4e3-0cdadf0feea9"), ColumnScheme.WEATHER, Path.of("its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9")) + def metaInformation = new FileIndividualTimeSeriesMetaInformation(UUID.fromString("8bc9120d-fb9b-4484-b4e3-0cdadf0feea9"), ColumnScheme.WEATHER, Path.of("its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9")) when: CsvTimeSeriesSource.getSource(";", timeSeriesFolderPath, fileNamingStrategy, metaInformation) @@ -111,7 +110,7 @@ class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { def "The factory method in csv time series source builds a time series source for all supported column types"() { given: - def metaInformation = new CsvIndividualTimeSeriesMetaInformation(uuid, columnScheme, path) + def metaInformation = new FileIndividualTimeSeriesMetaInformation(uuid, columnScheme, path) when: def actual = CsvTimeSeriesSource.getSource(";", timeSeriesFolderPath, fileNamingStrategy, metaInformation) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy new file mode 100644 index 000000000..8904cc4d7 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy @@ -0,0 +1,78 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.csv + +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation +import edu.ie3.datamodel.io.source.file.FileTimeSeriesMetaInformationSource +import spock.lang.Shared +import spock.lang.Specification + +import java.nio.file.Path + +class FileTimeSeriesMetaInformationSourceIT extends Specification implements CsvTestDataMeta { + @Shared + FileTimeSeriesMetaInformationSource source + + def setupSpec() { + source = new FileTimeSeriesMetaInformationSource(";", timeSeriesFolderPath, new FileNamingStrategy()) + } + + def "A CSV time series meta information source returns correct mapping of time series"() { + given: + def expectedTimeSeries = Set.of( + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("2fcb3e53-b94a-4b96-bea4-c469e499f1a1"), ColumnScheme.ENERGY_PRICE, Path.of('its_c_2fcb3e53-b94a-4b96-bea4-c469e499f1a1')), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("76c9d846-797c-4f07-b7ec-2245f679f5c7"), ColumnScheme.ACTIVE_POWER_AND_HEAT_DEMAND, Path.of('its_ph_76c9d846-797c-4f07-b7ec-2245f679f5c7')), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0"), ColumnScheme.HEAT_DEMAND, Path.of('its_h_c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0')), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("9185b8c1-86ba-4a16-8dea-5ac898e8caa5"), ColumnScheme.ACTIVE_POWER, Path.of('its_p_9185b8c1-86ba-4a16-8dea-5ac898e8caa5')), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("3fbfaa97-cff4-46d4-95ba-a95665e87c26"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_3fbfaa97-cff4-46d4-95ba-a95665e87c26')), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("46be1e57-e4ed-4ef7-95f1-b2b321cb2047"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of('its_pqh_46be1e57-e4ed-4ef7-95f1-b2b321cb2047')), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("1061af70-1c03-46e1-b960-940b956c429f"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_1061af70-1c03-46e1-b960-940b956c429f')), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("eeccbe3c-a47e-448e-8eca-1f369d3c24e6"), ColumnScheme.VOLTAGE, Path.of("its_v_eeccbe3c-a47e-448e-8eca-1f369d3c24e6")) + ) + + when: + def actual = source.timeSeriesMetaInformation + + then: + actual.size() == 8 + actual.every { + it.key == it.value.uuid && + expectedTimeSeries.contains(it.value) + } + } + + def "The CSV time series meta information source returns correct meta information for a given time series UUID"() { + when: + def timeSeriesUuid = UUID.fromString(uuid) + def result = source.getTimeSeriesMetaInformation(timeSeriesUuid) + + then: + result.present + result.get().columnScheme.scheme == columnScheme + + where: + uuid || columnScheme + "2fcb3e53-b94a-4b96-bea4-c469e499f1a1" || "c" + "76c9d846-797c-4f07-b7ec-2245f679f5c7" || "ph" + "c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0" || "h" + "9185b8c1-86ba-4a16-8dea-5ac898e8caa5" || "p" + "3fbfaa97-cff4-46d4-95ba-a95665e87c26" || "pq" + "46be1e57-e4ed-4ef7-95f1-b2b321cb2047" || "pqh" + "1061af70-1c03-46e1-b960-940b956c429f" || "pq" + "eeccbe3c-a47e-448e-8eca-1f369d3c24e6" || "v" + } + + def "The CSV time series meta information source returns an empty optional for an unknown time series UUID"() { + when: + def timeSeriesUuid = UUID.fromString("e9c13f5f-31da-44ea-abb7-59f616c3da16") + def result = source.getTimeSeriesMetaInformation(timeSeriesUuid) + + then: + result.empty + } +} From 1dfab2ace43997119f47c8a1a38ed8792ccbed15 Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 1 Oct 2025 15:05:50 +0200 Subject: [PATCH 06/10] spotless --- .../io/source/csv/CsvTimeSeriesMappingSource.java | 1 - .../file/FileTimeSeriesMetaInformationSource.java | 13 ++++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java index 62ccc0686..c6de18c1e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java @@ -8,7 +8,6 @@ import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.source.TimeSeriesMappingSource; - import java.nio.file.Path; import java.util.Map; import java.util.Optional; diff --git a/src/main/java/edu/ie3/datamodel/io/source/file/FileTimeSeriesMetaInformationSource.java b/src/main/java/edu/ie3/datamodel/io/source/file/FileTimeSeriesMetaInformationSource.java index abaa49432..34db88e58 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/file/FileTimeSeriesMetaInformationSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/file/FileTimeSeriesMetaInformationSource.java @@ -27,7 +27,6 @@ */ public class FileTimeSeriesMetaInformationSource extends TimeSeriesMetaInformationSource { - private final Map timeSeriesMetaInformation; /** @@ -50,12 +49,16 @@ public FileTimeSeriesMetaInformationSource( public FileTimeSeriesMetaInformationSource(FileDataSource dataSource) { // retrieve only the desired time series this.timeSeriesMetaInformation = - dataSource.getIndividualTimeSeriesMetaInformation( - TimeSeriesUtils.getAcceptedColumnSchemes().toArray(new ColumnScheme[0])) - .collect(Collectors.toMap(IndividualTimeSeriesMetaInformation::getUuid, Function.identity())); + dataSource + .getIndividualTimeSeriesMetaInformation( + TimeSeriesUtils.getAcceptedColumnSchemes().toArray(new ColumnScheme[0])) + .collect( + Collectors.toMap( + IndividualTimeSeriesMetaInformation::getUuid, Function.identity())); this.loadProfileMetaInformation = - dataSource.getLoadProfileMetaInformation() + dataSource + .getLoadProfileMetaInformation() .collect(Collectors.toMap(LoadProfileMetaInformation::getProfile, Function.identity())); } From a3860e97cb065c4753789621bd35391293477101 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 6 Oct 2025 16:39:56 +0200 Subject: [PATCH 07/10] tests --- .../io/source/csv/CsvDataSourceTest.groovy | 18 ++++++++++-------- .../source/csv/CsvTimeSeriesSourceTest.groovy | 8 +++++--- ...ileTimeSeriesMetaInformationSourceIT.groovy | 18 ++++++++++-------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy index 7287bdc1a..725408305 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy @@ -8,6 +8,7 @@ package edu.ie3.datamodel.io.source.csv import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.connectors.CsvFileConnector import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.file.FileType import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.models.input.system.LoadInput @@ -383,11 +384,11 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta { def "The CsvDataSource is able to build correct uuid to meta information mapping"() { given: def expected = [ - (UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf"), ColumnScheme.APPARENT_POWER, Path.of("its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf")), - (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")), - (UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of("its_pqh_5022a70e-a58f-4bac-b8ec-1c62376c216b")), - (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), - (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")) + (UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf"), ColumnScheme.APPARENT_POWER, Path.of("its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf"), FileType.CSV), + (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226"), FileType.CSV), + (UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of("its_pqh_5022a70e-a58f-4bac-b8ec-1c62376c216b"), FileType.CSV), + (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1"), FileType.CSV), + (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), FileType.CSV) ] when: @@ -400,9 +401,9 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta { def "The CsvDataSource is able to build correct uuid to meta information mapping when restricting column schemes"() { given: def expected = [ - (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), - (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")), - (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")) + (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1"), FileType.CSV), + (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), FileType.CSV), + (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226"), FileType.CSV) ] when: @@ -435,3 +436,4 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta { actual.get("g0").fullFilePath == Path.of("lpts_g0") } } + diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy index 033ffaa87..ff8bbbd62 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy @@ -9,6 +9,7 @@ import static edu.ie3.datamodel.models.StandardUnits.ENERGY_PRICE import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.factory.timeseries.TimeBasedSimpleValueFactory +import edu.ie3.datamodel.io.file.FileType import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation @@ -98,7 +99,7 @@ class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { def "The factory method in csv time series source refuses to build time series with unsupported column type"() { given: - def metaInformation = new FileIndividualTimeSeriesMetaInformation(UUID.fromString("8bc9120d-fb9b-4484-b4e3-0cdadf0feea9"), ColumnScheme.WEATHER, Path.of("its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9")) + def metaInformation = new FileIndividualTimeSeriesMetaInformation(UUID.fromString("8bc9120d-fb9b-4484-b4e3-0cdadf0feea9"), ColumnScheme.WEATHER, Path.of("its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9"), FileType.CSV) when: CsvTimeSeriesSource.getSource(";", timeSeriesFolderPath, fileNamingStrategy, metaInformation) @@ -110,7 +111,7 @@ class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { def "The factory method in csv time series source builds a time series source for all supported column types"() { given: - def metaInformation = new FileIndividualTimeSeriesMetaInformation(uuid, columnScheme, path) + def metaInformation = new FileIndividualTimeSeriesMetaInformation(uuid, columnScheme, path, FileType.CSV) when: def actual = CsvTimeSeriesSource.getSource(";", timeSeriesFolderPath, fileNamingStrategy, metaInformation) @@ -129,4 +130,5 @@ class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { UUID.fromString("46be1e57-e4ed-4ef7-95f1-b2b321cb2047") | ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND | Path.of("its_pqh_46be1e57-e4ed-4ef7-95f1-b2b321cb2047") || 2 | HeatAndSValue UUID.fromString("eeccbe3c-a47e-448e-8eca-1f369d3c24e6") | ColumnScheme.VOLTAGE | Path.of("its_v_eeccbe3c-a47e-448e-8eca-1f369d3c24e6") || 2 | VoltageValue } -} \ No newline at end of file +} + diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy index 8904cc4d7..c46ef24e2 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy @@ -7,6 +7,7 @@ package edu.ie3.datamodel.io.source.csv import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme +import edu.ie3.datamodel.io.file.FileType import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.io.source.file.FileTimeSeriesMetaInformationSource import spock.lang.Shared @@ -25,14 +26,14 @@ class FileTimeSeriesMetaInformationSourceIT extends Specification implements Csv def "A CSV time series meta information source returns correct mapping of time series"() { given: def expectedTimeSeries = Set.of( - new FileIndividualTimeSeriesMetaInformation(UUID.fromString("2fcb3e53-b94a-4b96-bea4-c469e499f1a1"), ColumnScheme.ENERGY_PRICE, Path.of('its_c_2fcb3e53-b94a-4b96-bea4-c469e499f1a1')), - new FileIndividualTimeSeriesMetaInformation(UUID.fromString("76c9d846-797c-4f07-b7ec-2245f679f5c7"), ColumnScheme.ACTIVE_POWER_AND_HEAT_DEMAND, Path.of('its_ph_76c9d846-797c-4f07-b7ec-2245f679f5c7')), - new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0"), ColumnScheme.HEAT_DEMAND, Path.of('its_h_c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0')), - new FileIndividualTimeSeriesMetaInformation(UUID.fromString("9185b8c1-86ba-4a16-8dea-5ac898e8caa5"), ColumnScheme.ACTIVE_POWER, Path.of('its_p_9185b8c1-86ba-4a16-8dea-5ac898e8caa5')), - new FileIndividualTimeSeriesMetaInformation(UUID.fromString("3fbfaa97-cff4-46d4-95ba-a95665e87c26"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_3fbfaa97-cff4-46d4-95ba-a95665e87c26')), - new FileIndividualTimeSeriesMetaInformation(UUID.fromString("46be1e57-e4ed-4ef7-95f1-b2b321cb2047"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of('its_pqh_46be1e57-e4ed-4ef7-95f1-b2b321cb2047')), - new FileIndividualTimeSeriesMetaInformation(UUID.fromString("1061af70-1c03-46e1-b960-940b956c429f"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_1061af70-1c03-46e1-b960-940b956c429f')), - new FileIndividualTimeSeriesMetaInformation(UUID.fromString("eeccbe3c-a47e-448e-8eca-1f369d3c24e6"), ColumnScheme.VOLTAGE, Path.of("its_v_eeccbe3c-a47e-448e-8eca-1f369d3c24e6")) + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("2fcb3e53-b94a-4b96-bea4-c469e499f1a1"), ColumnScheme.ENERGY_PRICE, Path.of('its_c_2fcb3e53-b94a-4b96-bea4-c469e499f1a1'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("76c9d846-797c-4f07-b7ec-2245f679f5c7"), ColumnScheme.ACTIVE_POWER_AND_HEAT_DEMAND, Path.of('its_ph_76c9d846-797c-4f07-b7ec-2245f679f5c7'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0"), ColumnScheme.HEAT_DEMAND, Path.of('its_h_c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("9185b8c1-86ba-4a16-8dea-5ac898e8caa5"), ColumnScheme.ACTIVE_POWER, Path.of('its_p_9185b8c1-86ba-4a16-8dea-5ac898e8caa5'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("3fbfaa97-cff4-46d4-95ba-a95665e87c26"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_3fbfaa97-cff4-46d4-95ba-a95665e87c26'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("46be1e57-e4ed-4ef7-95f1-b2b321cb2047"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of('its_pqh_46be1e57-e4ed-4ef7-95f1-b2b321cb2047'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("1061af70-1c03-46e1-b960-940b956c429f"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_1061af70-1c03-46e1-b960-940b956c429f'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("eeccbe3c-a47e-448e-8eca-1f369d3c24e6"), ColumnScheme.VOLTAGE, Path.of("its_v_eeccbe3c-a47e-448e-8eca-1f369d3c24e6"), FileType.CSV) ) when: @@ -76,3 +77,4 @@ class FileTimeSeriesMetaInformationSourceIT extends Specification implements Csv result.empty } } + From 0d8d35dddbb9b47ba84b28464f1228f4dcb4ef16 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 6 Oct 2025 16:47:02 +0200 Subject: [PATCH 08/10] spotless --- .../edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy | 3 +-- .../ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy | 1 - .../io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy index 725408305..2dc883c51 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy @@ -7,8 +7,8 @@ package edu.ie3.datamodel.io.source.csv import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.connectors.CsvFileConnector -import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.file.FileType +import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.models.input.system.LoadInput @@ -436,4 +436,3 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta { actual.get("g0").fullFilePath == Path.of("lpts_g0") } } - diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy index ff8bbbd62..0be466bc6 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy @@ -131,4 +131,3 @@ class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { UUID.fromString("eeccbe3c-a47e-448e-8eca-1f369d3c24e6") | ColumnScheme.VOLTAGE | Path.of("its_v_eeccbe3c-a47e-448e-8eca-1f369d3c24e6") || 2 | VoltageValue } } - diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy index c46ef24e2..1afaaa26c 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy @@ -5,9 +5,9 @@ */ package edu.ie3.datamodel.io.source.csv +import edu.ie3.datamodel.io.file.FileType import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme -import edu.ie3.datamodel.io.file.FileType import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.io.source.file.FileTimeSeriesMetaInformationSource import spock.lang.Shared @@ -77,4 +77,3 @@ class FileTimeSeriesMetaInformationSourceIT extends Specification implements Csv result.empty } } - From fa0ab94c31115a992c1fe818b9cbf5f2cd8427f7 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 6 Oct 2025 17:01:07 +0200 Subject: [PATCH 09/10] little fix in FileType enum --- src/main/java/edu/ie3/datamodel/io/file/FileType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/file/FileType.java b/src/main/java/edu/ie3/datamodel/io/file/FileType.java index 0c6f995dc..b9ac1f3f7 100644 --- a/src/main/java/edu/ie3/datamodel/io/file/FileType.java +++ b/src/main/java/edu/ie3/datamodel/io/file/FileType.java @@ -21,7 +21,7 @@ public enum FileType { public static FileType getFileType(String fileName) throws ParsingException { FileType[] fileTypes = FileType.values(); return Arrays.stream(fileTypes) - .filter(f -> f.fileEnding.equals(fileName)) + .filter(f -> fileName.endsWith(f.fileEnding)) .findFirst() .orElseThrow( () -> From 60b092fd7fbb3a3b488868049faba399447928ff Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 6 Oct 2025 17:15:26 +0200 Subject: [PATCH 10/10] CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d96d3ef44..e276ec064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Extend `GermanVoltageLevelUtils` with more synonymousIds [#143](https://github.com/ie3-institute/PowerSystemDataModel/issues/143) - Change spotless to use googleJavaFormat('1.28.0') [#1409](https://github.com/ie3-institute/PowerSystemDataModel/issues/1409) - Change `TimeSeries` to no longer extend `UniqueEntity` [#1441](https://github.com/ie3-institute/PowerSystemDataModel/issues/1441) - +- Refactor CSV handling into shared file-based infrastructure. [#1450](https://github.com/ie3-institute/PowerSystemDataModel/issues/1445) ## [8.1.0] - 2025-07-25 ### Added