From 86c4d1f078dbc5912691b5c70e06982c4b5188d7 Mon Sep 17 00:00:00 2001 From: Johannes Hiry Date: Fri, 21 May 2021 14:13:04 +0200 Subject: [PATCH 1/3] kill deprecated TimeSeriesReadingData --- .../io/connectors/CsvFileConnector.java | 176 ++++++------------ .../io/source/csv/CsvWeatherSource.java | 66 ++++--- .../io/connectors/CsvFileConnectorTest.groovy | 34 +--- 3 files changed, 99 insertions(+), 177 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index 66ad4f05e..4814a69a4 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -221,70 +221,27 @@ public Optional getIndividualTimeSeriesMeta } /** - * Initialises the readers for time series with the specified column schemes. They are given back - * grouped by the column scheme in order to allow for accounting the different content types. + * Receive the information for specific time series They are given back grouped by the column + * scheme in order to allow for accounting the different content types. * * @param columnSchemes the column schemes to initialize readers for. If no scheme is given, all * possible readers will be initialized. - * @return A mapping from column type to respective readers - * @deprecated Don't use {@link TimeSeriesReadingData}, as it contains a reader, that might not be - * closed + * @return A mapping from column scheme to the individual time series meta information */ - @Deprecated - public Map> initTimeSeriesReader( - ColumnScheme... columnSchemes) { + public Map> + getCsvIndividualTimeSeriesMetaInformation(ColumnScheme... columnSchemes) { return getIndividualTimeSeriesFilePaths() .parallelStream() .map( pathString -> { String filePathWithoutEnding = removeFileEnding(pathString); - return buildReadingData(filePathWithoutEnding, columnSchemes); + return buildCsvTimeSeriesMetaInformation(filePathWithoutEnding, columnSchemes); }) .filter(Optional::isPresent) .map(Optional::get) - .collect(Collectors.groupingBy(TimeSeriesReadingData::getColumnScheme, Collectors.toSet())); - } - - /** - * Initialises a reader to get grip on the file that contains mapping information between - * coordinate id and actual coordinate - * - * @return A {@link BufferedReader} - * @throws FileNotFoundException If the file is not present - */ - public BufferedReader initIdCoordinateReader() throws FileNotFoundException { - String filePath = entityPersistenceNamingStrategy.getIdCoordinateEntityName(); - return initReader(filePath); - } - - /** - * Returns a set of relative paths strings to time series files, with respect to the base folder - * path - * - * @return A set of relative paths to time series files, with respect to the base folder path - */ - private Set getIndividualTimeSeriesFilePaths() { - Path baseDirectoryPath = - Paths.get( - FilenameUtils.getFullPath(baseDirectoryName) - + FilenameUtils.getName(baseDirectoryName)); - try (Stream pathStream = Files.walk(baseDirectoryPath)) { - return pathStream - .map(baseDirectoryPath::relativize) - .filter( - path -> { - String withoutEnding = removeFileEnding(path.toString()); - return entityPersistenceNamingStrategy - .getIndividualTimeSeriesPattern() - .matcher(withoutEnding) - .matches(); - }) - .map(Path::toString) - .collect(Collectors.toSet()); - } catch (IOException e) { - log.error("Unable to determine time series files readers for time series.", e); - return Collections.emptySet(); - } + .collect( + Collectors.groupingBy( + CsvIndividualTimeSeriesMetaInformation::getColumnScheme, Collectors.toSet())); } /** @@ -295,12 +252,9 @@ private Set getIndividualTimeSeriesFilePaths() { * @param filePathString String describing the path to the time series file * @param columnSchemes the allowed column schemes. If no scheme is specified, all schemes are * allowed. - * @return An {@link Optional} to {@link TimeSeriesReadingData} - * @deprecated Don't use {@link TimeSeriesReadingData}, as it contains a reader, that might not be - * closed + * @return An {@link Optional} to {@link IndividualTimeSeriesMetaInformation} */ - @Deprecated - private Optional buildReadingData( + private Optional buildCsvTimeSeriesMetaInformation( String filePathString, ColumnScheme... columnSchemes) { try { FileNameMetaInformation metaInformation = @@ -327,16 +281,11 @@ private Optional buildReadingData( columnSchemes); return Optional.empty(); } - - BufferedReader reader = initReader(filePathString); return Optional.of( - new TimeSeriesReadingData( + new CsvIndividualTimeSeriesMetaInformation( individualMetaInformation.getUuid(), individualMetaInformation.getColumnScheme(), - reader)); - } catch (FileNotFoundException e) { - log.error("Cannot init the writer for time series file path '{}'.", filePathString, e); - return Optional.empty(); + filePathString)); } catch (IllegalArgumentException e) { log.error( "Error during extraction of meta information from file name '{}'.", filePathString, e); @@ -344,6 +293,48 @@ private Optional buildReadingData( } } + /** + * Initialises a reader to get grip on the file that contains mapping information between + * coordinate id and actual coordinate + * + * @return A {@link BufferedReader} + * @throws FileNotFoundException If the file is not present + */ + public BufferedReader initIdCoordinateReader() throws FileNotFoundException { + String filePath = entityPersistenceNamingStrategy.getIdCoordinateEntityName(); + return initReader(filePath); + } + + /** + * Returns a set of relative paths strings to time series files, with respect to the base folder + * path + * + * @return A set of relative paths to time series files, with respect to the base folder path + */ + private Set getIndividualTimeSeriesFilePaths() { + Path baseDirectoryPath = + Paths.get( + FilenameUtils.getFullPath(baseDirectoryName) + + FilenameUtils.getName(baseDirectoryName)); + try (Stream pathStream = Files.walk(baseDirectoryPath)) { + return pathStream + .map(baseDirectoryPath::relativize) + .filter( + path -> { + String withoutEnding = removeFileEnding(path.toString()); + return entityPersistenceNamingStrategy + .getIndividualTimeSeriesPattern() + .matcher(withoutEnding) + .matches(); + }) + .map(Path::toString) + .collect(Collectors.toSet()); + } catch (IOException e) { + log.error("Unable to determine time series files readers for time series.", e); + return Collections.emptySet(); + } + } + /** * Removes the file ending from input string * @@ -414,63 +405,6 @@ public void shutdown() { }); } - /** - * Class to bundle all information, that are necessary to read a single time series - * - * @deprecated Use the {@link CsvIndividualTimeSeriesMetaInformation} and build reader on demand - */ - @Deprecated - public static class TimeSeriesReadingData { - private final UUID uuid; - private final ColumnScheme columnScheme; - private final BufferedReader reader; - - public TimeSeriesReadingData(UUID uuid, ColumnScheme columnScheme, BufferedReader reader) { - this.uuid = uuid; - this.columnScheme = columnScheme; - this.reader = reader; - } - - public UUID getUuid() { - return uuid; - } - - public ColumnScheme getColumnScheme() { - return columnScheme; - } - - public BufferedReader getReader() { - return reader; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof TimeSeriesReadingData)) return false; - TimeSeriesReadingData that = (TimeSeriesReadingData) o; - return uuid.equals(that.uuid) - && columnScheme == that.columnScheme - && reader.equals(that.reader); - } - - @Override - public int hashCode() { - return Objects.hash(uuid, columnScheme, reader); - } - - @Override - public String toString() { - return "TimeSeriesReadingData{" - + "uuid=" - + uuid - + ", columnScheme=" - + columnScheme - + ", reader=" - + reader - + '}'; - } - } - /** Enhancing the {@link IndividualTimeSeriesMetaInformation} with the full path to csv file */ public static class CsvIndividualTimeSeriesMetaInformation extends IndividualTimeSeriesMetaInformation { diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 123aab015..30b152fa4 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -21,6 +21,7 @@ import edu.ie3.datamodel.utils.TimeSeriesUtil; import edu.ie3.util.interval.ClosedInterval; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.time.ZonedDateTime; import java.util.*; @@ -95,15 +96,16 @@ public CsvWeatherSource( * @return a map of coordinates to their time series */ private Map> getWeatherTimeSeries() { - /* Get only weather time series reader */ - Map> colTypeToReadingData = - connector.initTimeSeriesReader(ColumnScheme.WEATHER); + /* Get only weather time series meta information */ + Map> + colTypeToMetaData = + connector.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER); /* Reading in weather time series */ - Set weatherReadingData = - colTypeToReadingData.get(ColumnScheme.WEATHER); + Set weatherCsvMetaInformation = + colTypeToMetaData.get(ColumnScheme.WEATHER); - return readWeatherTimeSeries(weatherReadingData); + return readWeatherTimeSeries(weatherCsvMetaInformation, connector); } @Override @@ -151,35 +153,43 @@ private Map> trimMapToInterval( /** * Reads weather data to time series and maps them coordinate wise * - * @param weatherReadingData Data needed for reading + * @param weatherMetaInformation Data needed for reading * @return time series mapped to the represented coordinate */ private Map> readWeatherTimeSeries( - Set weatherReadingData) { + Set weatherMetaInformation, + CsvFileConnector connector) { final Map> weatherTimeSeries = new HashMap<>(); Function, Optional>> fieldToValueFunction = this::buildWeatherValue; - /* Reading in weather time series */ - for (CsvFileConnector.TimeSeriesReadingData data : weatherReadingData) { - filterEmptyOptionals( - buildStreamWithFieldsToAttributesMap(TimeBasedValue.class, data.getReader()) - .map(fieldToValueFunction)) - .collect(Collectors.groupingBy(tbv -> tbv.getValue().getCoordinate())) - .forEach( - (point, timeBasedValues) -> { - // We have to generate a random UUID as we'd risk running into duplicate key issues - // otherwise - IndividualTimeSeries timeSeries = - new IndividualTimeSeries<>(UUID.randomUUID(), new HashSet<>(timeBasedValues)); - if (weatherTimeSeries.containsKey(point)) { - IndividualTimeSeries mergedTimeSeries = - mergeTimeSeries(weatherTimeSeries.get(point), timeSeries); - weatherTimeSeries.put(point, mergedTimeSeries); - } else { - weatherTimeSeries.put(point, timeSeries); - } - }); + for (CsvFileConnector.CsvIndividualTimeSeriesMetaInformation data : weatherMetaInformation) { + // we need a reader for each file + try (BufferedReader reader = connector.initReader(data.getFullFilePath())) { + filterEmptyOptionals( + buildStreamWithFieldsToAttributesMap(TimeBasedValue.class, reader) + .map(fieldToValueFunction)) + .collect(Collectors.groupingBy(tbv -> tbv.getValue().getCoordinate())) + .forEach( + (point, timeBasedValues) -> { + // We have to generate a random UUID as we'd risk running into duplicate key + // issues + // otherwise + IndividualTimeSeries timeSeries = + new IndividualTimeSeries<>(UUID.randomUUID(), new HashSet<>(timeBasedValues)); + if (weatherTimeSeries.containsKey(point)) { + IndividualTimeSeries mergedTimeSeries = + mergeTimeSeries(weatherTimeSeries.get(point), timeSeries); + weatherTimeSeries.put(point, mergedTimeSeries); + } else { + weatherTimeSeries.put(point, timeSeries); + } + }); + } catch (FileNotFoundException e) { + log.error("Cannot read file {}. File not found!", data.getFullFilePath()); + } catch (IOException e) { + log.error("Cannot read file {}. Exception: {}", data.getFullFilePath(), e); + } } return weatherTimeSeries; } diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy index 8b73dabcf..8f62507f3 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy @@ -115,39 +115,28 @@ class CsvFileConnectorTest extends Specification { actual == expected } - def "The csv file connector returns empty Optional of TimeSeriesReadingData when pointed to non-individual time series"() { + def "The csv file connector returns empty Optional of CsvTimeSeriesMetaInformation when pointed to non-individual time series"() { given: def pathString = "lpts_h0_53990eea-1b5d-47e8-9134-6d8de36604bf" when: - def actual = cfc.buildReadingData(pathString) + def actual = cfc.buildCsvTimeSeriesMetaInformation(pathString) then: !actual.present } - def "The csv file connector returns empty Optional of TimeSeriesReadingData when pointed to non-existing file"() { - given: - def pathString = "its_pq_32f38421-f7fd-4295-8f9a-3a54b4e7dba9" - - when: - def actual = cfc.buildReadingData(pathString) - - then: - !actual.present - } - - def "The csv file connector is able to build correct reading information from valid input"() { + def "The csv file connector is able to build correct meta information from valid input"() { given: def pathString = "its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf" - def expected = new CsvFileConnector.TimeSeriesReadingData( + def expected = new CsvFileConnector.CsvIndividualTimeSeriesMetaInformation( UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf"), ColumnScheme.APPARENT_POWER, - Mock(BufferedReader) + "" ) when: - def actual = cfc.buildReadingData(pathString) + def actual = cfc.buildCsvTimeSeriesMetaInformation(pathString) then: actual.present @@ -158,17 +147,6 @@ class CsvFileConnectorTest extends Specification { } } - def "The csv file connector is able to init readers for all time series files"() { - when: - def actual = cfc.initTimeSeriesReader() - - then: - actual.size() == 5 - def energyPriceEntries = actual.get(ColumnScheme.ENERGY_PRICE) - Objects.nonNull(energyPriceEntries) - energyPriceEntries.size() == 2 - } - def "The csv file connector throws an Exception, if the foreseen file cannot be found"() { given: def cfc = new CsvFileConnector(tmpDirectory.toString(), new HierarchicFileNamingStrategy(new DefaultDirectoryHierarchy(tmpDirectory.toString(), "test"))) From 606f7317b535535d342da46d9dd9e86d391c7322 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 21 May 2021 15:24:16 +0200 Subject: [PATCH 2/3] Rearranging methods --- .../io/connectors/CsvFileConnector.java | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index 4814a69a4..e4bd141ae 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -221,7 +221,7 @@ public Optional getIndividualTimeSeriesMeta } /** - * Receive the information for specific time series They are given back grouped by the column + * Receive the information for specific time series. They are given back grouped by the column * scheme in order to allow for accounting the different content types. * * @param columnSchemes the column schemes to initialize readers for. If no scheme is given, all @@ -244,6 +244,46 @@ public Optional getIndividualTimeSeriesMeta CsvIndividualTimeSeriesMetaInformation::getColumnScheme, Collectors.toSet())); } + /** + * Removes the file ending from input string + * + * @param input String to manipulate + * @return input without possible ending + */ + private String removeFileEnding(String input) { + return input.replaceAll(FILE_ENDING + "$", ""); + } + + /** + * Returns a set of relative paths strings to time series files, with respect to the base folder + * path + * + * @return A set of relative paths to time series files, with respect to the base folder path + */ + private Set getIndividualTimeSeriesFilePaths() { + Path baseDirectoryPath = + Paths.get( + FilenameUtils.getFullPath(baseDirectoryName) + + FilenameUtils.getName(baseDirectoryName)); + try (Stream pathStream = Files.walk(baseDirectoryPath)) { + return pathStream + .map(baseDirectoryPath::relativize) + .filter( + path -> { + String withoutEnding = removeFileEnding(path.toString()); + return entityPersistenceNamingStrategy + .getIndividualTimeSeriesPattern() + .matcher(withoutEnding) + .matches(); + }) + .map(Path::toString) + .collect(Collectors.toSet()); + } catch (IOException e) { + log.error("Unable to determine time series files readers for time series.", e); + return Collections.emptySet(); + } + } + /** * Compose the needed information for reading in a single time series. If the file points to a * non-individual time series or a time series of a column scheme other than the specified ones, @@ -305,46 +345,6 @@ public BufferedReader initIdCoordinateReader() throws FileNotFoundException { return initReader(filePath); } - /** - * Returns a set of relative paths strings to time series files, with respect to the base folder - * path - * - * @return A set of relative paths to time series files, with respect to the base folder path - */ - private Set getIndividualTimeSeriesFilePaths() { - Path baseDirectoryPath = - Paths.get( - FilenameUtils.getFullPath(baseDirectoryName) - + FilenameUtils.getName(baseDirectoryName)); - try (Stream pathStream = Files.walk(baseDirectoryPath)) { - return pathStream - .map(baseDirectoryPath::relativize) - .filter( - path -> { - String withoutEnding = removeFileEnding(path.toString()); - return entityPersistenceNamingStrategy - .getIndividualTimeSeriesPattern() - .matcher(withoutEnding) - .matches(); - }) - .map(Path::toString) - .collect(Collectors.toSet()); - } catch (IOException e) { - log.error("Unable to determine time series files readers for time series.", e); - return Collections.emptySet(); - } - } - - /** - * Removes the file ending from input string - * - * @param input String to manipulate - * @return input without possible ending - */ - private String removeFileEnding(String input) { - return input.replaceAll(FILE_ENDING + "$", ""); - } - /** * Builds a new file definition consisting of file name and head line elements * From 73354203fe32423f0034d33bbe9f1a75b31be026 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 21 May 2021 16:10:50 +0200 Subject: [PATCH 3/3] Hand in a hotfix to reduce technical debt --- version.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/version.properties b/version.properties index 1dc344a4f..40ef89d58 100644 --- a/version.properties +++ b/version.properties @@ -1,8 +1,8 @@ #Generated by the Semver Plugin for Gradle -#Mon Sep 14 12:33:33 CEST 2020 +#Fri May 21 16:10:18 CEST 2021 version.buildmeta= version.major=2 version.minor=0 -version.patch=0 +version.patch=1 version.prerelease= -version.semver=2.0.0 +version.semver=2.0.1