-
Notifications
You must be signed in to change notification settings - Fork 5
/
CsvWindowedTimeSeriesSource.java
258 lines (231 loc) · 9.16 KB
/
CsvWindowedTimeSeriesSource.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
/*
* © 2021. TU Dortmund University,
* Institute of Energy Systems, Energy Efficiency and Energy Economics,
* Research group Distribution grid planning and operation
*/
package edu.ie3.datamodel.io.source.csv;
import edu.ie3.datamodel.exceptions.SourceException;
import edu.ie3.datamodel.io.factory.timeseries.TimeBasedSimpleValueFactory;
import edu.ie3.datamodel.io.naming.FileNamingStrategy;
import edu.ie3.datamodel.io.source.TimeSeriesSource;
import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries;
import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue;
import edu.ie3.datamodel.models.value.Value;
import edu.ie3.util.TimeUtil;
import edu.ie3.util.interval.ClosedInterval;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.time.Duration;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This csv source for {@link IndividualTimeSeries} utilizes the functionalities of a buffered file
* reader. That means, that you can only read in your time series linearly from the beginning to the
* end. As soon, as you have queried an instance in time, you are only able to query future
* instances! <strong>Moreover, you have to remember to close this source or use it within a
* try-with-resources!</strong>
*
* @param <V> Type of value to be returned
*/
public class CsvWindowedTimeSeriesSource<V extends Value> extends TimeSeriesSource<V>
implements AutoCloseable {
private static final Logger logger = LoggerFactory.getLogger(CsvWindowedTimeSeriesSource.class);
private final Duration maximumForeSight;
private ClosedInterval<ZonedDateTime> coveredInterval;
private final String filePath;
private final BufferedReader reader;
private final Stream<TimeBasedValue<V>> inputStream;
private final Map<ZonedDateTime, V> buffer = new HashMap<>();
private final CsvDataSource dataSource;
public CsvWindowedTimeSeriesSource(
String csvSep,
String folderPath,
String filePath,
FileNamingStrategy fileNamingStrategy,
Duration maximumForeSight,
Class<V> valueClass,
TimeBasedSimpleValueFactory<V> factory)
throws SourceException {
this.dataSource = new CsvDataSource(csvSep, folderPath, fileNamingStrategy);
this.maximumForeSight = maximumForeSight;
this.filePath = filePath;
try {
this.reader = dataSource.connector.initReader(filePath);
this.inputStream =
dataSource
.buildStreamWithFieldsToAttributesMap(TimeBasedValue.class, reader)
.map(fieldToValue -> this.buildTimeBasedValue(fieldToValue, valueClass, factory))
.filter(Optional::isPresent)
.map(Optional::get);
} catch (FileNotFoundException e) {
throw new SourceException(
"Opening the reader for time series file '" + filePath + "' failed.", e);
}
}
@Override
public IndividualTimeSeries<V> getTimeSeries() {
throw new UnsupportedOperationException(
"This time series source is not able to return a full time series.");
}
@Override
public IndividualTimeSeries<V> getTimeSeries(ClosedInterval<ZonedDateTime> timeInterval) {
if (!Objects.isNull(coveredInterval)
&& timeInterval.getLower().isBefore(coveredInterval.getLower()))
throw new RuntimeException(
"The buffer window already passed the start '"
+ timeInterval.getLower()
+ "' of your desired time frame.");
if (Objects.isNull(coveredInterval)
|| timeInterval.getUpper().isAfter(coveredInterval.getUpper())) {
/* If the buffer is empty, or you can foresee, that the questioned instance isn't within the buffer, fill it up */
fillUpBuffer(timeInterval.getUpper());
}
Set<TimeBasedValue<V>> tbvs =
getFromBuffer(timeInterval.getLower(), timeInterval.getUpper())
.map(entry -> new TimeBasedValue<>(entry.getKey(), entry.getValue()))
.collect(Collectors.toSet());
/* Reduce the buffer */
shrinkBuffer(timeInterval.getUpper());
return new IndividualTimeSeries<>(UUID.randomUUID(), tbvs);
}
@Override
public Optional<V> getValue(ZonedDateTime time) {
if (!Objects.isNull(coveredInterval) && time.isBefore(coveredInterval.getLower()))
throw new RuntimeException(
"The buffer window already passed your desired time instance '" + time + "'.");
if (Objects.isNull(coveredInterval) || time.isAfter(coveredInterval.getUpper())) {
/* If the buffer is empty, or you can foresee, that the questioned instance isn't within the buffer, fill it up */
fillUpBuffer(time);
}
Optional<V> value = getFromBuffer(time);
/* Reduce the buffer */
shrinkBuffer(time);
return value;
}
/**
* Get the value for the questioned time from the buffer
*
* @param time Questioned time
* @return Optional value, that might be within the buffer
*/
private Optional<V> getFromBuffer(ZonedDateTime time) {
return buffer.entrySet().stream()
.filter(
timeToValue ->
timeToValue.getKey().isBefore(time) || timeToValue.getKey().isEqual(time))
.max(Map.Entry.comparingByKey())
.map(Map.Entry::getValue);
}
/**
* Get a defined time slice from buffer
*
* @param start Start of the slice (included)
* @param end End of the slice (included)
* @return A stream for that slice
*/
private Stream<Map.Entry<ZonedDateTime, V>> getFromBuffer(
ZonedDateTime start, ZonedDateTime end) {
return buffer.entrySet().stream()
.filter(
timeToValue ->
(timeToValue.getKey().isAfter(start) || timeToValue.getKey().isEqual(start))
&& (timeToValue.getKey().isBefore(end) || timeToValue.getKey().isEqual(end)));
}
/**
* Fills up the buffer. This is done by adding the {@link this#maximumForeSight} to the time of
* interest. This marks the newest possible instance in the buffer to appear.
*
* @param timeOfInterest The time of interest
*/
private void fillUpBuffer(ZonedDateTime timeOfInterest) {
ZonedDateTime intendedNewestTimeInBuffer = timeOfInterest.plus(maximumForeSight);
inputStream
.filter(
tbv ->
tbv.getTime().isBefore(intendedNewestTimeInBuffer)
|| tbv.getTime().isEqual(intendedNewestTimeInBuffer))
.forEach(tbv -> buffer.put(tbv.getTime(), tbv.getValue()));
updateNewestTime(intendedNewestTimeInBuffer);
}
/**
* Removes all entries in the buffer, that are older than the provided instance in time.
*
* @param oldestEntry The oldest entry meant to remain within the buffer
*/
private void shrinkBuffer(ZonedDateTime oldestEntry) {
List<ZonedDateTime> timesToRemove =
buffer.keySet().stream().filter(bufferTime -> bufferTime.isBefore(oldestEntry)).toList();
timesToRemove.forEach(buffer::remove);
updateOldestTime(oldestEntry);
}
/**
* Expand the covered time frame
*
* @param time Newest covered time
*/
private void updateNewestTime(ZonedDateTime time) {
if (Objects.isNull(coveredInterval)) {
coveredInterval = new ClosedInterval<>(time, time);
} else {
coveredInterval = new ClosedInterval<>(coveredInterval.getLower(), time);
}
}
/**
* Reduce the covered time frame
*
* @param time Oldest covered time
*/
private void updateOldestTime(ZonedDateTime time) {
if (Objects.isNull(coveredInterval)) {
coveredInterval = new ClosedInterval<>(time, time);
} else {
coveredInterval = new ClosedInterval<>(time, coveredInterval.getUpper());
}
}
/**
* Determines all available time steps within the given time series
*
* @return A list of available {@link ZonedDateTime}s
*/
public List<ZonedDateTime> getAvailableTimeSteps() {
return getAvailableTimeSteps("time", "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'");
}
/**
* Determines all available time steps within the given time series
*
* @param timeField Field, where date time information is located
* @param timePattern Pattern of the date time strings
* @return A list of available {@link ZonedDateTime}s
*/
public List<ZonedDateTime> getAvailableTimeSteps(String timeField, String timePattern) {
TimeUtil timeUtil = new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, timePattern);
try (BufferedReader reader = dataSource.connector.initReader(this.filePath)) {
return dataSource
.buildStreamWithFieldsToAttributesMap(TimeBasedValue.class, reader)
.map(
fieldToValue ->
Optional.ofNullable(fieldToValue.get(timeField)).map(timeUtil::toZonedDateTime))
.filter(Optional::isPresent)
.map(Optional::get)
.toList();
} catch (IOException e) {
logger.warn(
"Opening a reader for time series file '"
+ filePath
+ "' failed. Unable to determine available time steps.",
e);
}
return Collections.emptyList();
}
@Override
public void close() throws Exception {
inputStream.close();
reader.close();
}
}