-
Notifications
You must be signed in to change notification settings - Fork 5
/
CsvRawGridSource.java
517 lines (470 loc) · 20.7 KB
/
CsvRawGridSource.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
/*
* © 2021. TU Dortmund University,
* Institute of Energy Systems, Energy Efficiency and Energy Economics,
* Research group Distribution grid planning and operation
*/
package edu.ie3.datamodel.io.source.csv;
import edu.ie3.datamodel.exceptions.FactoryException;
import edu.ie3.datamodel.io.factory.EntityFactory;
import edu.ie3.datamodel.io.factory.FactoryData;
import edu.ie3.datamodel.io.factory.input.*;
import edu.ie3.datamodel.io.naming.FileNamingStrategy;
import edu.ie3.datamodel.io.source.RawGridSource;
import edu.ie3.datamodel.io.source.TypeSource;
import edu.ie3.datamodel.models.input.*;
import edu.ie3.datamodel.models.input.connector.*;
import edu.ie3.datamodel.models.input.connector.type.LineTypeInput;
import edu.ie3.datamodel.models.input.connector.type.Transformer2WTypeInput;
import edu.ie3.datamodel.models.input.connector.type.Transformer3WTypeInput;
import edu.ie3.datamodel.models.input.container.RawGridElements;
import edu.ie3.datamodel.utils.options.Try;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Source that provides the capability to build entities that are hold by a {@link RawGridElements}
* as well as the {@link RawGridElements} container from .csv files.
*
* <p>This source is <b>not buffered</b> which means each call on a getter method always tries to
* read all data is necessary to return the requested objects in a hierarchical cascading way.
*
* <p>If performance is an issue, it is recommended to read the data cascading starting with reading
* nodes and then using the getters with arguments to avoid reading the same data multiple times.
*
* <p>The resulting sets are always unique on object <b>and</b> UUID base (with distinct UUIDs).
*
* @version 0.1
* @since 03.04.20
*/
public class CsvRawGridSource extends CsvDataSource implements RawGridSource {
// general fields
private final TypeSource typeSource;
// factories
private final NodeInputFactory nodeInputFactory;
private final LineInputFactory lineInputFactory;
private final Transformer2WInputFactory transformer2WInputFactory;
private final Transformer3WInputFactory transformer3WInputFactory;
private final SwitchInputFactory switchInputFactory;
private final MeasurementUnitInputFactory measurementUnitInputFactory;
public CsvRawGridSource(
String csvSep,
String gridFolderPath,
FileNamingStrategy fileNamingStrategy,
TypeSource typeSource) {
super(csvSep, gridFolderPath, fileNamingStrategy);
this.typeSource = typeSource;
// init factories
this.nodeInputFactory = new NodeInputFactory();
this.lineInputFactory = new LineInputFactory();
this.transformer2WInputFactory = new Transformer2WInputFactory();
this.transformer3WInputFactory = new Transformer3WInputFactory();
this.switchInputFactory = new SwitchInputFactory();
this.measurementUnitInputFactory = new MeasurementUnitInputFactory();
}
/** {@inheritDoc} */
@Override
public RawGridElements getGridData() {
/* read all needed entities start with the types and operators */
Set<OperatorInput> operators = typeSource.getOperators();
Set<LineTypeInput> lineTypes = typeSource.getLineTypes();
Set<Transformer2WTypeInput> transformer2WTypeInputs = typeSource.getTransformer2WTypes();
Set<Transformer3WTypeInput> transformer3WTypeInputs = typeSource.getTransformer3WTypes();
/* assets */
Set<NodeInput> nodes = getNodes(operators);
Set<LineInput> lineInputs = getLines(nodes, lineTypes, operators);
Set<Transformer2WInput> transformer2WInputs =
get2WTransformers(nodes, transformer2WTypeInputs, operators);
Set<Transformer3WInput> transformer3WInputs =
get3WTransformers(nodes, transformer3WTypeInputs, operators);
Set<SwitchInput> switches = getSwitches(nodes, operators);
Set<MeasurementUnitInput> measurementUnits = getMeasurementUnits(nodes, operators);
/* build and return the grid if it is not empty */
return new RawGridElements(
nodes, lineInputs, transformer2WInputs, transformer3WInputs, switches, measurementUnits);
}
/** {@inheritDoc} */
@Override
public Set<NodeInput> getNodes() {
return getNodes(typeSource.getOperators());
}
/**
* {@inheritDoc}
*
* <p>If the set with {@link OperatorInput} is not exhaustive, the corresponding operator is set
* to {@link OperatorInput#NO_OPERATOR_ASSIGNED}
*/
@Override
public Set<NodeInput> getNodes(Set<OperatorInput> operators) {
return Try.scanForExceptions(
assetInputEntityDataStream(NodeInput.class, operators)
.map(nodeInputFactory::get)
.collect(Collectors.toSet()),
NodeInput.class)
.get();
}
/** {@inheritDoc} */
@Override
public Set<LineInput> getLines() {
Set<OperatorInput> operators = typeSource.getOperators();
return getLines(getNodes(operators), typeSource.getLineTypes(), operators);
}
/**
* {@inheritDoc}
*
* <p>If one of the sets of {@link NodeInput} or {@link LineTypeInput} entities is not exhaustive
* for all available {@link LineInput} entities (e.g. a {@link NodeInput} or {@link LineTypeInput}
* entity is missing) or if an error during the building process occurs, the entity that misses
* something will be skipped (which can be seen as a filtering functionality) but all entities
* that are able to be built will be returned anyway and the elements that couldn't have been
* built are logged.
*
* <p>If the set with {@link OperatorInput} is not exhaustive, the corresponding operator is set
* to {@link OperatorInput#NO_OPERATOR_ASSIGNED}
*/
@Override
public Set<LineInput> getLines(
Set<NodeInput> nodes, Set<LineTypeInput> lineTypeInputs, Set<OperatorInput> operators) {
return Try.scanForExceptions(
typedEntityStream(LineInput.class, lineInputFactory, nodes, operators, lineTypeInputs)
.collect(Collectors.toSet()),
LineInput.class)
.get();
}
/** {@inheritDoc} */
@Override
public Set<Transformer2WInput> get2WTransformers() {
Set<OperatorInput> operators = typeSource.getOperators();
return get2WTransformers(getNodes(operators), typeSource.getTransformer2WTypes(), operators);
}
/**
* {@inheritDoc}
*
* <p>If one of the sets of {@link NodeInput} or {@link Transformer2WTypeInput} entities is not
* exhaustive for all available {@link Transformer2WInput} entities (e.g. a {@link NodeInput} or
* {@link Transformer2WTypeInput} entity is missing) or if an error during the building process
* occurs, the entity that misses something will be skipped (which can be seen as a filtering
* functionality) but all entities that are able to be built will be returned anyway and the
* elements that couldn't have been built are logged.
*
* <p>If the set with {@link OperatorInput} is not exhaustive, the corresponding operator is set
* to {@link OperatorInput#NO_OPERATOR_ASSIGNED}
*/
@Override
public Set<Transformer2WInput> get2WTransformers(
Set<NodeInput> nodes,
Set<Transformer2WTypeInput> transformer2WTypes,
Set<OperatorInput> operators) {
return Try.scanForExceptions(
typedEntityStream(
Transformer2WInput.class,
transformer2WInputFactory,
nodes,
operators,
transformer2WTypes)
.collect(Collectors.toSet()),
Transformer2WInput.class)
.get();
}
/** {@inheritDoc} */
@Override
public Set<Transformer3WInput> get3WTransformers() {
Set<OperatorInput> operators = typeSource.getOperators();
return get3WTransformers(getNodes(operators), typeSource.getTransformer3WTypes(), operators);
}
/**
* {@inheritDoc}
*
* <p>If one of the sets of {@link NodeInput} or {@link Transformer3WTypeInput} entities is not
* exhaustive for all available {@link Transformer3WInput} entities (e.g. a {@link NodeInput} or
* {@link Transformer3WTypeInput} entity is missing) or if an error during the building process
* occurs, the entity that misses something will be skipped (which can be seen as a filtering
* functionality) but all entities that are able to be built will be returned anyway and the
* elements that couldn't have been built are logged.
*
* <p>If the set with {@link OperatorInput} is not exhaustive, the corresponding operator is set
* to {@link OperatorInput#NO_OPERATOR_ASSIGNED}
*/
@Override
public Set<Transformer3WInput> get3WTransformers(
Set<NodeInput> nodes,
Set<Transformer3WTypeInput> transformer3WTypeInputs,
Set<OperatorInput> operators) {
return Try.scanForExceptions(
transformer3WEntityStream(nodes, transformer3WTypeInputs, operators)
.collect(Collectors.toSet()),
Transformer3WInput.class)
.get();
}
private Stream<Try<Transformer3WInput, FactoryException>> transformer3WEntityStream(
Set<NodeInput> nodes,
Set<Transformer3WTypeInput> transformer3WTypeInputs,
Set<OperatorInput> operators) {
return buildTransformer3WEntityData(
buildTypedConnectorEntityData(
buildUntypedConnectorInputEntityData(
assetInputEntityDataStream(Transformer3WInput.class, operators), nodes),
transformer3WTypeInputs),
nodes)
.filter(Optional::isPresent)
.map(Optional::get)
.map(transformer3WInputFactory::get);
}
/** {@inheritDoc} */
@Override
public Set<SwitchInput> getSwitches() {
Set<OperatorInput> operators = typeSource.getOperators();
return getSwitches(getNodes(operators), operators);
}
/**
* {@inheritDoc}
*
* <p>If one of the sets of {@link NodeInput} entities is not exhaustive for all available {@link
* SwitchInput} entities (e.g. a {@link NodeInput} entity is missing) or if an error during the
* building process occurs, the entity that misses something will be skipped (which can be seen as
* a filtering functionality) but all entities that are able to be built will be returned anyway
* and the elements that couldn't have been built are logged.
*
* <p>If the set with {@link OperatorInput} is not exhaustive, the corresponding operator is set
* to {@link OperatorInput#NO_OPERATOR_ASSIGNED}
*/
@Override
public Set<SwitchInput> getSwitches(Set<NodeInput> nodes, Set<OperatorInput> operators) {
return Try.scanForExceptions(
untypedConnectorInputEntityStream(
SwitchInput.class, switchInputFactory, nodes, operators)
.collect(Collectors.toSet()),
SwitchInput.class)
.get();
}
private <T extends ConnectorInput>
Stream<Try<T, FactoryException>> untypedConnectorInputEntityStream(
Class<T> entityClass,
EntityFactory<T, ConnectorInputEntityData> factory,
Set<NodeInput> nodes,
Set<OperatorInput> operators) {
return buildUntypedConnectorInputEntityData(
assetInputEntityDataStream(entityClass, operators), nodes)
.filter(Optional::isPresent)
.map(Optional::get)
.map(factory::get);
}
/** {@inheritDoc} */
@Override
public Set<MeasurementUnitInput> getMeasurementUnits() {
Set<OperatorInput> operators = typeSource.getOperators();
return getMeasurementUnits(getNodes(operators), operators);
}
/**
* {@inheritDoc}
*
* <p>If one of the sets of {@link NodeInput} entities is not exhaustive for all available {@link
* MeasurementUnitInput} entities (e.g. a {@link NodeInput} entity is missing) or if an error
* during the building process occurs, the entity that misses something will be skipped (which can
* be seen as a filtering functionality) but all entities that are able to be built will be
* returned anyway and the elements that couldn't have been built are logged.
*
* <p>If the set with {@link OperatorInput} is not exhaustive, the corresponding operator is set
* to {@link OperatorInput#NO_OPERATOR_ASSIGNED}
*/
@Override
public Set<MeasurementUnitInput> getMeasurementUnits(
Set<NodeInput> nodes, Set<OperatorInput> operators) {
return Try.scanForExceptions(
nodeAssetEntityStream(
MeasurementUnitInput.class, measurementUnitInputFactory, nodes, operators)
.collect(Collectors.toSet()),
MeasurementUnitInput.class)
.get();
}
private <T extends ConnectorInput, A extends AssetTypeInput>
Stream<Try<T, FactoryException>> typedEntityStream(
Class<T> entityClass,
EntityFactory<T, TypedConnectorInputEntityData<A>> factory,
Collection<NodeInput> nodes,
Collection<OperatorInput> operators,
Collection<A> types) {
return buildTypedConnectorEntityData(
buildUntypedConnectorInputEntityData(
assetInputEntityDataStream(entityClass, operators), nodes),
types)
.filter(Optional::isPresent)
.map(Optional::get)
.map(factory::get);
}
/**
* Converts a stream of {@link AssetInputEntityData} in connection with a collection of known
* {@link NodeInput}s to a stream of {@link ConnectorInputEntityData}.
*
* @param assetInputEntityDataStream Input stream of {@link AssetInputEntityData}
* @param nodes A collection of known nodes
* @return A stream on option to matching {@link ConnectorInputEntityData}
*/
private Stream<Optional<ConnectorInputEntityData>> buildUntypedConnectorInputEntityData(
Stream<AssetInputEntityData> assetInputEntityDataStream, Collection<NodeInput> nodes) {
return assetInputEntityDataStream
.parallel()
.map(
assetInputEntityData ->
buildUntypedConnectorInputEntityData(assetInputEntityData, nodes));
}
/**
* Converts a single given {@link AssetInputEntityData} in connection with a collection of known
* {@link NodeInput}s to {@link ConnectorInputEntityData}. If this is not possible, an empty
* option is given back.
*
* @param assetInputEntityData Input entity data to convert
* @param nodes A collection of known nodes
* @return An option to matching {@link ConnectorInputEntityData}
*/
private Optional<ConnectorInputEntityData> buildUntypedConnectorInputEntityData(
AssetInputEntityData assetInputEntityData, Collection<NodeInput> nodes) {
// get the raw data
Map<String, String> fieldsToAttributes = assetInputEntityData.getFieldsToValues();
// get the two connector nodes
String nodeAUuid = fieldsToAttributes.get(NODE_A);
String nodeBUuid = fieldsToAttributes.get(NODE_B);
Optional<NodeInput> nodeA = findFirstEntityByUuid(nodeAUuid, nodes);
Optional<NodeInput> nodeB = findFirstEntityByUuid(nodeBUuid, nodes);
// if nodeA or nodeB are not present we return an empty element and log a
// warning
if (nodeA.isEmpty() || nodeB.isEmpty()) {
String debugString =
Stream.of(
new AbstractMap.SimpleEntry<>(nodeA, NODE_A + ": " + nodeAUuid),
new AbstractMap.SimpleEntry<>(nodeB, NODE_B + ": " + nodeBUuid))
.filter(entry -> entry.getKey().isEmpty())
.map(AbstractMap.SimpleEntry::getValue)
.collect(Collectors.joining("\n"));
logSkippingWarning(
assetInputEntityData.getTargetClass().getSimpleName(),
fieldsToAttributes.get("uuid"),
fieldsToAttributes.get("id"),
debugString);
return Optional.empty();
}
// remove fields that are passed as objects to constructor
fieldsToAttributes.keySet().removeAll(new HashSet<>(Arrays.asList(NODE_A, NODE_B)));
return Optional.of(
new ConnectorInputEntityData(
new FactoryData.MapWithRowIndex(assetInputEntityData.getRowIndex(), fieldsToAttributes),
assetInputEntityData.getTargetClass(),
assetInputEntityData.getOperatorInput(),
nodeA.get(),
nodeB.get()));
}
/**
* Enriches the given untyped entity data with the equivalent asset type. If this is not possible,
* an empty Optional is returned
*
* @param noTypeConnectorEntityDataStream Stream of untyped entity data
* @param availableTypes Yet available asset types
* @param <T> Type of the asset type
* @return Stream of option to enhanced data
*/
private <T extends AssetTypeInput>
Stream<Optional<TypedConnectorInputEntityData<T>>> buildTypedConnectorEntityData(
Stream<Optional<ConnectorInputEntityData>> noTypeConnectorEntityDataStream,
Collection<T> availableTypes) {
return noTypeConnectorEntityDataStream
.parallel()
.map(
noTypeEntityDataOpt ->
noTypeEntityDataOpt.flatMap(
noTypeEntityData -> findAndAddType(noTypeEntityData, availableTypes)));
}
/**
* Finds the required asset type and if present, adds it to the untyped entity data
*
* @param untypedEntityData Untyped entity data to enrich
* @param availableTypes Yet available asset types
* @param <T> Type of the asset type
* @return Option to enhanced data
*/
private <T extends AssetTypeInput> Optional<TypedConnectorInputEntityData<T>> findAndAddType(
ConnectorInputEntityData untypedEntityData, Collection<T> availableTypes) {
Optional<T> assetTypeOption =
getAssetType(
availableTypes,
untypedEntityData.getFieldsToValues(),
untypedEntityData.getClass().getSimpleName());
return assetTypeOption.map(assetType -> addTypeToEntityData(untypedEntityData, assetType));
}
/**
* Enriches the given, untyped entity data with the provided asset type
*
* @param untypedEntityData Untyped entity data to enrich
* @param assetType Asset type to add
* @param <T> Type of the asset type
* @return The enriched entity data
*/
private <T extends AssetTypeInput> TypedConnectorInputEntityData<T> addTypeToEntityData(
ConnectorInputEntityData untypedEntityData, T assetType) {
Map<String, String> fieldsToAttributes = untypedEntityData.getFieldsToValues();
// remove fields that are passed as objects to constructor
fieldsToAttributes.keySet().remove(TYPE);
// build result object
return new TypedConnectorInputEntityData<>(
new FactoryData.MapWithRowIndex(untypedEntityData.getRowIndex(), fieldsToAttributes),
untypedEntityData.getTargetClass(),
untypedEntityData.getOperatorInput(),
untypedEntityData.getNodeA(),
untypedEntityData.getNodeB(),
assetType);
}
/**
* Enriches the Stream of options on {@link Transformer3WInputEntityData} with the information of
* the internal node
*
* @param typedConnectorEntityDataStream Stream of already typed input entity data
* @param nodes Yet available nodes
* @return A stream of options on enriched data
*/
private Stream<Optional<Transformer3WInputEntityData>> buildTransformer3WEntityData(
Stream<Optional<TypedConnectorInputEntityData<Transformer3WTypeInput>>>
typedConnectorEntityDataStream,
Collection<NodeInput> nodes) {
return typedConnectorEntityDataStream
.parallel()
.map(
typedEntityDataOpt ->
typedEntityDataOpt.flatMap(typeEntityData -> addThirdNode(typeEntityData, nodes)));
}
/**
* Enriches the third node to the already typed entity data of a three winding transformer. If no
* matching node can be found, return an empty Optional.
*
* @param typeEntityData Already typed entity data
* @param nodes Yet available nodes
* @return An option to the enriched data
*/
private Optional<Transformer3WInputEntityData> addThirdNode(
TypedConnectorInputEntityData<Transformer3WTypeInput> typeEntityData,
Collection<NodeInput> nodes) {
// get the raw data
Map<String, String> fieldsToAttributes = typeEntityData.getFieldsToValues();
// get nodeC of the transformer
String nodeCUuid = fieldsToAttributes.get("nodeC");
Optional<NodeInput> nodeC = findFirstEntityByUuid(nodeCUuid, nodes);
// if nodeC is not present we return an empty element and
// log a warning
if (nodeC.isEmpty()) {
logSkippingWarning(
typeEntityData.getTargetClass().getSimpleName(),
fieldsToAttributes.get("uuid"),
fieldsToAttributes.get("id"),
"nodeC: " + nodeCUuid);
return Optional.empty();
}
// remove fields that are passed as objects to constructor
fieldsToAttributes.keySet().remove("nodeC");
return Optional.of(
new Transformer3WInputEntityData(
new FactoryData.MapWithRowIndex(typeEntityData.getRowIndex(), fieldsToAttributes),
typeEntityData.getTargetClass(),
typeEntityData.getOperatorInput(),
typeEntityData.getNodeA(),
typeEntityData.getNodeB(),
nodeC.get(),
typeEntityData.getType()));
}
}