-
Notifications
You must be signed in to change notification settings - Fork 19
/
OnlineDbMVStore.java
2003 lines (1887 loc) · 117 KB
/
OnlineDbMVStore.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**
* Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium)
* Copyright (c) 2016-2017, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package eu.itesla_project.online.db;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.joda.JodaModule;
import com.google.common.base.Splitter;
import eu.itesla_project.cases.CaseType;
import eu.itesla_project.computation.local.LocalComputationManager;
import eu.itesla_project.iidm.datasource.DataSource;
import eu.itesla_project.iidm.datasource.FileDataSource;
import eu.itesla_project.iidm.export.Exporters;
import eu.itesla_project.iidm.import_.ImportConfig;
import eu.itesla_project.iidm.import_.Importers;
import eu.itesla_project.iidm.network.Country;
import eu.itesla_project.iidm.network.Network;
import eu.itesla_project.modules.contingencies.ActionParameters;
import eu.itesla_project.modules.histo.HistoDbAttributeId;
import eu.itesla_project.modules.histo.IIDM2DB;
import eu.itesla_project.modules.online.*;
import eu.itesla_project.modules.optimizer.CCOFinalStatus;
import eu.itesla_project.online.OnlineUtils;
import eu.itesla_project.online.db.debug.NetworkData;
import eu.itesla_project.online.db.debug.NetworkDataExporter;
import eu.itesla_project.online.db.debug.NetworkDataExtractor;
import eu.itesla_project.security.LimitViolation;
import eu.itesla_project.simulation.securityindexes.SecurityIndexType;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.h2.mvstore.MVMap;
import org.h2.mvstore.MVMapConcurrent;
import org.h2.mvstore.MVStore;
import org.joda.time.DateTime;
import org.joda.time.Interval;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.supercsv.io.CsvListWriter;
import org.supercsv.prefs.CsvPreference;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* @author Quinary <itesla@quinary.com>
*/
public class OnlineDbMVStore implements OnlineDb {
private static final String STORED_WORKFLOW_PREFIX = "wf-";
private static final String STORED_METRICS_STEPS_MAP_NAME = "storedSteps";
private static final String STORED_METRICS_STATES_MAP_SUFFIX = "_states";
private static final String STORED_METRICS_PARAMS_MAP_SUFFIX = "_params";
private static final String STORED_RESULTS_MAP_NAME = "wfResults";
private static final String STORED_RESULTS_ACTIONS_MAP_SUFFIX = "_actions";
private static final String STORED_RESULTS_ACTIONINFO_MAP_SUFFIX = "_actionplans"; // i do not change this, for backward compatibility
private static final String STORED_RESULTS_ACTIONINFO_ACTIONSFOUND_KEY_SUFFIX = "_actionsfound";
private static final String STORED_RESULTS_ACTIONINFO_STATUS_KEY_SUFFIX = "_status";
private static final String STORED_RESULTS_ACTIONINFO_CAUSE_KEY_SUFFIX = "_cause";
private static final String STORED_RESULTS_ACTIONINFO_ACTIONPLAN_KEY_SUFFIX = "_actionplan";
private static final String STORED_RESULTS_ACTIONS_EQUIPMENTS_MAP_SUFFIX = "_actionequipments";
private static final String STORED_RESULTS_ACTIONS_PARAMETERS_MAP_SUFFIX = "_actionparameters";
private static final String STORED_RESULTS_INDEXES_MAP_SUFFIX = "_indexes";
private static final String STORED_RESULTS_TIMEHORIZON_KEY = "time_orizon";
private static final String STORED_RESULTS_CONTINGENCIES_WITH_ACTIONS_KEY = "contingiencies_with_actions";
private static final String STORED_RESULTS_UNSAFE_CONTINGENCIES_KEY = "unsafe_contingiencies";
private static final String STORED_RULES_RESULTS_MAP_NAME = "wfRulesResults";
private static final String STORED_RULES_RESULTS_STATE_RESULTS_MAP_SUFFIX = "_rulesresults";
private static final String STORED_RULES_RESULTS_STATE_STATUS_MAP_SUFFIX = "_rulesstatus";
private static final String STORED_RULES_RESULTS_STATE_RULES_AVAILABLE_MAP_SUFFIX = "_rulesavailable";
private static final String STORED_RULES_RESULTS_STATE_INVALID_RULES_MAP_SUFFIX = "_rulesinvalid";
private static final String STORED_RULES_RESULTS_CONTINGENCIES_WITH_RULES_KEY = "contingiencies_with_rules";
private static final String STORED_WCA_RESULTS_MAP_NAME = "wfWcaResults";
private static final String STORED_WCA_RESULTS_CLUSTERS_MAP_NAME = "contingencies_wcaclusters";
private static final String STORED_WCA_RESULTS_CAUSES_MAP_NAME = "contingencies_wcacause";
private static final String STORED_PARAMETERS_MAP_NAME = "wfParameters";
private static final String STORED_PARAMETERS_BASECASE_KEY = "basecase";
private static final String STORED_PARAMETERS_STATE_NUMBER_KEY = "state_number";
private static final String STORED_PARAMETERS_HISTO_INTERVAL_KEY = "histo_interval";
private static final String STORED_PARAMETERS_OFFLINE_WF_ID_KEY = "offline_wf";
private static final String STORED_PARAMETERS_FEA_ID_KEY = "fe_analysis";
private static final String STORED_PARAMETERS_RULES_PURITY_KEY = "rules_purity";
private static final String STORED_PARAMETERS_STORE_STATES_KEY = "store_states";
private static final String STORED_PARAMETERS_ANALYSE_BASECASE_KEY = "analyse_basecase";
private static final String STORED_PARAMETERS_VALIDATION_KEY = "validation";
private static final String STORED_PARAMETERS_SECURITY_INDEXES_KEY = "security_indexes";
private static final String STORED_PARAMETERS_CASE_TYPE_KEY = "case_type";
private static final String STORED_PARAMETERS_COUNTRIES_KEY = "countries";
private static final String STORED_PARAMETERS_MERGE_OPTIMIZED_KEY = "merge_optimized";
private static final String STORED_PARAMETERS_LIMIT_REDUCTION_KEY = "limit_reduction";
private static final String STORED_PARAMETERS_HANDLE_VIOLATIONS_KEY = "handle_violations";
private static final String STORED_PARAMETERS_CONSTRAINT_MARGIN_KEY = "constraint_margin";
private static final String STORED_PARAMETERS_CASE_FILE_KEY = "case_file";
private static final String STORED_STATES_PROCESSING_STATUS_MAP_NAME = "statesProcessingStatus";
private static final String STORED_STATES_LIST_KEY = "states";
private static final String STORED_STATES_STATE_DETAILS_KEY = "stateStatusDetails";
private static final String STORED_STATE_PROCESSING_STATUS_MAP_SUFFIX = "_processingstatus";
private static final String STORED_WORKFLOW_STATES_FOLDER_PREFIX = "states-wf-";
private static final String STORED_STATE_PREFIX = "state-";
private static final String STORED_STATE_POST_PREFIX = "post-state-";
private static final String STORED_STATE_CONT_PREFIX = "-cont-";
private static final String STORED_VIOLATIONS_STEPS_MAP_NAME = "storedViolationsSteps";
private static final String STORED_VIOLATIONS_STATES_MAP_SUFFIX = "_violationsstates";
private static final String STORED_VIOLATIONS_STATES_MAP_NAME = "storedViolationsStates";
private static final String STORED_VIOLATIONS_STEPS_MAP_SUFFIX = "_violationssteps";
private static final String STORED_VIOLATIONS_MAP_PREFIX = "violations_";
private static final String STORED_PC_VIOLATIONS_CONTINGENCIES_MAP_NAME = "storedPCViolationsContingencies";
private static final String STORED_PC_VIOLATIONS_STATES_MAP_SUFFIX = "_pcviolationsstates";
private static final String STORED_PC_VIOLATIONS_STATES_MAP_NAME = "storedPCViolationsStates";
private static final String STORED_PC_VIOLATIONS_CONTINGENCIES_MAP_SUFFIX = "_pcviolationscontigencies";
private static final String STORED_PC_VIOLATIONS_MAP_PREFIX = "pcviolations_";
private static final String STORED_PC_LOADFLOW_CONTINGENCIES_MAP_NAME = "storedPCLoadflowContingencies";
private static final String STORED_PC_LOADFLOW_STATES_MAP_SUFFIX = "_pcloadflowstates";
private static final String STORED_PC_LOADFLOW_STATES_MAP_NAME = "storedPCLoadflowStates";
private static final String STORED_PC_LOADFLOW_CONTINGENCIES_MAP_SUFFIX = "_pcloadflowcontigencies";
private static final String STORED_WCA_RULES_RESULTS_MAP_NAME = "wfWcaRulesResults";
private static final String STORED_WCA_RULES_RESULTS_STATE_RESULTS_MAP_SUFFIX = "_wcarulesresults";
private static final String STORED_WCA_RULES_RESULTS_STATE_STATUS_MAP_SUFFIX = "_wcarulesstatus";
private static final String STORED_WCA_RULES_RESULTS_STATE_RULES_AVAILABLE_MAP_SUFFIX = "_wcarulesavailable";
private static final String STORED_WCA_RULES_RESULTS_STATE_INVALID_RULES_MAP_SUFFIX = "_wcarulesinvalid";
private static final String SERIALIZED_STATES_FILENAME = "network-states.csv";
private final String[] XIIDMEXTENSIONS = {".xiidm", ".iidm", ".xml"};
private static final Logger LOGGER = LoggerFactory.getLogger(OnlineDbMVStore.class);
private static final String STORED_PROCESS_PREFIX = "prc_";
private OnlineDbMVStoreConfig config = null;
HashMap<String, MVStore> storedWFMetrics = new HashMap<String, MVStore>();
ConcurrentHashMap<String, ConcurrentHashMap<Integer, Map<HistoDbAttributeId, Object>>> workflowsStates = new ConcurrentHashMap<String, ConcurrentHashMap<Integer, Map<HistoDbAttributeId, Object>>>();
MVMapConcurrent.Builder<String, String> mapBuilder;
public OnlineDbMVStore(OnlineDbMVStoreConfig config) {
this.config = config;
LOGGER.info(config.toString());
Path storageFolder = config.getOnlineDbDir();
if (!Files.exists(storageFolder)) {
try {
Files.createDirectories(storageFolder);
} catch (IOException e) {
String errorMessage = "online db folder " + storageFolder + " does not exist and cannot be created: " + e.getMessage();
LOGGER.error(errorMessage);
throw new RuntimeException(errorMessage, e);
}
}
mapBuilder = new MVMapConcurrent.Builder<String, String>();
}
public OnlineDbMVStore() {
this(OnlineDbMVStoreConfig.load());
}
private synchronized void closeStores() {
ArrayList<String> workflowIds = new ArrayList<String>();
for (String storedWorkflowId : storedWFMetrics.keySet()) {
MVStore wfMVStore = storedWFMetrics.get(storedWorkflowId);
wfMVStore.close();
workflowIds.add(storedWorkflowId);
}
for (String workflowId : workflowIds) {
storedWFMetrics.remove(workflowId);
}
}
private synchronized MVStore getStore(String workflowId) {
MVStore wfMVStore;
if (storedWFMetrics.containsKey(workflowId))
wfMVStore = storedWFMetrics.get(workflowId);
else {
LOGGER.debug("Opening file for workflow {}", workflowId);
wfMVStore = MVStore.open(config.getOnlineDbDir().toString() + File.separator + STORED_WORKFLOW_PREFIX + workflowId);
storedWFMetrics.put(workflowId, wfMVStore);
}
return wfMVStore;
}
@Override
public void storeMetrics(String workflowId, OnlineStep step, Map<String, String> metrics) {
LOGGER.info("Storing metrics for wf {} and step {}", workflowId, step.name());
storeMetrics(workflowId, step.name() + "__", metrics);
LOGGER.info("Storing metadata for wf {} and step {}", workflowId, step.name());
storeStepMetadata(workflowId, "_", step, metrics);
}
@Override
public void storeMetrics(String workflowId, Integer stateId, OnlineStep step, Map<String, String> metrics) {
String stateIdStr = String.valueOf(stateId);
LOGGER.info("Storing metrics for wf {}, step {} and state {}", workflowId, step.name(), stateIdStr);
storeMetrics(workflowId, step.name() + "_" + stateIdStr, metrics);
LOGGER.info("Storing metadata for wf {}, step {} and state {}", workflowId, step.name(), stateIdStr);
storeStepMetadata(workflowId, stateIdStr, step, metrics);
}
private void storeMetrics(String workflowId, String mapName, Map<String, String> metrics) {
try {
MVStore wfMVStore = getStore(workflowId);
Map<String, String> metricsMap = wfMVStore.openMap(mapName, mapBuilder);
for (String parameter : metrics.keySet()) {
metricsMap.put(parameter, metrics.get(parameter));
}
wfMVStore.commit();
} catch (Throwable e) {
String errorMessage = "Error storing metrics for wf " + workflowId + " in map " + mapName + ": " + e.getMessage();
LOGGER.error(errorMessage);
throw new RuntimeException(errorMessage, e);
}
}
private void storeStepMetadata(String workflowId, String stateId, OnlineStep step, Map<String, String> metrics) {
try {
MVStore wfMVStore = getStore(workflowId);
// save info about stored wf steps
MVMap<String, String> storedStepsMap = wfMVStore.openMap(STORED_METRICS_STEPS_MAP_NAME, mapBuilder);
storedStepsMap.putIfAbsent(step.name(), "1");
// save info about stored states per step
MVMap<String, String> stepStatesMap = wfMVStore.openMap(step.name() + STORED_METRICS_STATES_MAP_SUFFIX, mapBuilder);
stepStatesMap.putIfAbsent(stateId, "");
// save info about stored params per step
MVMap<String, String> stepParamsMap = wfMVStore.openMap(step.name() + STORED_METRICS_PARAMS_MAP_SUFFIX, mapBuilder);
for (String parameter : metrics.keySet()) {
stepParamsMap.putIfAbsent(parameter, "");
}
wfMVStore.commit();
} catch (Throwable e) {
String errorMessage = "Error storing metadata for wf " + workflowId + ", step " + step.name() + ", state " + stateId + ": " + e.getMessage();
LOGGER.error(errorMessage);
throw new RuntimeException(errorMessage, e);
}
}
@Override
public Map<String, String> getMetrics(String workflowId, OnlineStep step) {
LOGGER.info("Getting metrics from wf {} and step {}", workflowId, step.name());
return getMetrics(workflowId, step.name() + "__");
}
@Override
public Map<String, String> getMetrics(String workflowId, Integer stateId, OnlineStep step) {
String stateIdStr = String.valueOf(stateId);
LOGGER.info("Getting metrics from wf {}, step {} and state {}", workflowId, step.name(), stateIdStr);
return getMetrics(workflowId, step.name() + "_" + stateIdStr);
}
private Map<String, String> getMetrics(String workflowId, String mapName) {
if (isWorkflowStored(workflowId)) {
TreeMap<String, String> metrics = new TreeMap<>();
MVStore wfMVStore = getStore(workflowId);
if (wfMVStore.getMapNames().contains(mapName)) {
Map<String, String> storedMap = wfMVStore.openMap(mapName, mapBuilder);
for (String parameter : storedMap.keySet()) {
metrics.put(parameter, storedMap.get(parameter));
}
}
return metrics;
} else {
LOGGER.warn("No data about wf {}", workflowId);
return null;
}
}
@Override
public List<String[]> getAllMetrics(String workflowId, OnlineStep step) {
LOGGER.info("Preparing CSV data for wf {} and step {}", workflowId, step.name());
List<String[]> retTable = new ArrayList<>();
if (isWorkflowStored(workflowId)) {
try {
MVStore wfMVStore = getStore(workflowId);
// check if there are stored metrics
Map<String, String> storedStepsMap = wfMVStore.openMap(STORED_METRICS_STEPS_MAP_NAME, mapBuilder);
if (storedStepsMap.containsKey(step.name())) {
MVMap<String, String> stepParamsMap = wfMVStore.openMap(step.name() + STORED_METRICS_PARAMS_MAP_SUFFIX, mapBuilder);
MVMap<String, String> stepStatesMap = wfMVStore.openMap(step.name() + STORED_METRICS_STATES_MAP_SUFFIX, mapBuilder);
// gets headers
String[] headers = new String[stepParamsMap.keySet().size() + 1];
headers[0] = "state";
HashMap<String, Integer> paramsIndexes = new HashMap<>();
int i = 1;
for (String parameter : stepParamsMap.keySet()) {
headers[i] = parameter;
paramsIndexes.put(parameter, i);
i++;
}
retTable.add(headers);
// gets step general metrics, if stored
if (stepStatesMap.containsKey("_")) {
retTable.add(getStoredMapValues(wfMVStore, "_", step, stepParamsMap.keySet().size(), paramsIndexes));
}
// gets step metrics for each state, if stored
stepStatesMap.keySet().stream()
.filter(x -> (!"_".equals(x)))
.sorted(Comparator.comparing(Integer::valueOf))
.forEach(stateId -> {
retTable.add(getStoredMapValues(wfMVStore, stateId, step, stepParamsMap.keySet().size(), paramsIndexes));
});
}
} catch (Exception e) {
String errorMessage = "error getting cvs data for step " + step.name() + " and wf id " + workflowId;
LOGGER.error(errorMessage);
throw new RuntimeException(errorMessage, e);
}
} else {
LOGGER.warn("No data about wf {}", workflowId);
}
return retTable;
}
private String[] getStoredMapValues(MVStore wfMVStore, String stateId, OnlineStep step, int paramsN, HashMap<String, Integer> paramsIndexes) {
String[] values = new String[paramsN + 1];
values[0] = stateId;
Map<String, String> storedMap = wfMVStore.openMap(step.name() + "_" + stateId, mapBuilder);
for (String parameter : storedMap.keySet()) {
int index = paramsIndexes.get(parameter);
values[index] = storedMap.get(parameter);
}
return values;
}
private DateTime getWorkflowDate(String workflowId) {
DateTime workflowDate = null;
if (workflowId.contains("_")) {
String workflowStringDate = workflowId.substring(workflowId.lastIndexOf("_") + 1);
workflowDate = DateTimeFormat.forPattern("yyyyMMddHHmmssSSS").parseDateTime(workflowStringDate);
}
return workflowDate;
}
@Override
public List<OnlineWorkflowDetails> listWorkflows() {
LOGGER.info("Getting list of stored workflows");
List<OnlineWorkflowDetails> workflowIds = new ArrayList<OnlineWorkflowDetails>();
File[] files = config.getOnlineDbDir().toFile().listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.toLowerCase().startsWith(STORED_WORKFLOW_PREFIX);
}
});
for (File file : files) {
if (file.isFile()) {
String workflowId = file.getName().substring(STORED_WORKFLOW_PREFIX.length());
OnlineWorkflowDetails workflowDetails = new OnlineWorkflowDetails(workflowId);
workflowDetails.setWorkflowDate(getWorkflowDate(workflowId));
workflowIds.add(workflowDetails);
}
}
Collections.sort(workflowIds, new Comparator<OnlineWorkflowDetails>() {
@Override
public int compare(OnlineWorkflowDetails wfDetails1, OnlineWorkflowDetails wfDetails2) {
return wfDetails1.getWorkflowDate().compareTo(wfDetails2.getWorkflowDate());
}
});
LOGGER.info("Found {} workflow(s)", workflowIds.size());
return workflowIds;
}
@Override
public List<OnlineWorkflowDetails> listWorkflows(DateTime basecaseDate) {
LOGGER.info("Getting list of stored workflows run on basecase {}", basecaseDate);
String wfNamePrefix = DateTimeFormat.forPattern("yyyyMMdd_HHmm_").print(basecaseDate);
List<OnlineWorkflowDetails> workflowIds = new ArrayList<OnlineWorkflowDetails>();
File[] files = config.getOnlineDbDir().toFile().listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.toLowerCase().startsWith(STORED_WORKFLOW_PREFIX + wfNamePrefix);
}
});
for (File file : files) {
if (file.isFile()) {
String workflowId = file.getName().substring(STORED_WORKFLOW_PREFIX.length());
OnlineWorkflowDetails workflowDetails = new OnlineWorkflowDetails(workflowId);
workflowDetails.setWorkflowDate(getWorkflowDate(workflowId));
workflowIds.add(workflowDetails);
}
}
Collections.sort(workflowIds, new Comparator<OnlineWorkflowDetails>() {
@Override
public int compare(OnlineWorkflowDetails wfDetails1, OnlineWorkflowDetails wfDetails2) {
return wfDetails1.getWorkflowDate().compareTo(wfDetails2.getWorkflowDate());
}
});
LOGGER.info("Found {} workflow(s)", workflowIds.size());
return workflowIds;
}
@Override
public List<OnlineWorkflowDetails> listWorkflows(Interval basecaseInterval) {
LOGGER.info("Getting list of stored workflows run on basecases within the interval {}", basecaseInterval);
String dateFormatPattern = "yyyyMMdd_HHmm";
DateTimeFormatter formatter = DateTimeFormat.forPattern(dateFormatPattern);
List<OnlineWorkflowDetails> workflowIds = new ArrayList<OnlineWorkflowDetails>();
File[] files = config.getOnlineDbDir().toFile().listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.toLowerCase().startsWith(STORED_WORKFLOW_PREFIX);
}
});
for (File file : files) {
if (file.isFile()) {
String workflowId = file.getName().substring(STORED_WORKFLOW_PREFIX.length());
if (workflowId.length() > dateFormatPattern.length() && workflowId.substring(dateFormatPattern.length(), dateFormatPattern.length() + 1).equals("_")) {
String basecaseName = workflowId.substring(0, dateFormatPattern.length() - 1);
DateTime basecaseDate = DateTime.parse(basecaseName, formatter);
if (basecaseInterval.contains(basecaseDate.getMillis())) {
OnlineWorkflowDetails workflowDetails = new OnlineWorkflowDetails(workflowId);
workflowDetails.setWorkflowDate(getWorkflowDate(workflowId));
workflowIds.add(workflowDetails);
}
}
}
}
Collections.sort(workflowIds, new Comparator<OnlineWorkflowDetails>() {
@Override
public int compare(OnlineWorkflowDetails wfDetails1, OnlineWorkflowDetails wfDetails2) {
return wfDetails1.getWorkflowDate().compareTo(wfDetails2.getWorkflowDate());
}
});
LOGGER.info("Found {} workflow(s)", workflowIds.size());
return workflowIds;
}
@Override
public OnlineWorkflowDetails getWorkflowDetails(String workflowId) {
LOGGER.info("Getting details of stored workflow {}", workflowId);
OnlineWorkflowDetails workflowDetails = null;
File[] files = config.getOnlineDbDir().toFile().listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.toLowerCase().equals(STORED_WORKFLOW_PREFIX + workflowId);
}
});
if (files != null && files.length == 1) {
workflowDetails = new OnlineWorkflowDetails(workflowId);
workflowDetails.setWorkflowDate(new DateTime(files[0].lastModified()));
}
return workflowDetails;
}
@Override
public void storeResults(String workflowId, OnlineWorkflowResults results) {
Objects.requireNonNull(workflowId, "workflow id is null");
Objects.requireNonNull(results, "online workflow results is null");
LOGGER.info("Storing results for workflow {}", workflowId);
MVStore wfMVStore = getStore(workflowId);
// check if the results for this wf have already been stored
if (wfMVStore.hasMap(STORED_RESULTS_MAP_NAME))
removeWfResults(workflowId, wfMVStore);
MVMap<String, String> storedResultsMap = wfMVStore.openMap(STORED_RESULTS_MAP_NAME, mapBuilder);
// store time horizon
storedResultsMap.put(STORED_RESULTS_TIMEHORIZON_KEY, results.getTimeHorizon().getName());
// store contingencies with actions
storedResultsMap.put(STORED_RESULTS_CONTINGENCIES_WITH_ACTIONS_KEY, OnlineDbMVStoreUtils.contingenciesIdsToJson(results.getContingenciesWithActions()));
// store contingencies with actions
storedResultsMap.put(STORED_RESULTS_UNSAFE_CONTINGENCIES_KEY, OnlineDbMVStoreUtils.contingenciesIdsToJson(results.getUnsafeContingencies()));
// store actions for contingencies
for (String contingencyId : results.getContingenciesWithActions()) {
MVMap<String, String> storedActionsInfosMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_ACTIONINFO_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedActionsMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_ACTIONS_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedActionsEquipmentsMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_ACTIONS_EQUIPMENTS_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedActionsParametersMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_ACTIONS_PARAMETERS_MAP_SUFFIX, mapBuilder);
for (Integer stateId : results.getUnsafeStatesWithActions(contingencyId).keySet()) {
storedActionsInfosMap.put(stateId.toString() + STORED_RESULTS_ACTIONINFO_ACTIONSFOUND_KEY_SUFFIX,
Boolean.toString(results.getUnsafeStatesWithActions(contingencyId).get(stateId)));
storedActionsInfosMap.put(stateId.toString() + STORED_RESULTS_ACTIONINFO_STATUS_KEY_SUFFIX, results.getStateStatus(contingencyId, stateId).name());
if (results.getCause(contingencyId, stateId) != null)
storedActionsInfosMap.put(stateId.toString() + STORED_RESULTS_ACTIONINFO_CAUSE_KEY_SUFFIX, results.getCause(contingencyId, stateId));
if (results.getActionPlan(contingencyId, stateId) != null)
storedActionsInfosMap.put(stateId.toString() + STORED_RESULTS_ACTIONINFO_ACTIONPLAN_KEY_SUFFIX, results.getActionPlan(contingencyId, stateId));
List<String> actionsIds = results.getActionsIds(contingencyId, stateId);
if (actionsIds != null) {
for (String actionId : actionsIds) {
List<String> equipmentsIds = results.getEquipmentsIds(contingencyId, stateId, actionId);
storedActionsEquipmentsMap.put(stateId + "_" + actionId, OnlineDbMVStoreUtils.actionsIdsToJson(equipmentsIds));
for (String equipmentId : equipmentsIds) {
ActionParameters actionParameters = results.getParameters(contingencyId, stateId, actionId, equipmentId);
if (actionParameters != null)
storedActionsParametersMap.put(stateId + "_" + actionId + "_" + equipmentId, OnlineDbMVStoreUtils.actionParametersToJson(actionParameters));
}
}
} else {
actionsIds = new ArrayList<String>(); // I need anyway an empty list, for the getResults to work
}
storedActionsMap.put(stateId.toString(), OnlineDbMVStoreUtils.actionsIdsToJson(actionsIds));
}
}
// store indexes for contingencies
for (String contingencyId : results.getUnsafeContingencies()) {
MVMap<String, String> storedIndexesMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_INDEXES_MAP_SUFFIX, mapBuilder);
for (Integer stateId : results.getUnstableStates(contingencyId)) {
Map<String, Boolean> indexesData = results.getIndexesData(contingencyId, stateId);
storedIndexesMap.put(stateId.toString(), OnlineDbMVStoreUtils.indexesDataToJson(indexesData));
}
}
wfMVStore.commit();
}
private void removeWfResults(String workflowId, MVStore wfMVStore) {
LOGGER.debug("Removing existing wf results for workflow {}", workflowId);
MVMap<String, String> storedResultsMap = wfMVStore.openMap(STORED_RESULTS_MAP_NAME, mapBuilder);
// remove info about contingencies with action
Collection<String> contingenciesWithAction = OnlineDbMVStoreUtils.jsonToContingenciesIds(
storedResultsMap.get(STORED_RESULTS_CONTINGENCIES_WITH_ACTIONS_KEY));
for (String contingencyId : contingenciesWithAction) {
MVMap<String, String> storedActionsMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_ACTIONS_MAP_SUFFIX, mapBuilder);
wfMVStore.removeMap(storedActionsMap);
}
// remove info about unsafe contingencies
Collection<String> unsafeContingencies = OnlineDbMVStoreUtils.jsonToContingenciesIds(
storedResultsMap.get(STORED_RESULTS_UNSAFE_CONTINGENCIES_KEY));
for (String contingencyId : unsafeContingencies) {
MVMap<String, String> storedIndexesMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_INDEXES_MAP_SUFFIX, mapBuilder);
wfMVStore.removeMap(storedIndexesMap);
}
// remove info about stored wf results
wfMVStore.removeMap(storedResultsMap);
// commit removal
wfMVStore.commit();
}
@Override
public OnlineWorkflowResults getResults(String workflowId) {
Objects.requireNonNull(workflowId, "workflow id is null");
LOGGER.info("Getting results of wf {}", workflowId);
if (isWorkflowStored(workflowId)) {
MVStore wfMVStore = getStore(workflowId);
if (wfMVStore.hasMap(STORED_RESULTS_MAP_NAME)) {
MVMap<String, String> storedResultsMap = wfMVStore.openMap(STORED_RESULTS_MAP_NAME, mapBuilder);
// create workflow results
OnlineWorkflowResultsImpl wfResults = new OnlineWorkflowResultsImpl(
workflowId,
TimeHorizon.valueOf(storedResultsMap.get(STORED_RESULTS_TIMEHORIZON_KEY)));
// add contingencies with actiions
Collection<String> contingenciesWithAction = OnlineDbMVStoreUtils.jsonToContingenciesIds(
storedResultsMap.get(STORED_RESULTS_CONTINGENCIES_WITH_ACTIONS_KEY));
for (String contingencyId : contingenciesWithAction) {
MVMap<String, String> storedActionsMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_ACTIONS_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedActionsInfosMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_ACTIONINFO_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedActionsParametersMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_ACTIONS_PARAMETERS_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedActionsEquipmentsMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_ACTIONS_EQUIPMENTS_MAP_SUFFIX, mapBuilder);
for (String stateId : storedActionsMap.keySet()) {
boolean actionsFound = true;
if (storedActionsInfosMap.containsKey(stateId + STORED_RESULTS_ACTIONINFO_ACTIONSFOUND_KEY_SUFFIX))
actionsFound = Boolean.parseBoolean(storedActionsInfosMap.get(stateId + STORED_RESULTS_ACTIONINFO_ACTIONSFOUND_KEY_SUFFIX));
CCOFinalStatus status = CCOFinalStatus.MANUAL_CORRECTIVE_ACTION_FOUND;
if (storedActionsInfosMap.containsKey(stateId + STORED_RESULTS_ACTIONINFO_STATUS_KEY_SUFFIX))
status = CCOFinalStatus.valueOf(storedActionsInfosMap.get(stateId + STORED_RESULTS_ACTIONINFO_STATUS_KEY_SUFFIX));
String cause = storedActionsInfosMap.get(stateId + STORED_RESULTS_ACTIONINFO_CAUSE_KEY_SUFFIX);
String actionPlan = storedActionsInfosMap.get(stateId + STORED_RESULTS_ACTIONINFO_ACTIONPLAN_KEY_SUFFIX);
Map<String, Map<String, ActionParameters>> actions = null;
if (storedActionsMap.containsKey(stateId)) {
List<String> actionsIds = OnlineDbMVStoreUtils.jsonToActionsIds(storedActionsMap.get(stateId));
actions = new HashMap<String, Map<String, ActionParameters>>();
for (String actionId : actionsIds) {
Map<String, ActionParameters> equipments = new HashMap<String, ActionParameters>();
List<String> equipmentsIds = OnlineDbMVStoreUtils.jsonToActionsIds(storedActionsEquipmentsMap.get(stateId + "_" + actionId));
if (equipmentsIds != null) {
for (String equipmentId : equipmentsIds) {
ActionParameters actionParameters = OnlineDbMVStoreUtils.jsonToActionParameters(storedActionsParametersMap.get(stateId + "_" + actionId + "_" + equipmentId));
equipments.put(equipmentId, actionParameters);
}
}
actions.put(actionId, equipments);
}
}
wfResults.addContingenciesWithActions(contingencyId, Integer.parseInt(stateId), actionsFound, status, cause, actionPlan, actions);
}
}
// add unsafe contingencies
Collection<String> unsafeContingencies = OnlineDbMVStoreUtils.jsonToContingenciesIds(
storedResultsMap.get(STORED_RESULTS_UNSAFE_CONTINGENCIES_KEY));
for (String contingencyId : unsafeContingencies) {
MVMap<String, String> storedIndexesMap = wfMVStore.openMap(contingencyId + STORED_RESULTS_INDEXES_MAP_SUFFIX, mapBuilder);
for (String stateId : storedIndexesMap.keySet()) {
Map<String, Boolean> indexesData = OnlineDbMVStoreUtils.jsonToIndexesData(storedIndexesMap.get(stateId));
wfResults.addUnsafeContingencies(contingencyId, Integer.parseInt(stateId), indexesData);
}
}
return wfResults;
} else {
LOGGER.warn("No results of wf {} stored in online db", workflowId);
return null;
}
} else {
LOGGER.warn("No data about wf {}", workflowId);
return null;
}
}
@Override
public void storeRulesResults(String workflowId, OnlineWorkflowRulesResults results) {
Objects.requireNonNull(workflowId, "workflow id is null");
Objects.requireNonNull(results, "online workflow rules results is null");
LOGGER.info("Storing results of rules for workflow {}", workflowId);
MVStore wfMVStore = getStore(workflowId);
// check if the results for this wf have already been stored
if (wfMVStore.hasMap(STORED_RULES_RESULTS_MAP_NAME))
removeWfRulesResults(workflowId, wfMVStore);
MVMap<String, String> storedRulesResultsMap = wfMVStore.openMap(STORED_RULES_RESULTS_MAP_NAME, mapBuilder);
// store time horizon
storedRulesResultsMap.put(STORED_RESULTS_TIMEHORIZON_KEY, results.getTimeHorizon().getName());
// store contingencies with security rules results
storedRulesResultsMap.put(STORED_RULES_RESULTS_CONTINGENCIES_WITH_RULES_KEY,
OnlineDbMVStoreUtils.contingenciesIdsToJson(results.getContingenciesWithSecurityRulesResults()));
// store rules results for contingencies
for (String contingencyId : results.getContingenciesWithSecurityRulesResults()) {
MVMap<String, String> storedStateStatusMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_STATUS_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedStateResultsMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_RESULTS_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedStateAvailableRulesMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_RULES_AVAILABLE_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedStateInvalidRulesMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_INVALID_RULES_MAP_SUFFIX, mapBuilder);
for (Integer stateId : results.getStatesWithSecurityRulesResults(contingencyId)) {
// store state status
StateStatus status = results.getStateStatus(contingencyId, stateId);
storedStateStatusMap.put(stateId.toString(), status.name());
// store state rules results
Map<String, Boolean> stateResults = results.getStateResults(contingencyId, stateId);
storedStateResultsMap.put(stateId.toString(), OnlineDbMVStoreUtils.indexesDataToJson(stateResults));
// store state rules available flag
boolean rulesAvalable = results.areValidRulesAvailable(contingencyId, stateId);
storedStateAvailableRulesMap.put(stateId.toString(), Boolean.toString(rulesAvalable));
// store state invalid rules
List<SecurityIndexType> invalidRules = results.getInvalidRules(contingencyId, stateId);
storedStateInvalidRulesMap.put(stateId.toString(), OnlineDbMVStoreUtils.indexesTypesToJson(new HashSet<SecurityIndexType>(invalidRules)));
}
}
wfMVStore.commit();
}
private void removeWfRulesResults(String workflowId, MVStore wfMVStore) {
LOGGER.debug("Removing existing rules results for workflow {}", workflowId);
MVMap<String, String> storedRulesResultsMap = wfMVStore.openMap(STORED_RULES_RESULTS_MAP_NAME, mapBuilder);
// remove rules results
Collection<String> rulesContingencies = OnlineDbMVStoreUtils.jsonToContingenciesIds(
storedRulesResultsMap.get(STORED_RULES_RESULTS_CONTINGENCIES_WITH_RULES_KEY));
for (String contingencyId : rulesContingencies) {
MVMap<String, String> storedStateStatusMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_STATUS_MAP_SUFFIX, mapBuilder);
wfMVStore.removeMap(storedStateStatusMap);
MVMap<String, String> storedStateResultsMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_RESULTS_MAP_SUFFIX, mapBuilder);
wfMVStore.removeMap(storedStateResultsMap);
MVMap<String, String> storedStateAvailableRulesMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_RULES_AVAILABLE_MAP_SUFFIX, mapBuilder);
wfMVStore.removeMap(storedStateAvailableRulesMap);
MVMap<String, String> storedStateInvalidRulesMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_INVALID_RULES_MAP_SUFFIX, mapBuilder);
wfMVStore.removeMap(storedStateInvalidRulesMap);
}
// remove info about stored rules results
wfMVStore.removeMap(storedRulesResultsMap);
// commit removal
wfMVStore.commit();
}
@Override
public OnlineWorkflowRulesResults getRulesResults(String workflowId) {
Objects.requireNonNull(workflowId, "workflow id is null");
LOGGER.info("Getting rules results of wf {}", workflowId);
if (isWorkflowStored(workflowId)) {
MVStore wfMVStore = getStore(workflowId);
if (wfMVStore.hasMap(STORED_RULES_RESULTS_MAP_NAME)) {
MVMap<String, String> storedRulesResultsMap = wfMVStore.openMap(STORED_RULES_RESULTS_MAP_NAME, mapBuilder);
// create workflow rules results
OnlineWorkflowRulesResultsImpl wfRulesResults = new OnlineWorkflowRulesResultsImpl(
workflowId,
TimeHorizon.valueOf(storedRulesResultsMap.get(STORED_RESULTS_TIMEHORIZON_KEY)));
// add contingencies with rules results
Collection<String> contingenciesWithRules = OnlineDbMVStoreUtils.jsonToContingenciesIds(
storedRulesResultsMap.get(STORED_RULES_RESULTS_CONTINGENCIES_WITH_RULES_KEY));
for (String contingencyId : contingenciesWithRules) {
MVMap<String, String> storedStateStatusMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_STATUS_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedStateResultsMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_RESULTS_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedStateAvailableRulesMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_RULES_AVAILABLE_MAP_SUFFIX, mapBuilder);
MVMap<String, String> storedStateInvalidRulesMap = wfMVStore.openMap(contingencyId + STORED_RULES_RESULTS_STATE_INVALID_RULES_MAP_SUFFIX, mapBuilder);
for (String stateId : storedStateStatusMap.keySet()) {
Map<String, Boolean> stateResults = OnlineDbMVStoreUtils.jsonToIndexesData(storedStateResultsMap.get(stateId));
StateStatus stateStatus = StateStatus.valueOf(storedStateStatusMap.get(stateId));
boolean rulesAvailable = true;
if (storedStateAvailableRulesMap.containsKey(stateId))
rulesAvailable = Boolean.parseBoolean(storedStateAvailableRulesMap.get(stateId));
List<SecurityIndexType> invalidRules = new ArrayList<SecurityIndexType>();
if (storedStateInvalidRulesMap.containsKey(stateId))
invalidRules.addAll(OnlineDbMVStoreUtils.jsonToIndexesTypes(storedStateInvalidRulesMap.get(stateId)));
wfRulesResults.addContingencyWithSecurityRulesResults(contingencyId, Integer.parseInt(stateId), stateStatus, stateResults,
rulesAvailable, invalidRules);
}
}
return wfRulesResults;
} else {
LOGGER.warn("No rules results of wf {} stored in online db", workflowId);
return null;
}
} else {
LOGGER.warn("No data about wf {}", workflowId);
return null;
}
}
@Override
public void storeWcaResults(String workflowId, OnlineWorkflowWcaResults results) {
Objects.requireNonNull(workflowId, "workflow id is null");
Objects.requireNonNull(results, "online workflow wca results is null");
LOGGER.info("Storing results of WCA for workflow {}", workflowId);
MVStore wfMVStore = getStore(workflowId);
// check if the results for this wf have already been stored
if (wfMVStore.hasMap(STORED_WCA_RESULTS_MAP_NAME))
removeWfWcaResults(workflowId, wfMVStore);
MVMap<String, String> storedWcaResultsMap = wfMVStore.openMap(STORED_WCA_RESULTS_MAP_NAME, mapBuilder);
// store time horizon
storedWcaResultsMap.put(STORED_RESULTS_TIMEHORIZON_KEY, results.getTimeHorizon().getName());
// store wca results for contingencies
MVMap<String, String> storedClustersMap = wfMVStore.openMap(STORED_WCA_RESULTS_CLUSTERS_MAP_NAME, mapBuilder);
MVMap<String, String> storedCausesMap = wfMVStore.openMap(STORED_WCA_RESULTS_CAUSES_MAP_NAME, mapBuilder);
for (String contingencyId : results.getContingencies()) {
storedClustersMap.put(contingencyId, Integer.toString(results.getClusterIndex(contingencyId)));
List<String> causes = results.getCauses(contingencyId);
if (causes != null && causes.size() > 0)
storedCausesMap.put(contingencyId, causes.get(0));
}
wfMVStore.commit();
}
private void removeWfWcaResults(String workflowId, MVStore wfMVStore) {
LOGGER.debug("Removing existing WCA results for workflow {}", workflowId);
MVMap<String, String> storedWcaResultsMap = wfMVStore.openMap(STORED_WCA_RESULTS_MAP_NAME, mapBuilder);
// remove wca results
MVMap<String, String> storedClustersMap = wfMVStore.openMap(STORED_WCA_RESULTS_CLUSTERS_MAP_NAME, mapBuilder);
wfMVStore.removeMap(storedClustersMap);
MVMap<String, String> storedCausesMap = wfMVStore.openMap(STORED_WCA_RESULTS_CAUSES_MAP_NAME, mapBuilder);
wfMVStore.removeMap(storedCausesMap);
// remove info about stored wca results
wfMVStore.removeMap(storedWcaResultsMap);
// commit removal
wfMVStore.commit();
}
@Override
public OnlineWorkflowWcaResults getWcaResults(String workflowId) {
Objects.requireNonNull(workflowId, "workflow id is null");
LOGGER.info("Getting WCA results of wf {}", workflowId);
if (isWorkflowStored(workflowId)) {
MVStore wfMVStore = getStore(workflowId);
if (wfMVStore.hasMap(STORED_WCA_RESULTS_MAP_NAME)) {
MVMap<String, String> storedRulesResultsMap = wfMVStore.openMap(STORED_WCA_RESULTS_MAP_NAME, mapBuilder);
// create workflow rules results
OnlineWorkflowWcaResultsImpl wfWcaResults = new OnlineWorkflowWcaResultsImpl(
workflowId,
TimeHorizon.valueOf(storedRulesResultsMap.get(STORED_RESULTS_TIMEHORIZON_KEY)));
// add classification of contingencies in clusters
MVMap<String, String> storedClustersMap = wfMVStore.openMap(STORED_WCA_RESULTS_CLUSTERS_MAP_NAME, mapBuilder);
MVMap<String, String> storedCausesMap = wfMVStore.openMap(STORED_WCA_RESULTS_CAUSES_MAP_NAME, mapBuilder);
for (String contingencyId : storedClustersMap.keySet()) {
String cause = storedCausesMap.get(contingencyId);
wfWcaResults.addContingencyWithCluster(contingencyId,
Integer.valueOf(storedClustersMap.get(contingencyId)),
cause != null ? Arrays.asList(cause) : null);
}
return wfWcaResults;
} else {
LOGGER.warn("No WCA results of wf {} stored in online db", workflowId);
return null;
}
} else {
LOGGER.warn("No data about wf {}", workflowId);
return null;
}
}
@Override
public void storeWorkflowParameters(String workflowId, OnlineWorkflowParameters parameters) {
Objects.requireNonNull(workflowId, "workflow id is null");
Objects.requireNonNull(parameters, "online workflow parameters is null");
LOGGER.info("Storing configuration parameters for workflow {}", workflowId);
MVStore wfMVStore = getStore(workflowId);
// check if the parameters for this wf have already been stored
if (wfMVStore.hasMap(STORED_PARAMETERS_MAP_NAME))
removeWfParameters(workflowId, wfMVStore);
MVMap<String, String> storedParametersMap = wfMVStore.openMap(STORED_PARAMETERS_MAP_NAME, mapBuilder);
// store basecase
storedParametersMap.put(STORED_PARAMETERS_BASECASE_KEY, parameters.getBaseCaseDate().toString());
// store number of states
storedParametersMap.put(STORED_PARAMETERS_STATE_NUMBER_KEY, Integer.toString(parameters.getStates()));
// store interval of historical data
storedParametersMap.put(STORED_PARAMETERS_HISTO_INTERVAL_KEY, parameters.getHistoInterval().toString());
// store offline workflow id
storedParametersMap.put(STORED_PARAMETERS_OFFLINE_WF_ID_KEY, parameters.getOfflineWorkflowId());
// store time horizon
storedParametersMap.put(STORED_RESULTS_TIMEHORIZON_KEY, parameters.getTimeHorizon().getName());
// store forecast error analysis id
storedParametersMap.put(STORED_PARAMETERS_FEA_ID_KEY, parameters.getFeAnalysisId());
// store rules purity threshold
storedParametersMap.put(STORED_PARAMETERS_RULES_PURITY_KEY, Double.toString(parameters.getRulesPurityThreshold()));
// store flag store states
storedParametersMap.put(STORED_PARAMETERS_STORE_STATES_KEY, Boolean.toString(parameters.storeStates()));
// store flag analyse basecase
storedParametersMap.put(STORED_PARAMETERS_ANALYSE_BASECASE_KEY, Boolean.toString(parameters.analyseBasecase()));
// store flag validation
storedParametersMap.put(STORED_PARAMETERS_VALIDATION_KEY, Boolean.toString(parameters.validation()));
// store security indexes
if (parameters.getSecurityIndexes() != null)
storedParametersMap.put(STORED_PARAMETERS_SECURITY_INDEXES_KEY, OnlineDbMVStoreUtils.indexesTypesToJson(parameters.getSecurityIndexes()));
// store case type
storedParametersMap.put(STORED_PARAMETERS_CASE_TYPE_KEY, parameters.getCaseType().name());
// store countries
storedParametersMap.put(STORED_PARAMETERS_COUNTRIES_KEY, OnlineDbMVStoreUtils.countriesToJson(parameters.getCountries()));
// store merge optimized flag
storedParametersMap.put(STORED_PARAMETERS_MERGE_OPTIMIZED_KEY, Boolean.toString(parameters.isMergeOptimized()));
// store merge optimized flag
storedParametersMap.put(STORED_PARAMETERS_LIMIT_REDUCTION_KEY, Float.toString(parameters.getLimitReduction()));
// store handle violations in N flag
storedParametersMap.put(STORED_PARAMETERS_HANDLE_VIOLATIONS_KEY, Boolean.toString(parameters.isHandleViolationsInN()));
// store merge constraint margin
storedParametersMap.put(STORED_PARAMETERS_CONSTRAINT_MARGIN_KEY, Float.toString(parameters.getConstraintMargin()));
// store case file name
if (parameters.getCaseFile() != null) {
storedParametersMap.put(STORED_PARAMETERS_CASE_FILE_KEY, parameters.getCaseFile());
}
wfMVStore.commit();
}
private void removeWfParameters(String workflowId, MVStore wfMVStore) {
LOGGER.debug("Removing existing parameters for workflow {}", workflowId);
MVMap<String, String> storedParametersMap = wfMVStore.openMap(STORED_PARAMETERS_MAP_NAME, mapBuilder);
// remove parameters
wfMVStore.removeMap(storedParametersMap);
// commit removal
wfMVStore.commit();
}
@Override
public OnlineWorkflowParameters getWorkflowParameters(String workflowId) {
Objects.requireNonNull(workflowId, "workflow id is null");
LOGGER.info("Getting configuration parameters of wf {}", workflowId);
if (isWorkflowStored(workflowId)) {
MVStore wfMVStore = null;
try {
wfMVStore = MVStore.open(config.getOnlineDbDir().toString() + File.separator + STORED_WORKFLOW_PREFIX + workflowId);
if (wfMVStore.hasMap(STORED_PARAMETERS_MAP_NAME)) {
MVMap<String, String> storedParametersMap = wfMVStore.openMap(STORED_PARAMETERS_MAP_NAME, mapBuilder);
DateTime baseCaseDate = DateTime.parse(storedParametersMap.get(STORED_PARAMETERS_BASECASE_KEY));
int states = Integer.parseInt(storedParametersMap.get(STORED_PARAMETERS_STATE_NUMBER_KEY));
String offlineWorkflowId = storedParametersMap.get(STORED_PARAMETERS_OFFLINE_WF_ID_KEY);
TimeHorizon timeHorizon = TimeHorizon.fromName(storedParametersMap.get(STORED_RESULTS_TIMEHORIZON_KEY));
Interval histoInterval = Interval.parse(storedParametersMap.get(STORED_PARAMETERS_HISTO_INTERVAL_KEY));
String feAnalysisId = storedParametersMap.get(STORED_PARAMETERS_FEA_ID_KEY);
double rulesPurityThreshold = Double.parseDouble((storedParametersMap.get(STORED_PARAMETERS_RULES_PURITY_KEY) == null) ? "1" : storedParametersMap.get(STORED_PARAMETERS_RULES_PURITY_KEY));
boolean storeStates = Boolean.parseBoolean(storedParametersMap.get(STORED_PARAMETERS_STORE_STATES_KEY));
boolean analyseBasecase = Boolean.parseBoolean(storedParametersMap.get(STORED_PARAMETERS_ANALYSE_BASECASE_KEY));
boolean validation = Boolean.parseBoolean(storedParametersMap.get(STORED_PARAMETERS_VALIDATION_KEY));
Set<SecurityIndexType> securityIndexes = null;
if (storedParametersMap.containsKey(STORED_PARAMETERS_SECURITY_INDEXES_KEY))
securityIndexes = OnlineDbMVStoreUtils.jsonToIndexesTypes(storedParametersMap.get(STORED_PARAMETERS_SECURITY_INDEXES_KEY));
CaseType caseType = CaseType.valueOf(storedParametersMap.get(STORED_PARAMETERS_CASE_TYPE_KEY));
Set<Country> countries = OnlineDbMVStoreUtils.jsonToCountries(storedParametersMap.get(STORED_PARAMETERS_COUNTRIES_KEY));
boolean mergeOptimized = OnlineWorkflowParameters.DEFAULT_MERGE_OPTIMIZED;
if (storedParametersMap.containsKey(STORED_PARAMETERS_MERGE_OPTIMIZED_KEY))
mergeOptimized = Boolean.parseBoolean(storedParametersMap.get(STORED_PARAMETERS_MERGE_OPTIMIZED_KEY));
float limitReduction = OnlineWorkflowParameters.DEFAULT_LIMIT_REDUCTION;
if (storedParametersMap.containsKey(STORED_PARAMETERS_LIMIT_REDUCTION_KEY))
limitReduction = Float.parseFloat(storedParametersMap.get(STORED_PARAMETERS_LIMIT_REDUCTION_KEY));
boolean handleViolations = OnlineWorkflowParameters.DEFAULT_HANDLE_VIOLATIONS_IN_N;
if (storedParametersMap.containsKey(STORED_PARAMETERS_HANDLE_VIOLATIONS_KEY))
handleViolations = Boolean.parseBoolean(storedParametersMap.get(STORED_PARAMETERS_HANDLE_VIOLATIONS_KEY));
float constraintMargin = OnlineWorkflowParameters.DEFAULT_CONSTRAINT_MARGIN;
if (storedParametersMap.containsKey(STORED_PARAMETERS_CONSTRAINT_MARGIN_KEY))
constraintMargin = Float.parseFloat(storedParametersMap.get(STORED_PARAMETERS_CONSTRAINT_MARGIN_KEY));
OnlineWorkflowParameters onlineWfPars = new OnlineWorkflowParameters(baseCaseDate,
states,
histoInterval,
offlineWorkflowId,
timeHorizon,
feAnalysisId,
rulesPurityThreshold,
storeStates,
analyseBasecase,
validation,
securityIndexes,
caseType,
countries,
mergeOptimized,
limitReduction,
handleViolations,
constraintMargin);
if (storedParametersMap.containsKey(STORED_PARAMETERS_CASE_FILE_KEY)) {
onlineWfPars.setCaseFile(storedParametersMap.get(STORED_PARAMETERS_CASE_FILE_KEY));
}
return onlineWfPars;
} else {
LOGGER.warn("No configuration parameters of wf {} stored in online db", workflowId);
return null;
}
} finally {
if (wfMVStore != null) {
wfMVStore.close();
}
}
} else {
LOGGER.warn("No data about wf {}", workflowId);
return null;
}
}
@Override
public void storeStatesProcessingStatus(String workflowId, Map<Integer, ? extends StateProcessingStatus> statesProcessingStatus) {
Objects.requireNonNull(workflowId, "workflow id is null");
Objects.requireNonNull(statesProcessingStatus, "online workflow states processing status is null");
LOGGER.info("Storing states processing status for workflow {}", workflowId);
MVStore wfMVStore = getStore(workflowId);
// check if the states processing status for this wf have already been stored
if (wfMVStore.hasMap(STORED_STATES_PROCESSING_STATUS_MAP_NAME))
removeStatesProcessingStatus(workflowId, wfMVStore);
MVMap<String, String> statesProcessingStatusMap = wfMVStore.openMap(STORED_STATES_PROCESSING_STATUS_MAP_NAME, mapBuilder);
// store states with processing status
statesProcessingStatusMap.put(STORED_STATES_LIST_KEY, OnlineDbMVStoreUtils.stateIdsToJson(statesProcessingStatus.keySet()));
// store processing status for states
for (Integer stateId : statesProcessingStatus.keySet()) {
MVMap<String, String> stateProcessingStatusMap = wfMVStore.openMap(stateId.toString() + STORED_STATE_PROCESSING_STATUS_MAP_SUFFIX, mapBuilder);
for (String step : statesProcessingStatus.get(stateId).getStatus().keySet()) {
// store processing status
stateProcessingStatusMap.put(step, statesProcessingStatus.get(stateId).getStatus().get(step));
}
stateProcessingStatusMap.put(STORED_STATES_STATE_DETAILS_KEY, statesProcessingStatus.get(stateId).getDetail() == null ? "" : statesProcessingStatus.get(stateId).getDetail());
}
wfMVStore.commit();
}
private void removeStatesProcessingStatus(String workflowId, MVStore wfMVStore) {
LOGGER.debug("Removing existing states processing status for workflow {}", workflowId);
MVMap<String, String> statesProcessingStatusMap = wfMVStore.openMap(STORED_STATES_PROCESSING_STATUS_MAP_NAME, mapBuilder);
// remove processing status for states
Collection<Integer> stateWithProcessingStatus = OnlineDbMVStoreUtils.jsonToStatesIds(statesProcessingStatusMap.get(STORED_STATES_LIST_KEY));
for (Integer stateId : stateWithProcessingStatus) {
MVMap<String, String> stateProcessingStatusMap = wfMVStore.openMap(stateId.toString() + STORED_STATE_PROCESSING_STATUS_MAP_SUFFIX, mapBuilder);
wfMVStore.removeMap(stateProcessingStatusMap);
}
// remove info about states with processing status
wfMVStore.removeMap(statesProcessingStatusMap);
// commit removal
wfMVStore.commit();
}
@Override
public Map<Integer, ? extends StateProcessingStatus> getStatesProcessingStatus(String workflowId) {
Objects.requireNonNull(workflowId, "workflow id is null");
LOGGER.info("Getting states processing status of wf {}", workflowId);
if (isWorkflowStored(workflowId)) {
MVStore wfMVStore = getStore(workflowId);
if (wfMVStore.hasMap(STORED_STATES_PROCESSING_STATUS_MAP_NAME)) {
MVMap<String, String> statesProcessingStatusMap = wfMVStore.openMap(STORED_STATES_PROCESSING_STATUS_MAP_NAME, mapBuilder);
// create states processing status
Map<Integer, StateProcessingStatus> statesProcessingStatus = new HashMap<Integer, StateProcessingStatus>();
// add processing status for states
Collection<Integer> stateWithProcessingStatus = OnlineDbMVStoreUtils.jsonToStatesIds(statesProcessingStatusMap.get(STORED_STATES_LIST_KEY));
for (Integer stateId : stateWithProcessingStatus) {
MVMap<String, String> stateProcessingStatusMap = wfMVStore.openMap(stateId + STORED_STATE_PROCESSING_STATUS_MAP_SUFFIX, mapBuilder);
Map<String, String> processingStatus = new HashMap<String, String>();
for (String step : stateProcessingStatusMap.keySet()) {
if (!step.equals(STORED_STATES_STATE_DETAILS_KEY))
processingStatus.put(step, stateProcessingStatusMap.get(step));
}
statesProcessingStatus.put(stateId, new StateProcessingStatusImpl(processingStatus, stateProcessingStatusMap.get(STORED_STATES_STATE_DETAILS_KEY)));
}
return statesProcessingStatus;
} else {
LOGGER.warn("No states processing status of wf {} stored in online db", workflowId);
return null;
}
} else {
LOGGER.warn("No data about wf {}", workflowId);
return null;
}
}
@Override
public void storeState(String workflowId, Integer stateId, Network network, String contingencyId) {
String stateIdStr;
if (contingencyId != null) {
stateIdStr = OnlineUtils.getPostContingencyId(String.valueOf(stateId), contingencyId);
LOGGER.info("Storing post contingency state {} , contingency {} of workflow {}", stateIdStr, contingencyId, workflowId);
} else {
stateIdStr = String.valueOf(stateId);
LOGGER.info("Storing state {} of workflow {}", stateIdStr, workflowId);
}
if (network.getStateManager().getStateIds().contains(stateIdStr)) {
network.getStateManager().setWorkingState(stateIdStr);
Path workflowStatesFolder = getWorkflowStatesFolder(workflowId);