From e4768c13dec7463371cfa05fab9abb21ba262d83 Mon Sep 17 00:00:00 2001 From: massimoferraro Date: Mon, 10 Apr 2017 15:39:42 +0200 Subject: [PATCH] Use histodb client instead of REST API to access historical data during forecast errors analysis --- mcla-integration/pom.xml | 10 + .../forecast_errors/FEAHistoDBFacade.java | 112 +++----- .../ForecastErrorsAnalyzerConfig.java | 30 +- .../ForecastErrorsAnalyzerFactoryImpl.java | 11 +- .../ForecastErrorsAnalyzerImpl.java | 55 ++-- .../forecast_errors/HttpsClientHelper.java | 267 ------------------ .../mcla/FEAHistoDBFacadeTest.java | 94 ++++++ .../mcla/ForecastErrorsAnalyzerFactory.java | 7 +- .../ForecastErrorsAnalyzerParameters.java | 1 + .../ForecastErrorsAnalysis.java | 30 +- .../ForecastErrorsAnalysisConfig.java | 126 +++++---- 11 files changed, 273 insertions(+), 470 deletions(-) delete mode 100644 mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/HttpsClientHelper.java create mode 100644 mcla-integration/src/test/java/eu/itesla_project/mcla/FEAHistoDBFacadeTest.java diff --git a/mcla-integration/pom.xml b/mcla-integration/pom.xml index 6fda31d3..4e1d552b 100644 --- a/mcla-integration/pom.xml +++ b/mcla-integration/pom.xml @@ -69,11 +69,21 @@ ${project.version} + + com.google.jimfs + jimfs + test + junit junit test + + org.mockito + mockito-all + test + diff --git a/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/FEAHistoDBFacade.java b/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/FEAHistoDBFacade.java index 8fbb3437..09cdf211 100644 --- a/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/FEAHistoDBFacade.java +++ b/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/FEAHistoDBFacade.java @@ -1,91 +1,63 @@ /** * Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium) + * Copyright (c) 2017, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ package eu.itesla_project.mcla.forecast_errors; +import java.io.InputStream; +import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Objects; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; -import org.joda.time.DateTime; import org.joda.time.Interval; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.ISODateTimeFormat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import eu.itesla_project.modules.online.TimeHorizon; +import eu.itesla_project.modules.histo.HistoDbAttr; +import eu.itesla_project.modules.histo.HistoDbAttributeId; +import eu.itesla_project.modules.histo.HistoDbClient; +import eu.itesla_project.modules.histo.HistoDbHorizon; +import eu.itesla_project.modules.histo.HistoDbMetaAttributeId; +import eu.itesla_project.modules.histo.HistoDbMetaAttributeType; +import eu.itesla_project.modules.histo.HistoDbNetworkAttributeId; +import eu.itesla_project.modules.histo.HistoQueryType; /** * * @author Quinary */ -public class FEAHistoDBFacade { - - private static final Logger LOGGER = LoggerFactory.getLogger(FEAHistoDBFacade.class); - - private final int MAXRECORDSNUM = 100000; - - DataMiningFacadeRestConfig config; - TimeHorizon timeHorizon; - Interval histoInterval; - ArrayList generatorsIds; - ArrayList loadsIds; +public final class FEAHistoDBFacade { - public FEAHistoDBFacade(DataMiningFacadeRestConfig config, TimeHorizon timeHorizon, Interval histoInterval, - ArrayList generatorsIds, ArrayList loadsIds) { - Objects.requireNonNull(config, "config is null"); - Objects.requireNonNull(timeHorizon, "time horizon is null"); - Objects.requireNonNull(histoInterval, "histo interval is null"); - Objects.requireNonNull(generatorsIds, "generatorsIds is null"); - Objects.requireNonNull(loadsIds, "loadsIds is null"); - - this.config = config; - this.timeHorizon = timeHorizon; - this.histoInterval = histoInterval; - this.generatorsIds = generatorsIds; - this.loadsIds = loadsIds; - } + private FEAHistoDBFacade() { + } - public void historicalDataToCsvFile(Path historicalDataCsvFile) throws Exception { - String query = historicalDataQuery(); - LOGGER.info("Downloading data from HistoDB to file " + historicalDataCsvFile); - LOGGER.debug("HistoDB query = " + query); - HttpsClientHelper.remoteDataToFilePOST( - config.getRestServiceUrl(), - query, - config.getServiceUser(), - config.getServicePassword(), - historicalDataCsvFile.toString()); - } - - protected String historicalDataQuery() { - String query = "headers=true"; - query += "&count=" + MAXRECORDSNUM; - DateTimeFormatter dateFormatter = ISODateTimeFormat.date(); - DateTime intervalStart = histoInterval.getStart(); - DateTime intervalEnd = histoInterval.getEnd(); - query += "&time=[" + intervalStart.toString(dateFormatter) + "," + intervalEnd.toString(dateFormatter) + "]"; - switch (timeHorizon) { - case DACF: - query += "&horizon=" + timeHorizon.getName(); - break; - default: - throw new AssertionError(); - } - if ( timeHorizon.getForecastTime() >= 0 ) - query += "&forecast=" + timeHorizon.getForecastTime(); - query += "&cols=datetime,horizon,forecastTime"; - for ( String generatorId : generatorsIds ) { - query += "," + generatorId + "_P" + "," + generatorId + "_Q"; - } - for ( String loadId : loadsIds ) { - query += "," + loadId + "_P" + "," + loadId + "_Q"; - } - return query; - } + public static void historicalDataToCsvFile(HistoDbClient histoDbClient, List generatorsIds, List loadsIds, + Interval histoInterval, Path historicalDataCsvFile) throws Exception { + Set attributeIds = new LinkedHashSet<>(); + attributeIds.add(new HistoDbMetaAttributeId(HistoDbMetaAttributeType.datetime)); + attributeIds.add(new HistoDbMetaAttributeId(HistoDbMetaAttributeType.horizon)); + attributeIds.add(new HistoDbMetaAttributeId(HistoDbMetaAttributeType.forecastTime)); + generatorsIds.forEach( generatorId -> + { + attributeIds.add(new HistoDbNetworkAttributeId(generatorId, HistoDbAttr.P)); + attributeIds.add(new HistoDbNetworkAttributeId(generatorId, HistoDbAttr.Q)); + }); + loadsIds.forEach( loadId -> + { + attributeIds.add(new HistoDbNetworkAttributeId(loadId, HistoDbAttr.P)); + attributeIds.add(new HistoDbNetworkAttributeId(loadId, HistoDbAttr.Q)); + }); + try (InputStream is = histoDbClient.queryCsv(HistoQueryType.forecastDiff, + attributeIds, + histoInterval, + HistoDbHorizon.DACF, + false, + false)) { + Files.copy(is, historicalDataCsvFile); + } + } } diff --git a/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerConfig.java b/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerConfig.java index ccdd578d..898c9259 100644 --- a/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerConfig.java +++ b/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerConfig.java @@ -26,9 +26,6 @@ public class ForecastErrorsAnalyzerConfig { private Path binariesDir; private Path runtimeHomeDir; - private String histoDBUser; - private String histoDBPassword; - private String histoDBServiceUrl; private final boolean debug; private final Integer rngSeed; private final Integer checkModule0; @@ -53,9 +50,6 @@ public class ForecastErrorsAnalyzerConfig { public ForecastErrorsAnalyzerConfig( Path binariesDir, Path runtimeHomeDir, - String histoDBServiceUrl, - String histoDBUser, - String histoDBPassword, Integer checkModule0, double percpuGaussLoad, double percpuGaussRes, @@ -79,15 +73,9 @@ public ForecastErrorsAnalyzerConfig( ) { Objects.requireNonNull(binariesDir,"sampler compiled binaries directory is null"); Objects.requireNonNull(runtimeHomeDir,"matlab runtime directory is null"); - Objects.requireNonNull(histoDBServiceUrl, "histodb service url is null"); - Objects.requireNonNull(histoDBUser, "histodb user is null"); - Objects.requireNonNull(histoDBPassword, "histodb password is null"); this.binariesDir=binariesDir; this.runtimeHomeDir = runtimeHomeDir; - this.histoDBServiceUrl = histoDBServiceUrl; - this.histoDBUser = histoDBUser; - this.histoDBPassword = histoDBPassword; this.rngSeed = rngSeed; this.checkModule0=checkModule0; this.percpuGaussLoad=percpuGaussLoad; @@ -115,9 +103,6 @@ public static ForecastErrorsAnalyzerConfig load() { Path binariesDir = config.getPathProperty("binariesDir"); Path runtimeHomeDir = config.getPathProperty("runtimeHomeDir"); - String histoDBServiceUrl = config.getStringProperty("histoDBServiceUrl"); - String histoDBUser = config.getStringProperty("histoDBUser"); - String histoDBPassword = config.getStringProperty("histoDBPassword"); boolean debug = config.getBooleanProperty("debug", false); Integer checkModule0 = config.getOptionalIntProperty("checkModule0"); double percpuGaussLoad = config.getDoubleProperty("percpuGaussLoad"); @@ -139,7 +124,7 @@ public static ForecastErrorsAnalyzerConfig load() { double thresGUI = config.getDoubleProperty("thresGUI"); String nats = config.getStringProperty("nats","All"); - return new ForecastErrorsAnalyzerConfig(binariesDir, runtimeHomeDir, histoDBServiceUrl, histoDBUser, histoDBPassword, checkModule0, percpuGaussLoad, percpuGaussRes, correlationGauss, tolVar, nMinObsFract, nMinObsInterv, imputationMeth, nGaussians, kOutlier, tolerance, iterations, epsilo, conditionalSampling, tFlags, histo_estremeQ, thresGUI, nats, rngSeed, debug); + return new ForecastErrorsAnalyzerConfig(binariesDir, runtimeHomeDir, checkModule0, percpuGaussLoad, percpuGaussRes, correlationGauss, tolVar, nMinObsFract, nMinObsInterv, imputationMeth, nGaussians, kOutlier, tolerance, iterations, epsilo, conditionalSampling, tFlags, histo_estremeQ, thresGUI, nats, rngSeed, debug); } public Path getBinariesDir() { @@ -150,18 +135,6 @@ public Path getRuntimeHomeDir() { return runtimeHomeDir; } - public String getHistoDBUser() { - return histoDBUser; - } - - public String getHistoDBPassword() { - return histoDBPassword; - } - - public String getHistoDBServiceUrl() { - return histoDBServiceUrl; - } - public Integer getCheckModule0() { return checkModule0; } @@ -231,7 +204,6 @@ public boolean isDebug() { @Override public String toString() { return "ForecastErrorsAnalyzerConfig [binariesDir=" + binariesDir + ", runtimeHomeDir=" + runtimeHomeDir - + ", histoDBUser=" + histoDBUser + ", histoDBPassword=*" + ", histoDBServiceUrl=" + histoDBServiceUrl + ", check module0=" + checkModule0 + ", per cpu gauss load=" + percpuGaussLoad + ", per cpu gauss res=" + percpuGaussRes diff --git a/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerFactoryImpl.java b/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerFactoryImpl.java index e96283d5..ead902c0 100644 --- a/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerFactoryImpl.java +++ b/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerFactoryImpl.java @@ -1,5 +1,6 @@ /** * Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium) + * Copyright (c) 2017, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -8,6 +9,7 @@ import eu.itesla_project.computation.ComputationManager; import eu.itesla_project.iidm.network.Network; +import eu.itesla_project.modules.histo.HistoDbClient; import eu.itesla_project.modules.mcla.ForecastErrorsAnalyzer; import eu.itesla_project.modules.mcla.ForecastErrorsAnalyzerFactory; import eu.itesla_project.modules.mcla.ForecastErrorsDataStorage; @@ -18,9 +20,10 @@ */ public class ForecastErrorsAnalyzerFactoryImpl implements ForecastErrorsAnalyzerFactory { - @Override - public ForecastErrorsAnalyzer create(Network network, ComputationManager computationManager, ForecastErrorsDataStorage forecastErrorsDataStorage) { - return new ForecastErrorsAnalyzerImpl(network, computationManager, forecastErrorsDataStorage); - } + @Override + public ForecastErrorsAnalyzer create(Network network, ComputationManager computationManager, + ForecastErrorsDataStorage forecastErrorsDataStorage, HistoDbClient histoDbClient) { + return new ForecastErrorsAnalyzerImpl(network, computationManager, forecastErrorsDataStorage, histoDbClient); + } } diff --git a/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerImpl.java b/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerImpl.java index a164cd77..efb4607b 100644 --- a/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerImpl.java +++ b/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/ForecastErrorsAnalyzerImpl.java @@ -1,6 +1,6 @@ /** * Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium) - * Copyright (c) 2016, RTE (http://www.rte-france.com) + * Copyright (c) 2016-2017, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -11,10 +11,12 @@ import eu.itesla_project.iidm.network.Network; import eu.itesla_project.mcla.NetworkUtils; import eu.itesla_project.mcla.forecast_errors.data.ForecastErrorsHistoricalData; +import eu.itesla_project.modules.histo.HistoDbClient; import eu.itesla_project.modules.mcla.ForecastErrorsAnalyzer; import eu.itesla_project.modules.mcla.ForecastErrorsAnalyzerParameters; import eu.itesla_project.modules.mcla.ForecastErrorsDataStorage; import eu.itesla_project.modules.online.TimeHorizon; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +57,7 @@ public class ForecastErrorsAnalyzerImpl implements ForecastErrorsAnalyzer { private final ComputationManager computationManager; private final ForecastErrorsDataStorage forecastErrorsDataStorage; + private final HistoDbClient histoDbClient; private Network network; private ForecastErrorsAnalyzerConfig config = null; @@ -62,22 +65,21 @@ public class ForecastErrorsAnalyzerImpl implements ForecastErrorsAnalyzer { private ArrayList generatorsIds = new ArrayList(); private ArrayList loadsIds = new ArrayList(); - public ForecastErrorsAnalyzerImpl(Network network, ComputationManager computationManager, ForecastErrorsDataStorage forecastErrorsDataStorage, - ForecastErrorsAnalyzerConfig config) { - Objects.requireNonNull(network, "network is null"); - Objects.requireNonNull(computationManager, "computation manager is null"); - Objects.requireNonNull(forecastErrorsDataStorage, "forecast errors data storage is null"); - Objects.requireNonNull(config, "config is null"); - LOGGER.info(config.toString()); - this.network = network; - this.computationManager = computationManager; - this.forecastErrorsDataStorage = forecastErrorsDataStorage; - this.config = config; - } - - public ForecastErrorsAnalyzerImpl(Network network, ComputationManager computationManager, ForecastErrorsDataStorage forecastErrorsDataStorage) { - this(network, computationManager, forecastErrorsDataStorage, ForecastErrorsAnalyzerConfig.load()); - } + public ForecastErrorsAnalyzerImpl(Network network, ComputationManager computationManager, + ForecastErrorsDataStorage forecastErrorsDataStorage, + HistoDbClient histoDbClient, ForecastErrorsAnalyzerConfig config) { + this.network = Objects.requireNonNull(network, "network is null"); + this.computationManager = Objects.requireNonNull(computationManager, "computation manager is null"); + this.forecastErrorsDataStorage = Objects.requireNonNull(forecastErrorsDataStorage, "forecast errors data storage is null"); + this.histoDbClient = Objects.requireNonNull(histoDbClient, "HistoDb client is null"); + this.config = Objects.requireNonNull(config, "config is null"); + LOGGER.info(config.toString()); + } + + public ForecastErrorsAnalyzerImpl(Network network, ComputationManager computationManager, + ForecastErrorsDataStorage forecastErrorsDataStorage, HistoDbClient histoDbClient) { + this(network, computationManager, forecastErrorsDataStorage, histoDbClient, ForecastErrorsAnalyzerConfig.load()); + } @Override public String getName() { @@ -110,19 +112,12 @@ public void run(TimeHorizon timeHorizon) throws Exception { final Path workingDir = executor.getWorkingDir(); // get forecast errors historical data from histodb - FEAHistoDBFacade histoDBFacade = new FEAHistoDBFacade( - new DataMiningFacadeRestConfig( - config.getHistoDBServiceUrl(), - config.getHistoDBUser(), - config.getHistoDBPassword(), - workingDir, - config.isDebug()), - timeHorizon, - parameters.getHistoInterval(), - generatorsIds, - loadsIds); - Path historicalDataCsvFile = Paths.get(workingDir.toString(), FEACSVFILENAME); - histoDBFacade.historicalDataToCsvFile(historicalDataCsvFile); + Path historicalDataCsvFile = workingDir.resolve(FEACSVFILENAME); + FEAHistoDBFacade.historicalDataToCsvFile(histoDbClient, + generatorsIds, + loadsIds, + parameters.getHistoInterval(), + historicalDataCsvFile); //Path historicalDataCsvFile = Paths.get("/itesla_data/MAT", "forecastsDiff_7nodes.csv"); ForecastErrorsHistoricalData forecastErrorsHistoricalData = new HistoricalDataCreator(network, generatorsIds, loadsIds) diff --git a/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/HttpsClientHelper.java b/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/HttpsClientHelper.java deleted file mode 100644 index f78248c7..00000000 --- a/mcla-integration/src/main/java/eu/itesla_project/mcla/forecast_errors/HttpsClientHelper.java +++ /dev/null @@ -1,267 +0,0 @@ -/** - * Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium) - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ -package eu.itesla_project.mcla.forecast_errors; -import java.io.BufferedReader; -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.net.HttpURLConnection; -import java.net.URL; -import java.security.cert.X509Certificate; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - * @author Quinary - */ -public class HttpsClientHelper { - private static final Logger LOGGER = LoggerFactory.getLogger(HttpsClientHelper.class); - - public static void remoteDataToFile(String https_url, String username, - String password, String filePath) throws Exception { - remoteDataToFile(https_url, username, password, filePath, true); - } - - public static void remoteDataToFile(String https_url, String username, - String password, String filePath, boolean trustCerts) throws Exception - { - URL url; - - if (trustCerts == true) { - TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - - public void checkClientTrusted(X509Certificate[] certs, - String authType) { - } - - public void checkServerTrusted(X509Certificate[] certs, - String authType) { - } - } }; - SSLContext sc = SSLContext.getInstance("SSL"); - sc.init(null, trustAllCerts, new java.security.SecureRandom()); - HttpsURLConnection - .setDefaultSSLSocketFactory(sc.getSocketFactory()); - // Create & install all-trusting host name verifier - HttpsURLConnection - .setDefaultHostnameVerifier(new HostnameVerifier() { - public boolean verify(String hostname, - SSLSession session) { - return true; - } - }); - } - - url = new URL(https_url); - HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); - String userpass = username + ":" + password; - String basicAuth = "Basic " - + javax.xml.bind.DatatypeConverter.printBase64Binary(userpass - .getBytes()); - - con.setRequestProperty("Authorization", basicAuth); - File oFile = new File(filePath); - PrintWriter fw = new PrintWriter(oFile); - get_content(con, fw); - - } - - - public static void remoteDataToFilePOST(String https_url, String parameters, String username, - String password, String filePath) throws Exception { - remoteDataToFilePOST(https_url, parameters, username, password, filePath, true); - } - - public static void remoteDataToFilePOST(String https_url, String parameters, String username, - String password, String filePath, boolean trustCerts) throws Exception - { - URL url = new URL(https_url); - // Create & install all-trusting host name verifier - if ((trustCerts == true) && ("https".equalsIgnoreCase(url.getProtocol()))){ - TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - - public void checkClientTrusted(X509Certificate[] certs, - String authType) { - } - - public void checkServerTrusted(X509Certificate[] certs, - String authType) { - } - } }; - SSLContext sc = SSLContext.getInstance("SSL"); - sc.init(null, trustAllCerts, new java.security.SecureRandom()); - HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); - HttpsURLConnection - .setDefaultHostnameVerifier(new HostnameVerifier() { - public boolean verify(String hostname, - SSLSession session) { - return true; - } - }); - } - - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - String userpass = username + ":" + password; - String basicAuth = "Basic " + javax.xml.bind.DatatypeConverter.printBase64Binary(userpass.getBytes()); - con.setRequestProperty("Authorization", basicAuth); - con.setRequestMethod("POST"); - //con.setRequestProperty("User-Agent", USER_AGENT); - //con.setRequestProperty("Accept-Language", "en-US,en;q=0.5"); - con.setDoOutput(true); - - //send POST request - DataOutputStream wr = new DataOutputStream(con.getOutputStream()); - try { - wr.writeBytes(parameters); - } finally { - if (wr!=null) { - try { - wr.flush(); - wr.close(); - } catch (Throwable t) { - } - } - } - - int responseCode = con.getResponseCode(); - if (LOGGER.isTraceEnabled()) { - LOGGER.trace("Sending 'POST' request to URL : {} " , url); - LOGGER.trace("Post parameters : {}" , parameters); - LOGGER.trace("Response Code : {}" , responseCode); - } - switch (responseCode) { - case 200: - break; - default: throw new Exception("Unexpected HTTP code: " + responseCode); - } - - File oFile = new File(filePath); - PrintWriter fw = new PrintWriter(oFile); - get_content(con, fw); - } - - - - protected static void get_content(HttpURLConnection con, PrintWriter writer) throws Exception { - if ((con != null) && (writer != null)) { - BufferedReader br = null; - try { - br = new BufferedReader(new InputStreamReader( - con.getInputStream())); - - String input; - - - while ((input = br.readLine()) != null) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(input); - } - writer.println(input); - } - } finally { - try { - writer.flush(); - writer.close(); - } catch (Throwable t) { - } - if (br != null) { - try { - br.close(); - } catch (IOException e) { - } - } - } - - } - } - - public static void postCSVData(String https_url, String cvsData, String username, - String password, boolean trustCerts) throws Exception - { - URL url = new URL(https_url); - // Create & install all-trusting host name verifier - if ((trustCerts == true) && ("https".equalsIgnoreCase(url.getProtocol()))){ - TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - - public void checkClientTrusted(X509Certificate[] certs, - String authType) { - } - - public void checkServerTrusted(X509Certificate[] certs, - String authType) { - } - } }; - SSLContext sc = SSLContext.getInstance("SSL"); - sc.init(null, trustAllCerts, new java.security.SecureRandom()); - HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); - HttpsURLConnection - .setDefaultHostnameVerifier(new HostnameVerifier() { - public boolean verify(String hostname, - SSLSession session) { - return true; - } - }); - } - - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - String userpass = username + ":" + password; - String basicAuth = "Basic " + javax.xml.bind.DatatypeConverter.printBase64Binary(userpass.getBytes()); - con.setRequestProperty("Authorization", basicAuth); - // add csv content type, otherwise histodb does not know how to correctly handle the post - con.setRequestProperty("Content-type","text/csv"); - con.setRequestMethod("POST"); - //con.setRequestProperty("User-Agent", USER_AGENT); - //con.setRequestProperty("Accept-Language", "en-US,en;q=0.5"); - con.setDoOutput(true); - - //send POST request - DataOutputStream wr = new DataOutputStream(con.getOutputStream()); - try { - wr.writeBytes(cvsData); - } finally { - if (wr!=null) { - try { - wr.flush(); - wr.close(); - } catch (Throwable t) { - } - } - } - - int responseCode = con.getResponseCode(); - if (LOGGER.isTraceEnabled()) { - LOGGER.trace("Sending 'POST' request to URL : {} " , url); - LOGGER.trace("Response Code : {}" , responseCode); - } - switch (responseCode) { - case 200: - break; - default: throw new Exception("Unexpected HTTP code: " + responseCode); - } - } - -} \ No newline at end of file diff --git a/mcla-integration/src/test/java/eu/itesla_project/mcla/FEAHistoDBFacadeTest.java b/mcla-integration/src/test/java/eu/itesla_project/mcla/FEAHistoDBFacadeTest.java new file mode 100644 index 00000000..bb459228 --- /dev/null +++ b/mcla-integration/src/test/java/eu/itesla_project/mcla/FEAHistoDBFacadeTest.java @@ -0,0 +1,94 @@ +package eu.itesla_project.mcla; + +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.joda.time.Interval; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Matchers; +import org.mockito.Mockito; + +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; + +import eu.itesla_project.mcla.forecast_errors.FEAHistoDBFacade; +import eu.itesla_project.modules.histo.HistoDbClient; +import eu.itesla_project.modules.histo.HistoDbHorizon; +import eu.itesla_project.modules.histo.HistoQueryType; + +public class FEAHistoDBFacadeTest { + + private FileSystem fileSystem; + + @Before + public void setUp() { + fileSystem = Jimfs.newFileSystem(Configuration.unix()); + } + + @After + public void tearDown() throws Exception { + fileSystem.close(); + } + + @Test + public void testHistoricalDataToCsvFile() throws Exception { + List generatorsIds = Arrays.asList("generator1", "generator2"); + List loadsIds = Arrays.asList("load1", "load2", "load3"); + Interval histoInterval = Interval.parse("2013-01-01T00:00:00+01:00/2013-01-31T23:59:00+01:00"); + + String csvContent = String.join(System.lineSeparator(), + String.join(",", + "datetime", + "horizon", + "forecastTime", + generatorsIds.stream().map(generatorId -> String.join(",", generatorId + "_P", generatorId + "_Q")).collect(Collectors.joining(",")), + loadsIds.stream().map(loadId -> String.join(",", loadId + "_P", loadId + "_Q")).collect(Collectors.joining(","))), + String.join(",", + "Fri 01 Jan 2013 00:00:00 GMT","720","DACF", + "0.1","-0.1","0.2","-0.2", + "0.1","-0.1","0.2","-0.2","0.3","-0.3"), + String.join(",", + "Fri 01 Jan 2013 00:00:00 GMT","0","SN", + "0.11","-0.11","0.21","-0.21", + "0.11","-0.11","0.21","-0.21","0.31","-0.31")); + + HistoDbClient histoDbClient = Mockito.mock(HistoDbClient.class); + Mockito.when(histoDbClient.queryCsv(Matchers.eq(HistoQueryType.forecastDiff), + Matchers.any(), + Matchers.eq(histoInterval), + Matchers.eq(HistoDbHorizon.DACF), + Matchers.eq(false), + Matchers.eq(false))) + .thenReturn(new ByteArrayInputStream(csvContent.getBytes())); + + + String feaCsvFileName = "forecasterrors_historicaldata.csv"; + Path workingDir = Files.createDirectory(fileSystem.getPath("/working-dir")); + + Path historicalDataCsvFile = workingDir.resolve(feaCsvFileName); + FEAHistoDBFacade.historicalDataToCsvFile(histoDbClient, + generatorsIds, + loadsIds, + histoInterval, + historicalDataCsvFile); + + assertTrue(Files.exists(historicalDataCsvFile)); + try (InputStream expectedStream = new ByteArrayInputStream(csvContent.getBytes()); + InputStream actualStream = Files.newInputStream(historicalDataCsvFile)) { + assertTrue(IOUtils.contentEquals(expectedStream, actualStream)); + } + + } + +} diff --git a/modules/src/main/java/eu/itesla_project/modules/mcla/ForecastErrorsAnalyzerFactory.java b/modules/src/main/java/eu/itesla_project/modules/mcla/ForecastErrorsAnalyzerFactory.java index 82ba38d1..6a780827 100644 --- a/modules/src/main/java/eu/itesla_project/modules/mcla/ForecastErrorsAnalyzerFactory.java +++ b/modules/src/main/java/eu/itesla_project/modules/mcla/ForecastErrorsAnalyzerFactory.java @@ -1,5 +1,6 @@ /** * Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium) + * Copyright (c) 2017, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -8,13 +9,15 @@ import eu.itesla_project.computation.ComputationManager; import eu.itesla_project.iidm.network.Network; +import eu.itesla_project.modules.histo.HistoDbClient; /** * * @author Quinary */ public interface ForecastErrorsAnalyzerFactory { - - ForecastErrorsAnalyzer create(Network network, ComputationManager computationManager, ForecastErrorsDataStorage forecastErrorsDataStorage); + + ForecastErrorsAnalyzer create(Network network, ComputationManager computationManager, + ForecastErrorsDataStorage forecastErrorsDataStorage, HistoDbClient histoDbClient); } diff --git a/modules/src/main/java/eu/itesla_project/modules/mcla/ForecastErrorsAnalyzerParameters.java b/modules/src/main/java/eu/itesla_project/modules/mcla/ForecastErrorsAnalyzerParameters.java index 81ac7068..fc93a18e 100644 --- a/modules/src/main/java/eu/itesla_project/modules/mcla/ForecastErrorsAnalyzerParameters.java +++ b/modules/src/main/java/eu/itesla_project/modules/mcla/ForecastErrorsAnalyzerParameters.java @@ -1,5 +1,6 @@ /** * Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium) + * Copyright (c) 2017, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/online-workflow/src/main/java/eu/itesla_project/offline/forecast_errors/ForecastErrorsAnalysis.java b/online-workflow/src/main/java/eu/itesla_project/offline/forecast_errors/ForecastErrorsAnalysis.java index 56bb61f9..80de1181 100644 --- a/online-workflow/src/main/java/eu/itesla_project/offline/forecast_errors/ForecastErrorsAnalysis.java +++ b/online-workflow/src/main/java/eu/itesla_project/offline/forecast_errors/ForecastErrorsAnalysis.java @@ -1,20 +1,24 @@ /** * Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium) + * Copyright (c) 2017, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ package eu.itesla_project.offline.forecast_errors; -import eu.itesla_project.loadflow.api.LoadFlowFactory; -import eu.itesla_project.merge.MergeOptimizerFactory; -import eu.itesla_project.merge.MergeUtil; +import java.util.Objects; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import eu.itesla_project.cases.CaseRepository; import eu.itesla_project.computation.ComputationManager; import eu.itesla_project.iidm.network.Network; -import eu.itesla_project.cases.CaseRepository; +import eu.itesla_project.loadflow.api.LoadFlowFactory; +import eu.itesla_project.merge.MergeOptimizerFactory; +import eu.itesla_project.merge.MergeUtil; +import eu.itesla_project.modules.histo.HistoDbClient; import eu.itesla_project.modules.mcla.ForecastErrorsAnalyzer; import eu.itesla_project.modules.mcla.ForecastErrorsAnalyzerFactory; import eu.itesla_project.modules.mcla.ForecastErrorsAnalyzerParameters; @@ -37,18 +41,20 @@ public class ForecastErrorsAnalysis { private final ForecastErrorsAnalyzerFactory forecastErrorsAnalyzerFactory; private final LoadFlowFactory loadFlowFactory; private final MergeOptimizerFactory mergeOptimizerFactory; - - public ForecastErrorsAnalysis( - ComputationManager computationManager, - ForecastErrorsAnalysisConfig config, - ForecastErrorsAnalysisParameters parameters) throws InstantiationException, IllegalAccessException { - this.computationManager = computationManager; - this.parameters = parameters; + private final HistoDbClient histoDbClient; + + public ForecastErrorsAnalysis(ComputationManager computationManager, + ForecastErrorsAnalysisConfig config, + ForecastErrorsAnalysisParameters parameters) throws InstantiationException, IllegalAccessException { + Objects.requireNonNull(config); + this.computationManager = Objects.requireNonNull(computationManager); + this.parameters = Objects.requireNonNull(parameters); this.caseRepository = config.getCaseRepositoryFactoryClass().newInstance().create(computationManager); this.feDataStorage = config.getForecastErrorsDataStorageFactoryClass().newInstance().create(); this.forecastErrorsAnalyzerFactory = config.getForecastErrorsAnalyzerFactoryClass().newInstance(); this.loadFlowFactory = config.getLoadFlowFactoryClass().newInstance(); this.mergeOptimizerFactory = config.getMergeOtimizerFactoryClass().newInstance(); + this.histoDbClient = config.getHistoDbClientFactoryClass().newInstance().create(); logger.info(config.toString()); } @@ -71,7 +77,7 @@ private void runFEA(TimeHorizon[] timeHorizons) { logger.info("- Network id: " + network.getId()); logger.info("- Network name: "+ network.getName()); - ForecastErrorsAnalyzer feAnalyzer = forecastErrorsAnalyzerFactory.create(network, computationManager, feDataStorage); + ForecastErrorsAnalyzer feAnalyzer = forecastErrorsAnalyzerFactory.create(network, computationManager, feDataStorage, histoDbClient); feAnalyzer.init(new ForecastErrorsAnalyzerParameters( parameters.getHistoInterval(), parameters.getFeAnalysisId(), parameters.getIr(), parameters.getFlagPQ(), parameters.getMethod(), diff --git a/online-workflow/src/main/java/eu/itesla_project/offline/forecast_errors/ForecastErrorsAnalysisConfig.java b/online-workflow/src/main/java/eu/itesla_project/offline/forecast_errors/ForecastErrorsAnalysisConfig.java index 82e431e0..b9c2c0bd 100644 --- a/online-workflow/src/main/java/eu/itesla_project/offline/forecast_errors/ForecastErrorsAnalysisConfig.java +++ b/online-workflow/src/main/java/eu/itesla_project/offline/forecast_errors/ForecastErrorsAnalysisConfig.java @@ -1,19 +1,24 @@ /** * Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium) + * Copyright (c) 2017, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ package eu.itesla_project.offline.forecast_errors; +import java.util.Objects; + import eu.itesla_project.loadflow.api.LoadFlowFactory; import eu.itesla_project.merge.MergeOptimizerFactory; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.itesla_project.commons.config.ModuleConfig; import eu.itesla_project.commons.config.PlatformConfig; import eu.itesla_project.cases.CaseRepositoryFactory; +import eu.itesla_project.modules.histo.HistoDbClientFactory; import eu.itesla_project.modules.mcla.ForecastErrorsAnalyzerFactory; import eu.itesla_project.modules.mcla.ForecastErrorsDataStorageFactory; @@ -23,81 +28,90 @@ */ public class ForecastErrorsAnalysisConfig { - /* - * example of forecastErrorsAnalysis.properties file. - * + /* + * example of forecastErrorsAnalysis.properties file. + * caseRepositoryFactoryClass=eu.itesla_project.cases.EntsoeCaseRepositoryFactory forecastErrorsDataStorageFactoryClass=eu.itesla_project.matlab.mcla.ForecastErrorsDataStorageFactoryImpl forecastErrorsAnalyzerFactoryClass=eu.itesla_project.matlab.mcla.forecast_errors.ForecastErrorsAnalyzerFactoryImpl - - */ - - private static final Logger LOGGER = LoggerFactory.getLogger(ForecastErrorsAnalysisConfig.class); - - private final Class caseRepositoryFactoryClass; - private final Class forecastErrorsDataStorageFactoryClass; - private final Class forecastErrorsAnalyzerFactoryClass; - private final Class loadFlowFactoryClass; - private final Class mergeOtimizerFactoryClass; - - public ForecastErrorsAnalysisConfig(Class caseRepositoryFactoryClass, - Class forecastErrorsDataStorageFactoryClass, - Class forecastErrorsAnalyzerFactoryClass, - Class loadFlowFactoryClass, - Class mergeOtimizerFactoryClass) { - this.caseRepositoryFactoryClass = caseRepositoryFactoryClass; - this.forecastErrorsDataStorageFactoryClass = forecastErrorsDataStorageFactoryClass; - this.forecastErrorsAnalyzerFactoryClass = forecastErrorsAnalyzerFactoryClass; - this.loadFlowFactoryClass = loadFlowFactoryClass; - this.mergeOtimizerFactoryClass = mergeOtimizerFactoryClass; - } - - public static ForecastErrorsAnalysisConfig load() { + + */ + + private static final Logger LOGGER = LoggerFactory.getLogger(ForecastErrorsAnalysisConfig.class); + + private final Class caseRepositoryFactoryClass; + private final Class forecastErrorsDataStorageFactoryClass; + private final Class forecastErrorsAnalyzerFactoryClass; + private final Class loadFlowFactoryClass; + private final Class mergeOtimizerFactoryClass; + private final Class histoDbClientFactoryClass; + + public ForecastErrorsAnalysisConfig(Class caseRepositoryFactoryClass, + Class forecastErrorsDataStorageFactoryClass, + Class forecastErrorsAnalyzerFactoryClass, + Class loadFlowFactoryClass, + Class mergeOtimizerFactoryClass, + Class histoDbClientFactoryClass) { + this.caseRepositoryFactoryClass = Objects.requireNonNull(caseRepositoryFactoryClass); + this.forecastErrorsDataStorageFactoryClass = Objects.requireNonNull(forecastErrorsDataStorageFactoryClass); + this.forecastErrorsAnalyzerFactoryClass = Objects.requireNonNull(forecastErrorsAnalyzerFactoryClass); + this.loadFlowFactoryClass = Objects.requireNonNull(loadFlowFactoryClass); + this.mergeOtimizerFactoryClass = Objects.requireNonNull(mergeOtimizerFactoryClass); + this.histoDbClientFactoryClass = Objects.requireNonNull(histoDbClientFactoryClass); + } + + public static ForecastErrorsAnalysisConfig load() { ModuleConfig config = PlatformConfig.defaultConfig().getModuleConfig("forecastErrorsAnalysis"); - + Class caseRepositoryFactoryClass = config.getClassProperty("caseRepositoryFactoryClass", CaseRepositoryFactory.class); Class forecastErrorsDataStorageFactoryClass = config.getClassProperty("forecastErrorsDataStorageFactoryClass", ForecastErrorsDataStorageFactory.class); Class forecastErrorsAnalyzerFactoryClass = config.getClassProperty("forecastErrorsAnalyzerFactoryClass", ForecastErrorsAnalyzerFactory.class); - Class loadFlowFactoryClass = config.getClassProperty("loadFlowFactoryClass", LoadFlowFactory.class); - Class mergeOtimizerFactoryClass = config.getClassProperty("mergeOptimizerFactoryClass", MergeOptimizerFactory.class); + Class loadFlowFactoryClass = config.getClassProperty("loadFlowFactoryClass", LoadFlowFactory.class); + Class mergeOtimizerFactoryClass = config.getClassProperty("mergeOptimizerFactoryClass", MergeOptimizerFactory.class); + Class histoDbClientFactoryClass = config.getClassProperty("histoDbClientFactoryClass", HistoDbClientFactory.class); return new ForecastErrorsAnalysisConfig(caseRepositoryFactoryClass, forecastErrorsDataStorageFactoryClass, forecastErrorsAnalyzerFactoryClass, - loadFlowFactoryClass, mergeOtimizerFactoryClass); + loadFlowFactoryClass, mergeOtimizerFactoryClass, histoDbClientFactoryClass); } - public static Logger getLogger() { - return LOGGER; - } + public static Logger getLogger() { + return LOGGER; + } - public Class getCaseRepositoryFactoryClass() { + public Class getCaseRepositoryFactoryClass() { return caseRepositoryFactoryClass; } - - public Class getForecastErrorsDataStorageFactoryClass() { + + public Class getForecastErrorsDataStorageFactoryClass() { return forecastErrorsDataStorageFactoryClass; } - - public Class getForecastErrorsAnalyzerFactoryClass() { + + public Class getForecastErrorsAnalyzerFactoryClass() { return forecastErrorsAnalyzerFactoryClass; } - public Class getLoadFlowFactoryClass() { - return loadFlowFactoryClass; - } - - public Class getMergeOtimizerFactoryClass() { - return mergeOtimizerFactoryClass; - } - - @Override - public String toString() { - return "ForecastErrorsAnalsysConfig [caseRepositoryFactoryClass=" + caseRepositoryFactoryClass - + " forecastErrorsDataStorageFactoryClass=" + forecastErrorsDataStorageFactoryClass - + " forecastErrorsAnalyzerFactoryClass=" + forecastErrorsAnalyzerFactoryClass - + " loadFlowFactoryClass=" + loadFlowFactoryClass - + " mergeOtimizerFactoryClass=" + mergeOtimizerFactoryClass - + "]"; - } + public Class getLoadFlowFactoryClass() { + return loadFlowFactoryClass; + } + + public Class getMergeOtimizerFactoryClass() { + return mergeOtimizerFactoryClass; + } + + public Class getHistoDbClientFactoryClass() { + return histoDbClientFactoryClass; + } + + @Override + public String toString() { + return "ForecastErrorsAnalsysConfig [caseRepositoryFactoryClass=" + caseRepositoryFactoryClass + + " forecastErrorsDataStorageFactoryClass=" + forecastErrorsDataStorageFactoryClass + + " forecastErrorsAnalyzerFactoryClass=" + forecastErrorsAnalyzerFactoryClass + + " loadFlowFactoryClass=" + loadFlowFactoryClass + + " mergeOtimizerFactoryClass=" + mergeOtimizerFactoryClass + + " histoDbClientFactoryClass=" + histoDbClientFactoryClass + + "]"; + } }