Merge pull request #65 from itesla/improve_iidm_caserepo

changes iidm file extension convention to xiidm (xml is still OK) and…

case-repository/src/main/java/eu/itesla_project/cases/EntsoeAndXmlCaseRepository.java

@@ -17,13 +17,13 @@
 import java.util.List;
 
 /**
- * Common ENTSOE and XML case repository layout:
+ * Common ENTSOE and IIDM case repository layout:
  * <pre>
  * CIM/SN/2013/01/15/20130115_0620_SN2_FR0.zip
  *    /FO/...
  * UCT/SN/...
  *    /FO/...
- * XML/SN/2013/01/15/20130115_0720_SN2_FR0.xml
+ * IIDM/SN/2013/01/15/20130115_0720_SN2_FR0.xml
  *    /FO/...
  * </pre>
  *
@@ -37,7 +37,7 @@ public EntsoeAndXmlCaseRepository(EntsoeCaseRepositoryConfig config, Computation
         super(config,
                 Arrays.asList(new EntsoeFormat(Importers.getImporter("CIM1", computationManager), "CIM"),
                         new EntsoeFormat(Importers.getImporter("UCTE", computationManager), "UCT"), // official ENTSOE formats)
-                        new EntsoeFormat(Importers.getImporter("XML", computationManager), "XML")), // IIDM XML format
+                        new EntsoeFormat(Importers.getImporter("XIIDM", computationManager), "IIDM")), // XIIDM format
                 (directory, baseName) -> new GenericReadOnlyDataSource(directory, baseName));
     }
 

case-repository/src/test/java/eu/itesla_project/cases/EntsoeAndXmlCaseRepositoryTest.java

@@ -34,6 +34,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.List;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -112,7 +113,7 @@ public void setUp() throws Exception {
                 .thenAnswer(invocation -> {
                     DataSourceMock dataSource = invocation.getArgumentAt(0, DataSourceMock.class);
                     Path file = dataSource.getDirectory().resolve(dataSource.getBaseName() + ".zip");
-                    return Files.isRegularFile(file) && Files.exists(file);
+                    return Files.isRegularFile(file);
                 });
         Mockito.when(cimImporter.getFormat())
                 .thenReturn("CIM1");
@@ -125,42 +126,47 @@ public void setUp() throws Exception {
                 .thenAnswer(invocation -> {
                     DataSourceMock dataSource = invocation.getArgumentAt(0, DataSourceMock.class);
                     Path file = dataSource.getDirectory().resolve(dataSource.getBaseName() + ".uct");
-                    return Files.isRegularFile(file) && Files.exists(file);
+                    return Files.isRegularFile(file);
                 });
         Mockito.when(uctImporter.getFormat())
                 .thenReturn("UCTE");
         uctNetwork = Mockito.mock(Network.class);
         Mockito.when(uctImporter.import_(Matchers.isA(DataSource.class), Matchers.any()))
                 .thenReturn(uctNetwork);
 
-        Importer xmlImporter = Mockito.mock(Importer.class);
-        Mockito.when(xmlImporter.exists(Matchers.isA(DataSource.class)))
+        Importer iidmImporter = Mockito.mock(Importer.class);
+        Mockito.when(iidmImporter.exists(Matchers.isA(DataSource.class)))
                 .thenAnswer(invocation -> {
                     DataSourceMock dataSource = invocation.getArgumentAt(0, DataSourceMock.class);
-                    Path file = dataSource.getDirectory().resolve(dataSource.getBaseName() + ".xml");
-                    return Files.isRegularFile(file) && Files.exists(file);
+                    Path file_xiidm = dataSource.getDirectory().resolve(dataSource.getBaseName() + ".xiidm");
+                    Path file_xml = dataSource.getDirectory().resolve(dataSource.getBaseName() + ".xml");
+                    return ((Files.isRegularFile(file_xiidm)) || Files.isRegularFile(file_xml));
                 });
-        Mockito.when(xmlImporter.getFormat())
-                .thenReturn("XML");
+        Mockito.when(iidmImporter.getFormat())
+                .thenReturn("XIIDM");
         xmlNetwork = Mockito.mock(Network.class);
-        Mockito.when(xmlImporter.import_(Matchers.isA(DataSource.class), Matchers.any()))
+        Mockito.when(iidmImporter.import_(Matchers.isA(DataSource.class), Matchers.any()))
                 .thenReturn(xmlNetwork);
 
 
         caseRepository = new EntsoeAndXmlCaseRepository(new EntsoeAndXmlCaseRepositoryConfig(rootDir, HashMultimap.create()),
                 Arrays.asList(new EntsoeCaseRepository.EntsoeFormat(cimImporter, "CIM"),
                         new EntsoeCaseRepository.EntsoeFormat(uctImporter, "UCT"),
-                        new EntsoeCaseRepository.EntsoeFormat(xmlImporter, "XML")
+                        new EntsoeCaseRepository.EntsoeFormat(iidmImporter, "IIDM")
                 ),
                 (directory, baseName) -> new DataSourceMock(directory, baseName));
 
-        Path dir5 = fileSystem.getPath("/XML/SN/2013/01/14");
+        Path dir5 = fileSystem.getPath("/IIDM/SN/2013/01/14");
         Files.createDirectories(dir5);
         createFile(dir5, "20130114_0015_SN1_FR0.xml");
-        Path dir6 = fileSystem.getPath("/XML/SN/2016/01/01");
+        Path dir6 = fileSystem.getPath("/IIDM/SN/2016/01/01");
         Files.createDirectories(dir6);
         createFile(dir6, "20160101_0015_SN5_FR0.xml");
         createFile(dir6, "20160101_0045_SN5_FR0.xml");
+
+        Path dir7 = fileSystem.getPath("/IIDM/SN/2016/02/02");
+        Files.createDirectories(dir7);
+        createFile(dir7, "20160202_0115_SN2_FR0.xiidm");
     }
 
     @After
@@ -181,4 +187,14 @@ public void testDataAvailable() throws Exception {
         assertTrue(caseRepository.dataAvailable(CaseType.SN, EnumSet.of(Country.FR), Interval.parse("2016-01-01T00:00:00+01:00/2016-01-14T01:00:00+01:00"))
                 .equals(Sets.newHashSet(DateTime.parse("2016-01-01T00:15:00+01:00"), DateTime.parse("2016-01-01T00:45:00+01:00"))));
     }
+
+    @Test
+    public void testLoadXiidm() throws Exception {
+        // check that, when cim and ucte is forbidden for france, xml is loaded
+        // file suffix .xiidm
+        caseRepository.getConfig().getForbiddenFormatsByGeographicalCode().put(UcteGeographicalCode.FR, "CIM1");
+        caseRepository.getConfig().getForbiddenFormatsByGeographicalCode().put(UcteGeographicalCode.FR, "UCTE");
+        assertTrue(caseRepository.load(DateTime.parse("2016-02-02T01:15:00+01:00"), CaseType.SN, Country.FR).equals(Collections.singletonList(xmlNetwork)));
+    }
+
 }

histoDB/histodb-dataserver/src/main/java/eu/itesla_project/histodb/server/CimHistoImporter.java

@@ -55,7 +55,7 @@ public class CimHistoImporter {
             } else {
                 log.warn("CIM importer implementation not found");
             }
-            importerXml = Importers.getImporter("XML", computationManager);
+            importerXml = Importers.getImporter("XIIDM", computationManager);
             if (importerXml!=null) {
                 importers.add(importerXml);
             } else {

iidm-import-export-api/pom.xml

@@ -50,5 +50,9 @@
             <artifactId>computation-local</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+        </dependency>
     </dependencies>
 </project>

iidm-import-export-api/src/main/java/eu/itesla_project/iidm/datasource/Bzip2FileDataSource.java

@@ -0,0 +1,76 @@
+/**
+ * Copyright (c) 2016, RTE (http://www.rte-france.com)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+package eu.itesla_project.iidm.datasource;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Objects;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+
+
+/**
+ * @author Christian Biasuzzi <christian.biasuzzi@techrain.it>
+ */
+public class Bzip2FileDataSource implements DataSource {
+
+    private final Path directory;
+
+    private final String baseName;
+
+    private final DataSourceObserver observer;
+
+    public Bzip2FileDataSource(Path directory, String baseName, DataSourceObserver observer) {
+        this.directory = Objects.requireNonNull(directory);
+        this.baseName = Objects.requireNonNull(baseName);
+        this.observer = observer;
+    }
+
+    public Bzip2FileDataSource(Path directory, String baseName) {
+        this(directory, baseName, null);
+    }
+
+    @Override
+    public String getBaseName() {
+        return baseName;
+    }
+
+    private Path getPath(String suffix, String ext) {
+        return directory.resolve(DataSourceUtil.getFileName(baseName, suffix, ext + ".bz2"));
+    }
+
+    @Override
+    public OutputStream newOutputStream(String suffix, String ext, boolean append) throws IOException {
+        Path path = getPath(suffix, ext);
+        OutputStream os = new BZip2CompressorOutputStream(new BufferedOutputStream(Files.newOutputStream(path, append ? StandardOpenOption.APPEND : StandardOpenOption.CREATE)));
+        return observer != null ? new ObservableOutputStream(os, path.toString(), observer) : os;
+    }
+
+    @Override
+    public boolean exists(String suffix, String ext) {
+        Path path = getPath(suffix, ext);
+        return Files.isRegularFile(path);
+    }
+
+    @Override
+    public boolean exists(String fileName) throws IOException {
+        Path path = directory.resolve(fileName + ".bz2");
+        return Files.isRegularFile(path);
+    }
+
+    @Override
+    public InputStream newInputStream(String suffix, String ext) throws IOException {
+        return new BZip2CompressorInputStream(new BufferedInputStream(Files.newInputStream(getPath(suffix, ext))));
+    }
+
+    @Override
+    public InputStream newInputStream(String fileName) throws IOException {
+        return new BZip2CompressorInputStream(new BufferedInputStream(Files.newInputStream(directory.resolve(fileName + ".bz2"))));
+    }
+}

iidm-import-export-api/src/main/java/eu/itesla_project/iidm/datasource/GenericReadOnlyDataSource.java

@@ -1,5 +1,6 @@
 /**
  * Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium)
+ * Copyright (c) 2016, RTE (http://www.rte-france.com)
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
@@ -18,9 +19,10 @@ public class GenericReadOnlyDataSource implements ReadOnlyDataSource {
     private final ReadOnlyDataSource[] dataSources;
 
     public GenericReadOnlyDataSource(Path directory, String baseName, DataSourceObserver observer) {
-        dataSources = new DataSource[] { new FileDataSource(directory, baseName, observer),
-                                         new GzFileDataSource(directory, baseName, observer),
-                                         new ZipFileDataSource(directory, baseName + ".zip", baseName, observer) };
+        dataSources = new DataSource[]{ new FileDataSource(directory, baseName, observer),
+                new GzFileDataSource(directory, baseName, observer),
+                new ZipFileDataSource(directory, baseName + ".zip", baseName, observer),
+                new Bzip2FileDataSource(directory, baseName, observer)};
     }
 
     public GenericReadOnlyDataSource(Path directory, String baseName) {

iidm-import-export-api/src/main/java/eu/itesla_project/iidm/import_/Importers.java

@@ -340,6 +340,8 @@ public static ReadOnlyDataSource createReadOnly(Path directory, String fileNameO
             return new ZipFileDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4)));
         } else if (fileNameOrBaseName.endsWith(".gz")) {
             return new GzFileDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 3)));
+        } else if (fileNameOrBaseName.endsWith(".bz2")) {
+            return new Bzip2FileDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4)));
         } else {
             return new FileDataSource(directory, getBaseName(fileNameOrBaseName));
         }

iidm-xml-import-export/pom.xml

@@ -19,7 +19,7 @@
 
     <artifactId>iidm-xml-import-export</artifactId>
 
-    <name>XML import export</name>
+    <name>IIDM XML import export</name>
 
     <dependencies>
         <dependency>

iidm-xml-import-export/src/main/java/eu/itesla_project/iidm/xml/XMLExporter.java

@@ -64,7 +64,7 @@ public class XMLExporter implements Exporter, XmlConstants {
 
     @Override
     public String getFormat() {
-        return "XML";
+        return "XIIDM";
     }
 
     @Override
@@ -96,12 +96,12 @@ public void export(Network network, Properties parameters, DataSource dataSource
         try {
             long startTime = System.currentTimeMillis();
 
-            try (OutputStream os = dataSource.newOutputStream(null, "xml", false);
+            try (OutputStream os = dataSource.newOutputStream(null, "xiidm", false);
                  BufferedOutputStream bos = new BufferedOutputStream(os)) {
                 NetworkXml.write(network, options, bos);
             }
 
-            LOGGER.debug("XML export done in {} ms", (System.currentTimeMillis() - startTime));
+            LOGGER.debug("XIIDM export done in {} ms", (System.currentTimeMillis() - startTime));
         } catch (IOException e) {
             throw new RuntimeException(e);
         }

iidm-xml-import-export/src/main/java/eu/itesla_project/iidm/xml/XMLImporter.java

@@ -38,7 +38,7 @@ public class XMLImporter implements Importer, XmlConstants {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(XMLImporter.class);
 
-    private final String[] EXTENSIONS = { "xml", "iidm" };
+    private final String[] EXTENSIONS = { "xiidm", "iidm", "xml" };
 
     private static final Supplier<XMLInputFactory> XML_INPUT_FACTORY_SUPPLIER = Suppliers.memoize(XMLInputFactory::newInstance);
 
@@ -47,7 +47,7 @@ public class XMLImporter implements Importer, XmlConstants {
 
     @Override
     public String getFormat() {
-        return "XML";
+        return "XIIDM";
     }
 
     @Override
@@ -123,7 +123,7 @@ public Network import_(ReadOnlyDataSource dataSource, Properties parameters) {
             try (InputStream is = dataSource.newInputStream(null, ext)) {
                 network = NetworkXml.read(is, new XmlImportConfig(throwExceptionIfExtensionNotFound));
             }
-            LOGGER.debug("XML import done in {} ms", (System.currentTimeMillis() - startTime));
+            LOGGER.debug("XIIDM import done in {} ms", (System.currentTimeMillis() - startTime));
         } catch (IOException e) {
             throw new ITeslaException(e);
         }

online-workflow/src/main/java/eu/itesla_project/online/db/OnlineDbMVStore.java

@@ -1021,7 +1021,7 @@ public void storeState(String workflowId, Integer stateId, Network network) {
             parameters.setProperty("iidm.export.xml.with-branch-state-variables", "true");
             parameters.setProperty("iidm.export.xml.with-breakers", "true");
             parameters.setProperty("iidm.export.xml.with-properties", "true");
-            Exporters.export("XML", network, parameters, dataSource);
+            Exporters.export("XIIDM", network, parameters, dataSource);
             // store network state values, for later serialization
             Map<HistoDbAttributeId, Object> networkValues = IIDM2DB.extractCimValues(network, new IIDM2DB.Config(network.getId(), true, true)).getSingleValueMap();
             ConcurrentHashMap<Integer, Map<HistoDbAttributeId, Object>> workflowStates = new ConcurrentHashMap<Integer, Map<HistoDbAttributeId,Object>>();
@@ -1116,9 +1116,9 @@ public Network getState(String workflowId, Integer stateId) {
                 if ( stateFileName.endsWith(".xml"));
                 String basename = stateFileName.substring(0, stateFileName.length()-4);
                 DataSource dataSource = new FileDataSource(stateFolder, basename);
-                //Network network = Importers.import_("XML", dataSource, null);
+                //Network network = Importers.import_("XIIDM", dataSource, null);
                 // with the new post processors configuration, the post processing is applied also to xml import
-                Importer xmlImporter = Importers.getImporter("XML");
+                Importer xmlImporter = Importers.getImporter("XIIDM");
                 Importer noppImporter = Importers.removePostProcessors(xmlImporter);
                 Network network = noppImporter.import_(dataSource, null);
                 return network;

pom.xml

@@ -122,6 +122,7 @@
         <gson.version>2.6.2</gson.version>
         <jsonlib.version>2.4</jsonlib.version>
         <xmlunit.version>2.1.1</xmlunit.version>
+        <commons-compress.version>1.12</commons-compress.version>
     </properties>
 
     <licenses>
@@ -433,6 +434,11 @@
                 <classifier>jdk15</classifier>
                 <version>${jsonlib.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-compress</artifactId>
+                <version>${commons-compress.version}</version>
+            </dependency>
 
             <dependency>
                 <groupId>junit</groupId>