Datasource: add mainExtension, existsStrict and use it in exists() me…

…thods of importers This allows to import the correct network when a path is provided, the importers will respect the full path and not declare that they can import from the datasource TODO explain more
powsybl · Jun 25, 2024 · ab5ebe1 · ab5ebe1
1 parent 6e18e4f
commit ab5ebe1
Show file tree

Hide file tree

Showing 30 changed files with 152 additions and 35 deletions.
diff --git a/cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/CgmesImport.java b/cgmes/cgmes-conversion/src/main/java/com/powsybl/cgmes/conversion/CgmesImport.java
@@ -189,6 +189,11 @@ public boolean exists(String suffix, String ext) throws IOException {
             return ds.exists(suffix, ext) && filter.test(DataSourceUtil.getFileName(getBaseName(), suffix, ext));
         }
 
+        @Override
+        public boolean existsStrict(String suffix, String ext) throws IOException {
+            return ds.existsStrict(suffix, ext) && filter.test(DataSourceUtil.getFileName(getBaseName(), suffix, ext));
+        }
+
         @Override
         public boolean exists(String fileName) throws IOException {
             return ds.exists(fileName) && filter.test(fileName);

diff --git a/...nversion/src/test/java/com/powsybl/cgmes/conversion/test/BusbarSectionConversionTest.java b/...nversion/src/test/java/com/powsybl/cgmes/conversion/test/BusbarSectionConversionTest.java
@@ -25,7 +25,7 @@ class BusbarSectionConversionTest extends AbstractSerDeTest {
     @Test
     void isolatedBusbarSectionWithoutTopologicalNodeInBusBreakerModelTest() {
         Network network = Network.read(
-                new ResourceDataSource("bbs-busbreaker",
+                new ResourceDataSource("",
                 new ResourceSet("/", "bbs-busbreaker_EQ.xml", "bbs-busbreaker_TP_withoutTN.xml")));
         // The network can be imported without any issue
         assertNotNull(network);
@@ -39,7 +39,7 @@ void isolatedBusbarSectionWithoutTopologicalNodeInBusBreakerModelTest() {
     @Test
     void isolatedBusbarSectionWithoutLinkToTopologicalNodeInBusBreakerModelTest() {
         Network network = Network.read(
-                new ResourceDataSource("bbs-busbreaker",
+                new ResourceDataSource("",
                         new ResourceSet("/", "bbs-busbreaker_EQ.xml", "bbs-busbreaker_TP_withoutLinkToTN.xml")));
         // The network can be imported without any issue
         assertNotNull(network);
@@ -55,7 +55,7 @@ void isolatedBusbarSectionWithoutLinkToTopologicalNodeInBusBreakerModelTest() {
     @Test
     void isolatedBusbarSectionWithTopologicalNodeInBusBreakerModelTest() {
         Network network = Network.read(
-                new ResourceDataSource("bbs-busbreaker",
+                new ResourceDataSource("",
                 new ResourceSet("/", "bbs-busbreaker_EQ.xml", "bbs-busbreaker_TP.xml")));
         // The network can be imported without any issue
         assertNotNull(network);

diff --git a/cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/ConversionTester.java b/cgmes/cgmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/ConversionTester.java
@@ -141,7 +141,7 @@ private void testConversion(Network expected, GridModelReference gm, ComparisonC
                 exportCgmes(gm.name(), impl, network);
             }
             if (testExportImportCgmes) {
-                testExportImportCgmes(network, ds, fs, i, iparams, config);
+                testExportImportCgmes(gm.name(), network, ds, fs, i, iparams, config);
             }
             if (validateBusBalances) {
                 validateBusBalances(network);
@@ -186,7 +186,7 @@ private static String subsetFromName(String name) {
                 .orElse("unknown");
     }
 
-    private void testExportImportCgmes(Network network, ReadOnlyDataSource originalDs, FileSystem fs, CgmesImport i, Properties iparams,
+    private void testExportImportCgmes(String namee, Network network, ReadOnlyDataSource originalDs, FileSystem fs, CgmesImport i, Properties iparams,
                                        ComparisonConfig config) throws IOException {
 
         // We copy everything from the original data source to the temporary destination with a normalized name
@@ -196,7 +196,7 @@ private void testExportImportCgmes(Network network, ReadOnlyDataSource originalD
         // Create a temporary directory to store the exported files
         Path path = fs.getPath("temp-export-cgmes");
         Files.createDirectories(path);
-        String baseName = originalDs.getBaseName();
+        String baseName = namee;
         DataSource ds = new ZipFileDataSource(path, baseName);
 
         // Copy the original files to the temporary destination, ensuring a normalized name

diff --git a/...gmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/GroundConversionTest.java b/...gmes-conversion/src/test/java/com/powsybl/cgmes/conversion/test/GroundConversionTest.java
@@ -55,7 +55,7 @@ void groundConversionRemoveTest() {
         Properties importParams = new Properties();
         importParams.put(CgmesImport.POST_PROCESSORS, "RemoveGrounds");
         Network network = Network.read(
-                new ResourceDataSource("groundTest.xml", new ResourceSet("/", "groundTest.xml")),
+                new ResourceDataSource("groundTest", new ResourceSet("/", "groundTest.xml")),
                 importParams);
 
         assertEquals(0, network.getGroundCount());

diff --git a/...onversion/src/test/java/com/powsybl/cgmes/conversion/test/export/EquipmentExportTest.java b/...onversion/src/test/java/com/powsybl/cgmes/conversion/test/export/EquipmentExportTest.java
@@ -1217,7 +1217,7 @@ private Network exportImport(Network expected, ReadOnlyDataSource dataSource, bo
         Path exportedEq = exportToCgmesEQ(expected, transformersWithHighestVoltageAtEnd1);
 
         // From reference data source we use only boundaries
-        Path repackaged = tmpDir.resolve("repackaged.zip");
+        Path repackaged = tmpDir.resolve("test.zip");
         Repackager r = new Repackager(dataSource)
                 .with("test_EQ.xml", exportedEq);
         if (importTP) {

diff --git a/...sion/src/test/java/com/powsybl/cgmes/conversion/test/export/StateVariablesExportTest.java b/...sion/src/test/java/com/powsybl/cgmes/conversion/test/export/StateVariablesExportTest.java
@@ -614,7 +614,7 @@ private boolean test(ReadOnlyDataSource dataSource, boolean exportTp, boolean ex
         }
 
         // Zip with new SV (and eventually a new TP)
-        Path repackaged = tmpDir.resolve("repackaged.zip");
+        Path repackaged = tmpDir.resolve("test.zip");
         Repackager r = new Repackager(dataSource)
                 .with("test_SV.xml", exportedSv)
                 .with("test_EQ_BD.xml", Repackager::eqBd)

diff --git a/...c/test/java/com/powsybl/cgmes/conversion/test/export/SteadyStateHypothesisExportTest.java b/...c/test/java/com/powsybl/cgmes/conversion/test/export/SteadyStateHypothesisExportTest.java
@@ -141,7 +141,7 @@ private boolean test(ReadOnlyDataSource dataSource, DifferenceEvaluator knownDif
         }
 
         // Zip with new SSH
-        Path repackaged = tmpDir.resolve("repackaged.zip");
+        Path repackaged = tmpDir.resolve("test.zip");
         Repackager r = new Repackager(dataSource)
                 .with("test_EQ.xml", Repackager::eq)
                 .with("test_TP.xml", Repackager::tp)

diff --git a/...conversion/src/test/java/com/powsybl/cgmes/conversion/test/export/TopologyExportTest.java b/...conversion/src/test/java/com/powsybl/cgmes/conversion/test/export/TopologyExportTest.java
@@ -78,7 +78,7 @@ private boolean test(ReadOnlyDataSource dataSource, boolean importSsh) throws IO
         }
 
         // Zip with new TP
-        Path repackaged = tmpDir.resolve("repackaged.zip");
+        Path repackaged = tmpDir.resolve("test.zip");
         Repackager r = new Repackager(dataSource)
                 .with("test_EQ.xml", Repackager::eq)
                 .with("test_TP.xml", exportedTp)

diff --git a/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesOnDataSource.java b/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/CgmesOnDataSource.java
@@ -33,6 +33,14 @@ public ReadOnlyDataSource dataSource() {
     }
 
     public boolean exists() {
+        // The user has requested a basename but we need unrestricted access to all files, abort
+        try {
+            if (!dataSource().getBaseName().isEmpty() && !(dataSource().existsStrict(null, "xml") || dataSource().existsStrict("_EQ", "xml") || dataSource().existsStrict("_SV", "xml"))) {
+                return false;
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
         // check that RDF and CIM16 are defined as namespaces in the data source
         Set<String> foundNamespaces = namespaces();
         if (!foundNamespaces.contains(RDF_NAMESPACE)) {

diff --git a/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/GridModelReferenceResources.java b/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/GridModelReferenceResources.java
@@ -33,7 +33,7 @@ public GridModelReferenceResources(
 
     @Override
     public ReadOnlyDataSource dataSource() {
-        ReadOnlyDataSource ds = new ResourceDataSource(baseNameFromResourceNames(), resourceSets);
+        ReadOnlyDataSource ds = new ResourceDataSource("", resourceSets);
         if (LOG.isInfoEnabled()) {
             try {
                 LOG.info("List of names in data source for {} = {}", name(), Arrays.toString(ds.listNames(".*").toArray()));

diff --git a/cgmes/cgmes-model/src/test/java/com/powsybl/cgmes/model/CgmesOnDataSourceTest.java b/cgmes/cgmes-model/src/test/java/com/powsybl/cgmes/model/CgmesOnDataSourceTest.java
@@ -21,7 +21,7 @@
 class CgmesOnDataSourceTest {
 
     private static void doTestExists(String filename, String cimVersion, boolean expectedExists) {
-        ReadOnlyDataSource dataSource = new ResourceDataSource("incomplete",
+        ReadOnlyDataSource dataSource = new ResourceDataSource("",
                 new ResourceSet("/", filename));
         CgmesOnDataSource cgmesOnDataSource = new CgmesOnDataSource(dataSource);
         boolean exists = "14".equals(cimVersion) ? cgmesOnDataSource.existsCim14() : cgmesOnDataSource.exists();

diff --git a/commons/src/main/java/com/powsybl/commons/datasource/Bzip2FileDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/Bzip2FileDataSource.java
@@ -18,6 +18,10 @@
  */
 public class Bzip2FileDataSource extends FileDataSource {
 
+    public Bzip2FileDataSource(Path directory, String baseName, String mainExtension, DataSourceObserver observer) {
+        super(directory, baseName, mainExtension, observer);
+    }
+
     public Bzip2FileDataSource(Path directory, String baseName, DataSourceObserver observer) {
         super(directory, baseName, observer);
     }

diff --git a/commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java b/commons/src/main/java/com/powsybl/commons/datasource/DataSourceUtil.java
@@ -39,24 +39,34 @@ static String getBaseName(String fileName) {
         return pos == -1 ? fileName : fileName.substring(0, pos);
     }
 
+    static String getMainExtension(String fileName) {
+        Objects.requireNonNull(fileName);
+        int pos = fileName.indexOf('.'); // find first dot in case of double extension (.xml.gz)
+        return pos == -1 ? "" : fileName.substring(pos + 1);
+    }
+
     static DataSource createDataSource(Path directory, String basename, CompressionFormat compressionExtension, DataSourceObserver observer) {
+        return createDataSource(directory, basename, "", compressionExtension, observer);
+    }
+
+    static DataSource createDataSource(Path directory, String basename, String mainExtension, CompressionFormat compressionExtension, DataSourceObserver observer) {
         Objects.requireNonNull(directory);
         Objects.requireNonNull(basename);
 
         if (compressionExtension == null) {
-            return new FileDataSource(directory, basename, observer);
+            return new FileDataSource(directory, basename, mainExtension, observer);
         } else {
             switch (compressionExtension) {
                 case BZIP2:
-                    return new Bzip2FileDataSource(directory, basename, observer);
+                    return new Bzip2FileDataSource(directory, basename, mainExtension, observer);
                 case GZIP:
-                    return new GzFileDataSource(directory, basename, observer);
+                    return new GzFileDataSource(directory, basename, mainExtension, observer);
                 case XZ:
-                    return new XZFileDataSource(directory, basename, observer);
+                    return new XZFileDataSource(directory, basename, mainExtension, observer);
                 case ZIP:
-                    return new ZipFileDataSource(directory, basename, observer);
+                    return new ZipFileDataSource(directory, basename + ".zip", basename, mainExtension, observer);
                 case ZSTD:
-                    return new ZstdFileDataSource(directory, basename, observer);
+                    return new ZstdFileDataSource(directory, basename, mainExtension, observer);
                 default:
                     throw new IllegalStateException("Unexpected CompressionFormat value: " + compressionExtension);
             }
@@ -68,17 +78,23 @@ static DataSource createDataSource(Path directory, String fileNameOrBaseName, Da
         Objects.requireNonNull(fileNameOrBaseName);
 
         if (fileNameOrBaseName.endsWith(".zst")) {
-            return new ZstdFileDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4)), observer);
+            String filenameWithoutCompressionExtension = fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4);
+            return new ZstdFileDataSource(directory, getBaseName(filenameWithoutCompressionExtension), getMainExtension(filenameWithoutCompressionExtension), observer);
         } else if (fileNameOrBaseName.endsWith(".zip")) {
-            return new ZipFileDataSource(directory, fileNameOrBaseName, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4)), observer);
+            String filenameWithoutCompressionExtension = fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4);
+            return new ZipFileDataSource(directory, filenameWithoutCompressionExtension + ".zip", getBaseName(filenameWithoutCompressionExtension), getMainExtension(filenameWithoutCompressionExtension), observer);
         } else if (fileNameOrBaseName.endsWith(".xz")) {
-            return new XZFileDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 3)), observer);
+            String filenameWithoutCompressionExtension = fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 3);
+            return new XZFileDataSource(directory, getBaseName(filenameWithoutCompressionExtension), getMainExtension(filenameWithoutCompressionExtension), observer);
         } else if (fileNameOrBaseName.endsWith(".gz")) {
-            return new GzFileDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 3)), observer);
+            String filenameWithoutCompressionExtension = fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 3);
+            return new GzFileDataSource(directory, getBaseName(filenameWithoutCompressionExtension), getMainExtension(filenameWithoutCompressionExtension), observer);
         } else if (fileNameOrBaseName.endsWith(".bz2")) {
-            return new Bzip2FileDataSource(directory, getBaseName(fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4)), observer);
+            String filenameWithoutCompressionExtension = fileNameOrBaseName.substring(0, fileNameOrBaseName.length() - 4);
+            return new Bzip2FileDataSource(directory, getBaseName(filenameWithoutCompressionExtension), getMainExtension(filenameWithoutCompressionExtension), observer);
         } else {
-            return new FileDataSource(directory, getBaseName(fileNameOrBaseName), observer);
+            return new FileDataSource(directory, getBaseName(fileNameOrBaseName), getMainExtension(fileNameOrBaseName), observer);
         }
     }
+
 }
diff --git a/commons/src/main/java/com/powsybl/commons/datasource/FileDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/FileDataSource.java
@@ -29,15 +29,22 @@ public class FileDataSource implements DataSource {
 
     private final String baseName;
 
+    private final String mainExtension;
+
     private final DataSourceObserver observer;
 
     public FileDataSource(Path directory, String baseName) {
-        this(directory, baseName, null);
+        this(directory, baseName, "", null);
     }
 
     public FileDataSource(Path directory, String baseName, DataSourceObserver observer) {
+        this(directory, baseName, "", observer);
+    }
+
+    public FileDataSource(Path directory, String baseName, String mainExtension, DataSourceObserver observer) {
         this.directory = Objects.requireNonNull(directory);
         this.baseName = Objects.requireNonNull(baseName);
+        this.mainExtension = Objects.requireNonNull(mainExtension);
         this.observer = observer;
     }
 
@@ -115,4 +122,10 @@ public Set<String> listNames(String regex) throws IOException {
                     .collect(Collectors.toSet());
         }
     }
+
+    @Override
+    public boolean existsStrict(String suffix, String ext) throws IOException {
+        return (mainExtension.isEmpty() || mainExtension.equals(ext)) && exists(suffix, ext);
+    }
+
 }
diff --git a/commons/src/main/java/com/powsybl/commons/datasource/GenericReadOnlyDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/GenericReadOnlyDataSource.java
@@ -59,6 +59,16 @@ public boolean exists(String suffix, String ext) throws IOException {
         return false;
     }
 
+    @Override
+    public boolean existsStrict(String suffix, String ext) throws IOException {
+        for (ReadOnlyDataSource dataSource : dataSources) {
+            if (dataSource.existsStrict(suffix, ext)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     @Override
     public boolean exists(String fileName) throws IOException {
         for (ReadOnlyDataSource dataSource : dataSources) {

diff --git a/commons/src/main/java/com/powsybl/commons/datasource/GzFileDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/GzFileDataSource.java
@@ -19,6 +19,10 @@
  */
 public class GzFileDataSource extends FileDataSource {
 
+    public GzFileDataSource(Path directory, String baseName, String mainExtension, DataSourceObserver observer) {
+        super(directory, baseName, mainExtension, observer);
+    }
+
     public GzFileDataSource(Path directory, String baseName, DataSourceObserver observer) {
         super(directory, baseName, observer);
     }

diff --git a/commons/src/main/java/com/powsybl/commons/datasource/MultipleReadOnlyDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/MultipleReadOnlyDataSource.java
@@ -51,6 +51,17 @@ public boolean exists(String suffix, String ext) throws IOException {
         });
     }
 
+    @Override
+    public boolean existsStrict(String suffix, String ext) throws IOException {
+        return dataSources.stream().anyMatch(dataSource -> {
+            try {
+                return dataSource.existsStrict(suffix, ext);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        });
+    }
+
     @Override
     public boolean exists(String fileName) throws IOException {
         return dataSources.stream().anyMatch(dataSource -> {

diff --git a/commons/src/main/java/com/powsybl/commons/datasource/ReadOnlyDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/ReadOnlyDataSource.java
@@ -22,6 +22,8 @@ public interface ReadOnlyDataSource {
 
     boolean exists(String fileName) throws IOException;
 
+    boolean existsStrict(String suffix, String ext) throws IOException;
+
     InputStream newInputStream(String suffix, String ext) throws IOException;
 
     InputStream newInputStream(String fileName) throws IOException;

diff --git a/commons/src/main/java/com/powsybl/commons/datasource/ReadOnlyMemDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/ReadOnlyMemDataSource.java
@@ -29,12 +29,19 @@ public class ReadOnlyMemDataSource implements ReadOnlyDataSource {
 
     private final String baseName;
 
+    private final String mainExtension;
+
     public ReadOnlyMemDataSource() {
-        this("");
+        this("", "");
     }
 
     public ReadOnlyMemDataSource(String baseName) {
+        this(baseName, "");
+    }
+
+    public ReadOnlyMemDataSource(String baseName, String mainExtension) {
         this.baseName = Objects.requireNonNull(baseName);
+        this.mainExtension = Objects.requireNonNull(mainExtension);
     }
 
     public byte[] getData(String suffix, String ext) {
@@ -67,6 +74,11 @@ public boolean exists(String suffix, String ext) throws IOException {
         return exists(DataSourceUtil.getFileName(baseName, suffix, ext));
     }
 
+    @Override
+    public boolean existsStrict(String suffix, String ext) throws IOException {
+        return (mainExtension.isEmpty() || mainExtension.equals(ext)) && exists(suffix, ext);
+    }
+
     @Override
     public boolean exists(String fileName) throws IOException {
         Objects.requireNonNull(fileName);

diff --git a/commons/src/main/java/com/powsybl/commons/datasource/ResourceDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/ResourceDataSource.java
@@ -22,14 +22,21 @@ public class ResourceDataSource implements ReadOnlyDataSource {
 
     private final String baseName;
 
+    private final String mainExtension;
+
     private final List<ResourceSet> resourceSets;
 
     public ResourceDataSource(String baseName, ResourceSet... resourceSets) {
         this(baseName, Arrays.asList(resourceSets));
     }
 
     public ResourceDataSource(String baseName, List<ResourceSet> resourceSets) {
+        this(baseName, "", resourceSets);
+    }
+
+    public ResourceDataSource(String baseName, String mainExtension, List<ResourceSet> resourceSets) {
         this.baseName = Objects.requireNonNull(baseName);
+        this.mainExtension = Objects.requireNonNull(mainExtension);
         this.resourceSets = Objects.requireNonNull(resourceSets);
     }
 
@@ -43,6 +50,11 @@ public boolean exists(String suffix, String ext) {
         return exists(DataSourceUtil.getFileName(baseName, suffix, ext));
     }
 
+    @Override
+    public boolean existsStrict(String suffix, String ext) {
+        return (mainExtension.isEmpty() || mainExtension.equals(ext)) && exists(suffix, ext);
+    }
+
     @Override
     public boolean exists(String fileName) {
         Objects.requireNonNull(fileName);

diff --git a/commons/src/main/java/com/powsybl/commons/datasource/XZFileDataSource.java b/commons/src/main/java/com/powsybl/commons/datasource/XZFileDataSource.java
@@ -18,6 +18,10 @@
  */
 public class XZFileDataSource extends FileDataSource {
 
+    public XZFileDataSource(Path directory, String baseName, String mainExtension, DataSourceObserver observer) {
+        super(directory, baseName, mainExtension, observer);
+    }
+
     public XZFileDataSource(Path directory, String baseName, DataSourceObserver observer) {
         super(directory, baseName, observer);
     }