Added some tests for legacy file information (#548)

* Added some tests for legacy file information
* Improved handling of wrong file input

pde/storage/movie.py

@@ -46,6 +46,9 @@ class MovieStorage(StorageBase):
     lossless compression for various configurations. Not all video players support this
     codec, but `VLC <https://www.videolan.org>`_ usually works quite well.
 
+    Note that important metainformation is stored as a comment in the movie, so this
+    data must not be deleted or altered if the video should be read again.
+
     Warning:
         This storage potentially compresses data and can thus lead to loss of some
         information. The data quality depends on many parameters, but most important are
@@ -162,9 +165,12 @@ def _get_metadata(self) -> str:
 
     def _read_metadata(self) -> None:
         """read metadata from video and store it in :attr:`info`"""
-        import ffmpeg
+        import ffmpeg  # lazy loading so it's not a hard dependence
 
-        info = ffmpeg.probe(self.filename)
+        path = Path(self.filename)
+        if not path.exists():
+            raise OSError(f"File `{path}` does not exist")
+        info = ffmpeg.probe(path)
 
         # sanity checks on the video
         nb_streams = info["format"]["nb_streams"]
@@ -253,7 +259,7 @@ def start_writing(self, field: FieldBase, info: InfoDict | None = None) -> None:
             info (dict):
                 Supplies extra information that is stored in the storage
         """
-        import ffmpeg
+        import ffmpeg  # lazy loading so it's not a hard dependence
 
         if self._is_writing:
             raise RuntimeError(f"{self.__class__.__name__} is already in writing mode")
@@ -331,6 +337,7 @@ def start_writing(self, field: FieldBase, info: InfoDict | None = None) -> None:
         self._ffmpeg = f_output.run_async(pipe_stdin=True)  # start process
 
         self.info["num_frames"] = 0
+        self._warned_normalization = False
         self._state = "writing"
 
     def _append_data(self, data: np.ndarray, time: float) -> None:
@@ -383,6 +390,13 @@ def _append_data(self, data: np.ndarray, time: float) -> None:
         # map data values to frame values
         frame_data = np.zeros(self._frame_shape, dtype=self._format.dtype)
         for i, norm in enumerate(self._norms):
+            if not self._warned_normalization:
+                if np.any(data[i, ...] < norm.vmin) or np.any(data[i, ...] > norm.vmax):
+                    self._logger.warning(
+                        f"Data outside range specified by `vmin={norm.vmin}` and "
+                        f"`vmax={norm.vmax}`"
+                    )
+                self._warned_normalization = True  # only warn once
             data_norm = norm(data[i, ...])
             frame_data[..., i] = self._format.data_to_frame(data_norm)
 
@@ -407,6 +421,8 @@ def __len__(self):
     @property
     def times(self):
         """:class:`~numpy.ndarray`: The times at which data is available"""
+        if "video_format" not in self.info:
+            self._read_metadata()
         t_start = self.info.get("t_start")
         if t_start is None:
             t_start = 0
@@ -431,7 +447,7 @@ def _get_field(self, t_index: int) -> FieldBase:
             :class:`~pde.fields.FieldBase`:
             The field class containing the grid and data
         """
-        import ffmpeg
+        import ffmpeg  # lazy loading so it's not a hard dependence
 
         if t_index < 0:
             t_index += len(self)
@@ -473,7 +489,7 @@ def _get_field(self, t_index: int) -> FieldBase:
 
     def __iter__(self) -> Iterator[FieldBase]:
         """iterate over all stored fields"""
-        import ffmpeg
+        import ffmpeg  # lazy loading so it's not a hard dependence
 
         if "width" not in self.info:
             self._read_metadata()

scripts/create_storage_test_resources.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""
+This script creates storage files for backwards compatibility tests
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+PACKAGE_PATH = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(PACKAGE_PATH))
+
+import pde
+
+
+def create_storage_test_resources(path, num):
+    """test storing scalar field as movie"""
+    grid = pde.CylindricalSymGrid(3, [1, 2], [2, 2])
+    field = pde.ScalarField(grid, [[1, 3], [2, 4]])
+    eq = pde.DiffusionPDE()
+    info = {"payload": "storage-test"}
+    movie_writer = pde.MovieStorage(
+        path / f"storage_{num}.avi", info=info, vmax=4, bits_per_channel=16
+    )
+    file_writer = pde.FileStorage(path / f"storage_{num}.hdf5", info=info)
+    eq.solve(
+        field,
+        t_range=3.5,
+        dt=0.1,
+        backend="numpy",
+        tracker=[movie_writer.tracker(2), file_writer.tracker(2)],
+    )
+
+
+def main():
+    """main function creating all the requirements"""
+    root = Path(PACKAGE_PATH)
+    create_storage_test_resources(root / "tests" / "storage" / "resources", 1)
+
+
+if __name__ == "__main__":
+    main()

tests/storage/test_movie_storages.py

@@ -2,14 +2,18 @@
 .. codeauthor:: David Zwicker <david.zwicker@ds.mpg.de>
 """
 
+from pathlib import Path
+
 import numpy as np
 import pytest
 
 import pde
-from pde import MovieStorage
+from pde import FileStorage, MovieStorage
 from pde.tools.ffmpeg import formats
 from pde.tools.misc import module_available
 
+RESOURCES_PATH = Path(__file__).resolve().parent / "resources"
+
 
 @pytest.mark.skipif(not module_available("ffmpeg"), reason="requires `ffmpeg-python`")
 @pytest.mark.parametrize("dim", [1, 2])
@@ -167,3 +171,34 @@ def test_complex_data(tmp_path, rng):
         pde.DiffusionPDE().solve(
             field, t_range=3.5, backend="numpy", tracker=writer.tracker(2)
         )
+
+
+@pytest.mark.skipif(not module_available("ffmpeg"), reason="requires `ffmpeg-python`")
+def test_wrong_format():
+    """test how wrong files are dealt with"""
+    reader = MovieStorage(RESOURCES_PATH / "does_not_exist.avi")
+    with pytest.raises(OSError):
+        reader.times
+
+    reader = MovieStorage(RESOURCES_PATH / "empty.avi")
+    with pytest.raises(Exception):
+        reader.times
+
+    reader = MovieStorage(RESOURCES_PATH / "no_metadata.avi")
+    np.testing.assert_allclose(reader.times, [0, 1])
+    with pytest.raises(RuntimeError):
+        reader[0]
+
+
+@pytest.mark.skipif(not module_available("ffmpeg"), reason="requires `ffmpeg-python`")
+@pytest.mark.parametrize("path", RESOURCES_PATH.glob("*.hdf5"))
+def test_stored_files(path):
+    """test stored files"""
+    file_reader = FileStorage(path)
+    movie_reader = MovieStorage(path.with_suffix(".avi"))
+
+    np.testing.assert_allclose(file_reader.times, movie_reader.times)
+    assert file_reader.info["payload"] == movie_reader.info["payload"]
+    for a, b in zip(file_reader, movie_reader):
+        assert a.grid == b.grid
+        np.testing.assert_allclose(a.data, b.data, atol=1e-4)