Add option to choose start time to MSI SAFE reader

satpy/etc/readers/msi_safe.yaml

@@ -1,7 +1,7 @@
 reader:
   name: msi_safe
   short_name: MSI SAFE
-  long_name: Sentinel-2 A and B MSI data in SAFE format
+  long_name: Sentinel-2 A and B MSI data in SAFE format, supporting L1C format only.
   description: SAFE Reader for MSI data (Sentinel-2)
   status: Nominal
   supports_fsspec: false
@@ -10,16 +10,16 @@ reader:
   reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader
 
 file_types:
-  safe_granule:
+  safe_granule_l1c:
         file_reader: !!python/name:satpy.readers.msi_safe.SAFEMSIL1C
-        file_patterns: ['{fmission_id:3s}_MSIL1C_{observation_time:%Y%m%dT%H%M%S}_N{fprocessing_baseline_number:4d}_R{relative_orbit_number:3d}_T{dtile_number:5s}_{dproduct_discriminator:%Y%m%dT%H%M%S}.SAFE/GRANULE/L1C_T{gtile_number:5s}_A{absolute_orbit_number:6d}_{gfile_discriminator:%Y%m%dT%H%M%S}/IMG_DATA/T{tile_number:5s}_{file_discriminator:%Y%m%dT%H%M%S}_{band_name:3s}.jp2']
+        file_patterns: ['{fmission_id:3s}_MSI{proclevel:3s}_{observation_time:%Y%m%dT%H%M%S}_N{fprocessing_baseline_number:4d}_R{relative_orbit_number:3d}_T{dtile_number:5s}_{dproduct_discriminator:%Y%m%dT%H%M%S}.SAFE/GRANULE/L1C_T{gtile_number:5s}_A{absolute_orbit_number:6d}_{gfile_discriminator:%Y%m%dT%H%M%S}/IMG_DATA/T{tile_number:5s}_{file_discriminator:%Y%m%dT%H%M%S}_{band_name:3s}.jp2']
         requires: [safe_metadata, safe_tile_metadata]
   safe_tile_metadata:
         file_reader: !!python/name:satpy.readers.msi_safe.SAFEMSITileMDXML
-        file_patterns: ['{fmission_id:3s}_MSIL1C_{observation_time:%Y%m%dT%H%M%S}_N{fprocessing_baseline_number:4d}_R{relative_orbit_number:3d}_T{dtile_number:5s}_{dproduct_discriminator:%Y%m%dT%H%M%S}.SAFE/GRANULE/L1C_T{gtile_number:5s}_A{absolute_orbit_number:6d}_{gfile_discriminator:%Y%m%dT%H%M%S}/MTD_TL.xml']
+        file_patterns: ['{fmission_id:3s}_MSI{proclevel:3s}_{observation_time:%Y%m%dT%H%M%S}_N{fprocessing_baseline_number:4d}_R{relative_orbit_number:3d}_T{dtile_number:5s}_{dproduct_discriminator:%Y%m%dT%H%M%S}.SAFE/GRANULE/L1C_T{gtile_number:5s}_A{absolute_orbit_number:6d}_{gfile_discriminator:%Y%m%dT%H%M%S}/MTD_TL.xml']
   safe_metadata:
         file_reader: !!python/name:satpy.readers.msi_safe.SAFEMSIMDXML
-        file_patterns: ['{fmission_id:3s}_MSIL1C_{observation_time:%Y%m%dT%H%M%S}_N{fprocessing_baseline_number:4d}_R{relative_orbit_number:3d}_T{dtile_number:5s}_{dproduct_discriminator:%Y%m%dT%H%M%S}.SAFE/MTD_MSIL1C.xml']
+        file_patterns: ['{fmission_id:3s}_MSI{proclevel:3s}_{observation_time:%Y%m%dT%H%M%S}_N{fprocessing_baseline_number:4d}_R{relative_orbit_number:3d}_T{dtile_number:5s}_{dproduct_discriminator:%Y%m%dT%H%M%S}.SAFE/MTD_MSIL1C.xml']
 
 
 datasets:
@@ -36,7 +36,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B02:
     name: B02
@@ -50,7 +50,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B03:
     name: B03
@@ -64,7 +64,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B04:
     name: B04
@@ -78,7 +78,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B05:
     name: B05
@@ -92,7 +92,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B06:
     name: B06
@@ -106,7 +106,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B07:
     name: B07
@@ -120,7 +120,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B08:
     name: B08
@@ -134,7 +134,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B8A:
     name: B8A
@@ -148,7 +148,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B09:
     name: B09
@@ -162,7 +162,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B10:
     name: B10
@@ -176,7 +176,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B11:
     name: B11
@@ -190,7 +190,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
   B12:
     name: B12
@@ -204,7 +204,7 @@ datasets:
       radiance:
         standard_name: toa_outgoing_radiance_per_unit_wavelength
         units: W m-2 um-1 sr-1
-    file_type: safe_granule
+    file_type: safe_granule_l1c
 
 
   solar_zenith_angle:

satpy/readers/msi_safe.py

@@ -28,13 +28,25 @@
                         reader_kwargs={'mask_saturated': False})
     scene.load(['B01'])
 
-L1B format description for the files read here:
+MSI data typically have the same start time across multiple tiles, which can cause
+problems if iterating over multiple tiles, as the saved imagery from one tile
+may be overwritten by the next tile.
+To overcome this, the user can specify `use_tile_time`, which will determine the start
+time from the tile metadata rather than from the filename::
+
+    scene = satpy.Scene(filenames,
+                        reader='msi_safe',
+                        reader_kwargs={'use_tile_time': True})
+    scene.load(['B01'])
+
+L1C format description for the files read here:
 
   https://sentinels.copernicus.eu/documents/247904/0/Sentinel-2-product-specifications-document-V14-9.pdf/
 
 """
 
 import logging
+from datetime import datetime
 
 import dask.array as da
 import defusedxml.ElementTree as ET
@@ -58,18 +70,22 @@
 class SAFEMSIL1C(BaseFileHandler):
     """File handler for SAFE MSI files (jp2)."""
 
-    def __init__(self, filename, filename_info, filetype_info, mda, tile_mda, mask_saturated=True):
+    def __init__(self, filename, filename_info, filetype_info, mda, tile_mda, mask_saturated=True, use_tile_time=False):
         """Initialize the reader."""
         super(SAFEMSIL1C, self).__init__(filename, filename_info,
                                          filetype_info)
         del mask_saturated
-        self._start_time = filename_info["observation_time"]
-        self._end_time = filename_info["observation_time"]
         self._channel = filename_info["band_name"]
         self._tile_mda = tile_mda
         self._mda = mda
         self.platform_name = PLATFORMS[filename_info["fmission_id"]]
 
+        if use_tile_time:
+            self._start_time = self._tile_mda.start_time()
+        else:
+            self._start_time = filename_info["observation_time"]
+        self._end_time = filename_info["observation_time"]
+
     def get_dataset(self, key, info):
         """Load a dataset."""
         if self._channel != key["name"]:
@@ -110,7 +126,7 @@
 class SAFEMSIXMLMetadata(BaseFileHandler):
     """Base class for SAFE MSI XML metadata filehandlers."""
 
-    def __init__(self, filename, filename_info, filetype_info, mask_saturated=True):
+    def __init__(self, filename, filename_info, filetype_info, mask_saturated=True, use_tile_time=False):
         """Init the reader."""
         super().__init__(filename, filename_info, filetype_info)
         self._start_time = filename_info["observation_time"]
@@ -226,7 +242,7 @@
 class SAFEMSITileMDXML(SAFEMSIXMLMetadata):
     """File handle for sentinel 2 safe XML tile metadata."""
 
-    def __init__(self, filename, filename_info, filetype_info, mask_saturated=True):
+    def __init__(self, filename, filename_info, filetype_info, mask_saturated=True, use_tile_time=False):
         """Init the reader."""
         super().__init__(filename, filename_info, filetype_info, mask_saturated)
         self.geocoding = self.root.find(".//Tile_Geocoding")
@@ -267,6 +283,11 @@
         cols = int(self.geocoding.find('Size[@resolution="' + str(resolution) + '"]/NCOLS').text)
         return cols, rows
 
+    def start_time(self):
+        """Get the observation time from the tile metadata."""
+        timestr = self.root.find(".//SENSING_TIME").text
+        return datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S.%fZ")
+
     @staticmethod
     def _do_interp(minterp, xcoord, ycoord):
         interp_points2 = np.vstack((ycoord.ravel(), xcoord.ravel()))

satpy/tests/reader_tests/test_msi_safe.py

@@ -17,6 +17,7 @@
 # satpy.  If not, see <http://www.gnu.org/licenses/>.
 """Module for testing the satpy.readers.msi_safe module."""
 import unittest.mock as mock
+from datetime import datetime
 from io import BytesIO, StringIO
 
 import numpy as np
@@ -25,6 +26,10 @@
 
 from satpy.tests.utils import make_dataid
 
+# Datetimes used for checking start time is correctly set.
+fname_dt = datetime(2020, 10, 1, 18, 35, 41)
+tilemd_dt = datetime(2020, 10, 1, 16, 34, 23, 153611)
+
 mtd_tile_xml = b"""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <n1:Level-1C_Tile_ID xmlns:n1="https://psd-14.sentinel2.eo.esa.int/PSD/S2_PDI_Level-1C_Tile_Metadata.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://psd-14.sentinel2.eo.esa.int/PSD/S2_PDI_Level-1C_Tile_Metadata.xsd /gpfs/dpc/app/s2ipf/FORMAT_METADATA_TILE_L1C/02.14.00/scripts/../../../schemas/02.17.00/PSD/S2_PDI_Level-1C_Tile_Metadata.xsd">
 
@@ -873,6 +878,10 @@ def setup_method(self):
         self.old_xml_fh = SAFEMSIMDXML(StringIO(mtd_l1c_old_xml), filename_info, mock.MagicMock())
         self.xml_fh = SAFEMSIMDXML(StringIO(mtd_l1c_xml), filename_info, mock.MagicMock(), mask_saturated=True)
 
+    def test_start_time(self):
+        """Ensure start time is read correctly from XML."""
+        assert self.xml_tile_fh.start_time() == tilemd_dt
+
     def test_satellite_zenith_array(self):
         """Test reading the satellite zenith array."""
         info = dict(xml_tag="Viewing_Incidence_Angles_Grids", xml_item="Zenith")
@@ -971,10 +980,11 @@ class TestSAFEMSIL1C:
     def setup_method(self):
         """Set up the test."""
         from satpy.readers.msi_safe import SAFEMSITileMDXML
-        self.filename_info = dict(observation_time=None, fmission_id="S2A", band_name="B01", dtile_number=None)
+        self.filename_info = dict(observation_time=fname_dt, fmission_id="S2A", band_name="B01", dtile_number=None)
         self.fake_data = xr.Dataset({"band_data": xr.DataArray([[[0, 1], [65534, 65535]]], dims=["band", "x", "y"])})
         self.tile_mda = mock.create_autospec(SAFEMSITileMDXML)(BytesIO(mtd_tile_xml),
                                                                self.filename_info, mock.MagicMock())
+        self.tile_mda.start_time.return_value = tilemd_dt
 
     @pytest.mark.parametrize(("mask_saturated", "calibration", "expected"),
                              [(True, "reflectance", [[np.nan, 0.01 - 10], [645.34, np.inf]]),
@@ -991,3 +1001,15 @@ def test_calibration_and_masking(self, mask_saturated, calibration, expected):
         with mock.patch("xarray.open_dataset", return_value=self.fake_data):
             res = self.jp2_fh.get_dataset(make_dataid(name="B01", calibration=calibration), info=dict())
             np.testing.assert_allclose(res, expected)
+
+    @pytest.mark.parametrize(("use_obs_time", "expected"),
+                             [(True, tilemd_dt),
+                              (False, fname_dt)])
+    def test_start_time(self, use_obs_time, expected):
+        """Test that the correct start time is returned."""
+        from satpy.readers.msi_safe import SAFEMSIL1C, SAFEMSIMDXML
+
+        mda = SAFEMSIMDXML(StringIO(mtd_l1c_xml), self.filename_info, mock.MagicMock())
+        self.jp2_fh = SAFEMSIL1C("somefile", self.filename_info, mock.MagicMock(),
+                                 mda, self.tile_mda, use_tile_time=use_obs_time)
+        assert expected == self.jp2_fh.start_time