Add support for SEVIRI Native files without archive header

satpy/readers/seviri_l1b_native.py

@@ -44,8 +44,10 @@
     create_coef_dict, pad_data_horizontally,
     pad_data_vertically
 )
-from satpy.readers.seviri_l1b_native_hdr import (GSDTRecords, native_header,
-                                                 native_trailer)
+from satpy.readers.seviri_l1b_native_hdr import (
+    GSDTRecords, get_native_header, native_trailer,
+    DEFAULT_15_SECONDARY_PRODUCT_HEADER
+)
 from satpy.readers._geos_area import get_area_definition, get_geos_area_naming
 
 logger = logging.getLogger('native_msg')
@@ -88,11 +90,18 @@ def __init__(self, filename, filename_info, filetype_info,
 
         # Read header, prepare dask-array, read trailer and initialize image boundaries
         # Available channels are known only after the header has been read
+        self.header_type = get_native_header(self._has_archive_header())
         self._read_header()
         self.dask_array = da.from_array(self._get_memmap(), chunks=(CHUNK_SIZE,))
         self._read_trailer()
         self.image_boundaries = ImageBoundaries(self.header, self.trailer, self.mda)
 
+    def _has_archive_header(self):
+        """Check whether the file includes an ASCII archive header."""
+        ascii_startswith = b'FormatName                  : NATIVE'
+        with open(self.filename, mode='rb') as istream:
+            return istream.read(36) == ascii_startswith
+
     @property
     def start_time(self):
         """Read the repeat cycle start time from metadata."""
@@ -160,7 +169,7 @@ def _get_memmap(self):
         """Get the memory map for the SEVIRI data."""
         with open(self.filename) as fp:
             data_dtype = self._get_data_dtype()
-            hdr_size = native_header.itemsize
+            hdr_size = self.header_type.itemsize
 
             return np.memmap(fp, dtype=data_dtype,
                              shape=(self.mda['number_of_lines'],),
@@ -169,10 +178,15 @@ def _get_memmap(self):
     def _read_header(self):
         """Read the header info."""
         data = np.fromfile(self.filename,
-                           dtype=native_header, count=1)
+                           dtype=self.header_type, count=1)
 
         self.header.update(recarray2dict(data))
 
+        if '15_SECONDARY_PRODUCT_HEADER' not in self.header:
+            # No archive header, that means we have a complete file
+            # including all channels.
+            self.header['15_SECONDARY_PRODUCT_HEADER'] = DEFAULT_15_SECONDARY_PRODUCT_HEADER
+
         data15hd = self.header['15_DATA_HEADER']
         sec15hd = self.header['15_SECONDARY_PRODUCT_HEADER']
 
@@ -249,7 +263,7 @@ def _read_header(self):
 
     def _read_trailer(self):
 
-        hdr_size = native_header.itemsize
+        hdr_size = self.header_type.itemsize
         data_size = (self._get_data_dtype().itemsize *
                      self.mda['number_of_lines'])
 

satpy/readers/seviri_l1b_native_hdr.py

@@ -21,6 +21,9 @@
 
 from satpy.readers.eum_base import (time_cds_short, time_cds,
                                     time_cds_expanded)
+from satpy.readers.seviri_base import (
+    VISIR_NUM_LINES, VISIR_NUM_COLUMNS, HRV_NUM_COLUMNS, HRV_NUM_LINES
+)
 
 
 class GSDTRecords(object):
@@ -76,13 +79,18 @@ class GSDTRecords(object):
 class Msg15NativeHeaderRecord(object):
     """SEVIRI Level 1.5 header for native-format."""
 
-    def get(self):
-        """Get header data."""
-        # 450400 bytes
-        record = [
-            ('15_MAIN_PRODUCT_HEADER', L15MainProductHeaderRecord().get()),
-            ('15_SECONDARY_PRODUCT_HEADER',
-             L15SecondaryProductHeaderRecord().get()),
+    def get(self, with_archive_header):
+        """Get the header type."""
+        # 450400 bytes including archive header
+        # 445286 bytes excluding archive header
+        record = []
+        if with_archive_header:
+            record += [
+                ('15_MAIN_PRODUCT_HEADER', L15MainProductHeaderRecord().get()),
+                ('15_SECONDARY_PRODUCT_HEADER',
+                 L15SecondaryProductHeaderRecord().get()),
+            ]
+        record += [
             ('GP_PK_HEADER', GSDTRecords.gp_pk_header),
             ('GP_PK_SH1', GSDTRecords.gp_pk_sh1),
             ('15_DATA_HEADER', L15DataHeaderRecord().get())
@@ -1020,10 +1028,33 @@ def get(self):
         return np.dtype(record).newbyteorder('>')
 
 
+def get_native_header(with_archive_header=True):
+    """Get Native format header type.
+
+    There are two variants, one including an ASCII archive header and one
+    without that header. The header is prepended if the data are ordered
+    through the EUMETSAT data center.
+    """
+    return Msg15NativeHeaderRecord().get(with_archive_header)
+
+
+DEFAULT_15_SECONDARY_PRODUCT_HEADER = {
+    'NorthLineSelectedRectangle': {'Value': VISIR_NUM_LINES},
+    'SouthLineSelectedRectangle': {'Value': 1},
+    'EastColumnSelectedRectangle': {'Value': 1},
+    'WestColumnSelectedRectangle': {'Value': VISIR_NUM_COLUMNS},
+    'NumberColumnsVISIR': {'Value': VISIR_NUM_COLUMNS},
+    'NumberLinesVISIR': {'Value': VISIR_NUM_LINES},
+    'NumberColumnsHRV': {'Value': HRV_NUM_COLUMNS},
+    'NumberLinesHRV': {'Value': HRV_NUM_LINES},
+    'SelectedBandIDs': {'Value': 'XXXXXXXXXXXX'}
+}
+"""Default secondary product header for files containing all channels."""
+
+
 hrit_epilogue = np.dtype(
     Msg15NativeTrailerRecord().seviri_l15_trailer).newbyteorder('>')
 hrit_prologue = HritPrologue().get()
 impf_configuration = np.dtype(
     L15DataHeaderRecord().impf_configuration).newbyteorder('>')
-native_header = Msg15NativeHeaderRecord().get()
 native_trailer = Msg15NativeTrailerRecord().get()

satpy/tests/reader_tests/test_seviri_l1b_native.py

@@ -665,19 +665,23 @@ def prepare_area_defs(self, test_dict):
         trailer = self.create_test_trailer(is_rapid_scan)
         expected_area_def = test_dict['expected_area_def']
 
-        with mock.patch('satpy.readers.seviri_l1b_native.np.fromfile') as fromfile:
+        with mock.patch('satpy.readers.seviri_l1b_native.np.fromfile') as fromfile, \
+                mock.patch('satpy.readers.seviri_l1b_native.recarray2dict') as recarray2dict, \
+                mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap') as _get_memmap, \
+                mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer'), \
+                mock.patch(
+                    'satpy.readers.seviri_l1b_native.NativeMSGFileHandler._has_archive_header'
+                ) as _has_archive_header:
+            _has_archive_header.return_value = True
             fromfile.return_value = header
-            with mock.patch('satpy.readers.seviri_l1b_native.recarray2dict') as recarray2dict:
-                recarray2dict.side_effect = (lambda x: x)
-                with mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap') as _get_memmap:
-                    _get_memmap.return_value = np.arange(3)
-                    with mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer'):
-                        fh = NativeMSGFileHandler(None, {}, None)
-                        fh.fill_disk = fill_disk
-                        fh.header = header
-                        fh.trailer = trailer
-                        fh.image_boundaries = ImageBoundaries(header, trailer, fh.mda)
-                        calc_area_def = fh.get_area_def(dataset_id)
+            recarray2dict.side_effect = (lambda x: x)
+            _get_memmap.return_value = np.arange(3)
+            fh = NativeMSGFileHandler(None, {}, None)
+            fh.fill_disk = fill_disk
+            fh.header = header
+            fh.trailer = trailer
+            fh.image_boundaries = ImageBoundaries(header, trailer, fh.mda)
+            calc_area_def = fh.get_area_def(dataset_id)
 
         return (calc_area_def, expected_area_def)
 
@@ -954,17 +958,21 @@ def prepare_is_roi(self, test_dict):
         trailer = self.create_test_trailer(is_rapid_scan)
         expected_is_roi = test_dict['is_roi']
 
-        with mock.patch('satpy.readers.seviri_l1b_native.np.fromfile') as fromfile:
+        with mock.patch('satpy.readers.seviri_l1b_native.np.fromfile') as fromfile, \
+                mock.patch('satpy.readers.seviri_l1b_native.recarray2dict') as recarray2dict, \
+                mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap') as _get_memmap, \
+                mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer'), \
+                mock.patch(
+                    'satpy.readers.seviri_l1b_native.NativeMSGFileHandler._has_archive_header'
+                ) as _has_archive_header:
+            _has_archive_header.return_value = True
             fromfile.return_value = header
-            with mock.patch('satpy.readers.seviri_l1b_native.recarray2dict') as recarray2dict:
-                recarray2dict.side_effect = (lambda x: x)
-                with mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap') as _get_memmap:
-                    _get_memmap.return_value = np.arange(3)
-                    with mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer'):
-                        fh = NativeMSGFileHandler(None, {}, None)
-                        fh.header = header
-                        fh.trailer = trailer
-                        calc_is_roi = fh.is_roi()
+            recarray2dict.side_effect = (lambda x: x)
+            _get_memmap.return_value = np.arange(3)
+            fh = NativeMSGFileHandler(None, {}, None)
+            fh.header = header
+            fh.trailer = trailer
+            calc_is_roi = fh.is_roi()
 
         return (calc_is_roi, expected_is_roi)
 
@@ -1131,3 +1139,33 @@ def test_file_pattern(self, reader):
         ]
         files = reader.select_files_from_pathnames(filenames)
         assert len(files) == 4
+
+
+@pytest.mark.parametrize(
+    'file_content,exp_header_size',
+    [
+        (b'FormatName                  : NATIVE', 450400),  # with ascii header
+        (b'foobar', 445286),  # without ascii header
+    ]
+)
+def test_header_type(file_content, exp_header_size):
+    """Test identification of the file header type."""
+    header = TestNativeMSGArea.create_test_header(
+        dataset_id=make_dataid(name='VIS006', resolution=3000),
+        earth_model=1,
+        is_full_disk=True,
+        is_rapid_scan=0
+    )
+    if file_content == b'foobar':
+        header.pop('15_SECONDARY_PRODUCT_HEADER')
+    with mock.patch('satpy.readers.seviri_l1b_native.np.fromfile') as fromfile, \
+            mock.patch('satpy.readers.seviri_l1b_native.recarray2dict') as recarray2dict, \
+            mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap') as _get_memmap, \
+            mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer'), \
+            mock.patch("builtins.open", mock.mock_open(read_data=file_content)):
+        fromfile.return_value = header
+        recarray2dict.side_effect = (lambda x: x)
+        _get_memmap.return_value = np.arange(3)
+        fh = NativeMSGFileHandler('myfile', {}, None)
+        assert fh.header_type.itemsize == exp_header_size
+        assert '15_SECONDARY_PRODUCT_HEADER' in fh.header