Optimise the FCI L1c/netcdf_utils by introducing on-demand variables collection and caching

.github/workflows/ci.yaml

@@ -8,7 +8,7 @@ concurrency:
 on: [push, pull_request]
 
 env:
-  CACHE_NUMBER: 0
+  CACHE_NUMBER: 1
 
 jobs:
   lint:

satpy/etc/readers/fci_l1c_nc.yaml

@@ -18,7 +18,60 @@ file_types:
     file_reader: !!python/name:satpy.readers.fci_l1c_nc.FCIL1cNCFileHandler
     file_patterns: [ '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+{data_source}-1C-RRAD-FDHSI-{coverage}-{subsetting}-{component1}-BODY-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ]
     expected_segments: 40
-
+    required_netcdf_variables:
+    - attr/platform
+    - data/mtg_geos_projection
+    - data/{channel_name}/measured/start_position_row
+    - data/{channel_name}/measured/end_position_row
+    - data/{channel_name}/measured/radiance_to_bt_conversion_coefficient_wavenumber
+    - data/{channel_name}/measured/radiance_to_bt_conversion_coefficient_a
+    - data/{channel_name}/measured/radiance_to_bt_conversion_coefficient_b
+    - data/{channel_name}/measured/radiance_to_bt_conversion_constant_c1
+    - data/{channel_name}/measured/radiance_to_bt_conversion_constant_c2
+    - data/{channel_name}/measured/radiance_unit_conversion_coefficient
+    - data/{channel_name}/measured/channel_effective_solar_irradiance
+    - data/{channel_name}/measured/effective_radiance
+    - data/{channel_name}/measured/x
+    - data/{channel_name}/measured/y
+    - data/{channel_name}/measured/pixel_quality
+    - data/{channel_name}/measured/index_map
+    - data/mtg_geos_projection/attr/inverse_flattening
+    - data/mtg_geos_projection/attr/longitude_of_projection_origin
+    - data/mtg_geos_projection/attr/longitude_of_projection_origin
+    - data/mtg_geos_projection/attr/perspective_point_height
+    - data/mtg_geos_projection/attr/perspective_point_height
+    - data/mtg_geos_projection/attr/perspective_point_height
+    - data/mtg_geos_projection/attr/semi_major_axis
+    - data/swath_direction
+    - data/swath_number
+    - index
+    - state/celestial/earth_sun_distance
+    - state/celestial/earth_sun_distance
+    - state/celestial/subsolar_latitude
+    - state/celestial/subsolar_longitude
+    - state/celestial/sun_satellite_distance
+    - state/platform/platform_altitude
+    - state/platform/subsatellite_latitude
+    - state/platform/subsatellite_longitude
+    - time
+    variable_name_replacements:
+      channel_name:
+      - vis_04
+      - vis_05
+      - vis_06
+      - vis_08
+      - vis_09
+      - nir_13
+      - nir_16
+      - nir_22
+      - ir_38
+      - wv_63
+      - wv_73
+      - ir_87
+      - ir_97
+      - ir_105
+      - ir_123
+      - ir_133
 
 datasets:
   vis_04:

satpy/readers/fci_l1c_nc.py

@@ -186,7 +186,7 @@ def __init__(self, filename, filename_info, filetype_info):
         """Initialize file handler."""
         super().__init__(filename, filename_info,
                          filetype_info,
-                         cache_var_size=10000,
+                         cache_var_size=0,
                          cache_handle=True)
         logger.debug('Reading: {}'.format(self.filename))
         logger.debug('Start: {}'.format(self.start_time))
@@ -210,15 +210,15 @@ def get_segment_position_info(self):
         This is used in the GEOVariableSegmentYAMLReader to compute optimal chunk sizes for missing chunks.
         """
         segment_position_info = {
-            '1km': {'start_position_row': self['data/vis_04/measured/start_position_row'].item(),
-                    'end_position_row': self['data/vis_04/measured/end_position_row'].item(),
-                    'segment_height': self['data/vis_04/measured/end_position_row'].item() -
-                    self['data/vis_04/measured/start_position_row'].item() + 1,
+            '1km': {'start_position_row': self.get_and_cache_npxr("data/vis_04/measured/start_position_row").item(),
+                    'end_position_row': self.get_and_cache_npxr('data/vis_04/measured/end_position_row').item(),
+                    'segment_height': self.get_and_cache_npxr('data/vis_04/measured/end_position_row').item() -
+                    self.get_and_cache_npxr('data/vis_04/measured/start_position_row').item() + 1,
                     'segment_width': 11136},
-            '2km': {'start_position_row': self['data/ir_105/measured/start_position_row'].item(),
-                    'end_position_row': self['data/ir_105/measured/end_position_row'].item(),
-                    'segment_height': self['data/ir_105/measured/end_position_row'].item() -
-                    self['data/ir_105/measured/start_position_row'].item() + 1,
+            '2km': {'start_position_row': self.get_and_cache_npxr('data/ir_105/measured/start_position_row').item(),
+                    'end_position_row': self.get_and_cache_npxr('data/ir_105/measured/end_position_row').item(),
+                    'segment_height': self.get_and_cache_npxr('data/ir_105/measured/end_position_row').item() -
+                    self.get_and_cache_npxr('data/ir_105/measured/start_position_row').item() + 1,
                     'segment_width': 5568}
         }
 
@@ -296,7 +296,7 @@ def _get_dataset_measurand(self, key, info=None):
         res.attrs.update(attrs)
 
         res.attrs["platform_name"] = self._platform_name_translate.get(
-            self["/attr/platform"], self["/attr/platform"])
+            self["attr/platform"], self["attr/platform"])
 
         # remove unpacking parameters for calibrated data
         if key['calibration'] in ['brightness_temperature', 'reflectance']:
@@ -318,10 +318,10 @@ def orbital_param(self):
         actual_subsat_lon = float(np.nanmean(self._get_aux_data_lut_vector('subsatellite_longitude')))
         actual_subsat_lat = float(np.nanmean(self._get_aux_data_lut_vector('subsatellite_latitude')))
         actual_sat_alt = float(np.nanmean(self._get_aux_data_lut_vector('platform_altitude')))
-
-        nominal_and_proj_subsat_lon = float(self["data/mtg_geos_projection/attr/longitude_of_projection_origin"])
+        mtg_geos_proj = self.get_and_cache_npxr("data/mtg_geos_projection")
+        nominal_and_proj_subsat_lon = float(mtg_geos_proj.longitude_of_projection_origin)
         nominal_and_proj_subsat_lat = 0
-        nominal_and_proj_sat_alt = float(self["data/mtg_geos_projection/attr/perspective_point_height"])
+        nominal_and_proj_sat_alt = float(mtg_geos_proj.perspective_point_height)
 
         orb_param_dict = {
             'orbital_parameters': {
@@ -356,7 +356,7 @@ def _get_dataset_index_map(self, dsname):
 
     def _get_aux_data_lut_vector(self, aux_data_name):
         """Load the lut vector of an auxiliary variable."""
-        lut = self[AUX_DATA[aux_data_name]]
+        lut = self.get_and_cache_npxr(AUX_DATA[aux_data_name])
 
         fv = default_fillvals.get(lut.dtype.str[1:], np.nan)
         lut = lut.where(lut != fv)
@@ -372,7 +372,7 @@ def _get_dataset_aux_data(self, dsname):
         # get index map
         index_map = self._get_dataset_index_map(_get_channel_name_from_dsname(dsname))
         # subtract minimum of index variable (index_offset)
-        index_map -= np.min(self['index'])
+        index_map -= np.min(self.get_and_cache_npxr('index'))
 
         # get lut values from 1-d vector variable
         lut = self._get_aux_data_lut_vector(_get_aux_data_name_from_dsname(dsname))
@@ -408,11 +408,12 @@ def calc_area_extent(self, key):
         logger.debug('Row/Cols: {} / {}'.format(nlines, ncols))
 
         # Calculate full globe line extent
-        h = float(self["data/mtg_geos_projection/attr/perspective_point_height"])
+        mtg_geos_proj = self.get_and_cache_npxr("data/mtg_geos_projection")
+        h = float(mtg_geos_proj.perspective_point_height)
 
         extents = {}
         for coord in "xy":
-            coord_radian = self["data/{:s}/measured/{:s}".format(channel_name, coord)]
+            coord_radian = self.get_and_cache_npxr("data/{:s}/measured/{:s}".format(channel_name, coord))
 
             # TODO remove this check when old versions of IDPF test data (<v4) are deprecated.
             if coord == "x" and coord_radian.scale_factor > 0:
@@ -475,11 +476,12 @@ def get_area_def(self, key):
         if key['resolution'] in self._cache:
             return self._cache[key['resolution']]
 
-        a = float(self["data/mtg_geos_projection/attr/semi_major_axis"])
-        h = float(self["data/mtg_geos_projection/attr/perspective_point_height"])
-        rf = float(self["data/mtg_geos_projection/attr/inverse_flattening"])
-        lon_0 = float(self["data/mtg_geos_projection/attr/longitude_of_projection_origin"])
-        sweep = str(self["data/mtg_geos_projection"].sweep_angle_axis)
+        mtg_geos_proj = self.get_and_cache_npxr("data/mtg_geos_projection")
+        a = float(mtg_geos_proj.semi_major_axis)
+        h = float(mtg_geos_proj.perspective_point_height)
+        rf = float(mtg_geos_proj.inverse_flattening)
+        lon_0 = float(mtg_geos_proj.longitude_of_projection_origin)
+        sweep = str(mtg_geos_proj.sweep_angle_axis)
 
         area_extent, nlines, ncols = self.calc_area_extent(key)
         logger.debug('Calculated area extent: {}'
@@ -552,8 +554,8 @@ def calibrate_counts_to_rad(self, data, key):
                     data.attrs.get("add_offset", 0))
 
         measured = self.get_channel_measured_group_path(key['name'])
-        data.attrs.update({'radiance_unit_conversion_coefficient': self[measured +
-                                                                        '/radiance_unit_conversion_coefficient']})
+        data.attrs.update({'radiance_unit_conversion_coefficient':
+                          self.get_and_cache_npxr(measured + '/radiance_unit_conversion_coefficient')})
         return data
 
     def calibrate_rad_to_bt(self, radiance, key):
@@ -562,13 +564,13 @@ def calibrate_rad_to_bt(self, radiance, key):
 
         measured = self.get_channel_measured_group_path(key['name'])
 
-        vc = self[measured + "/radiance_to_bt_conversion_coefficient_wavenumber"]
+        vc = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_coefficient_wavenumber")
 
-        a = self[measured + "/radiance_to_bt_conversion_coefficient_a"]
-        b = self[measured + "/radiance_to_bt_conversion_coefficient_b"]
+        a = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_coefficient_a")
+        b = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_coefficient_b")
 
-        c1 = self[measured + "/radiance_to_bt_conversion_constant_c1"]
-        c2 = self[measured + "/radiance_to_bt_conversion_constant_c2"]
+        c1 = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_constant_c1")
+        c2 = self.get_and_cache_npxr(measured + "/radiance_to_bt_conversion_constant_c2")
 
         for v in (vc, a, b, c1, c2):
             if v == v.attrs.get("FillValue",
@@ -591,7 +593,7 @@ def calibrate_rad_to_refl(self, radiance, key):
         """VIS channel calibration."""
         measured = self.get_channel_measured_group_path(key['name'])
 
-        cesi = self[measured + "/channel_effective_solar_irradiance"]
+        cesi = self.get_and_cache_npxr(measured + "/channel_effective_solar_irradiance")
 
         if cesi == cesi.attrs.get(
                 "FillValue", default_fillvals.get(cesi.dtype.str[1:])):
@@ -600,7 +602,8 @@ def calibrate_rad_to_refl(self, radiance, key):
                 "cannot produce reflectance for {:s}.".format(measured))
             return radiance * np.nan
 
-        sun_earth_distance = np.mean(self["state/celestial/earth_sun_distance"]) / 149597870.7  # [AU]
+        sun_earth_distance = np.mean(
+            self.get_and_cache_npxr("state/celestial/earth_sun_distance")) / 149597870.7  # [AU]
 
         # TODO remove this check when old versions of IDPF test data (<v5) are deprecated.
         if sun_earth_distance < 0.9 or sun_earth_distance > 1.1: