From 3bf992501a67898557539efa65cf4ba6e04316d8 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Wed, 2 Dec 2020 16:42:59 +0100 Subject: [PATCH 01/13] Updates to FCI reader to include CT, CTTH, GII and the latest filename formats for FCI L2 PF products --- satpy/etc/readers/fci_l2_nc.yaml | 183 +++++++++++++++++++++++++++- satpy/readers/fci_l2_nc.py | 203 ++++++++++++++++++++++++++----- 2 files changed, 350 insertions(+), 36 deletions(-) diff --git a/satpy/etc/readers/fci_l2_nc.yaml b/satpy/etc/readers/fci_l2_nc.yaml index d4b8c65b1e..55a6f13b20 100644 --- a/satpy/etc/readers/fci_l2_nc.yaml +++ b/satpy/etc/readers/fci_l2_nc.yaml @@ -8,16 +8,38 @@ reader: file_types: # EUMETSAT MTG FCI L2 Optimal Cloud Analysis files in NetCDF4 format + # Filename examples + # FCI_SIM_OCA_2L_2KM_{creation_time:%Y%m%d}_1700.nc + # W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-2-ASR--FD------NC4E_C_EUMT_20201105031219_L2PF_DEV_20170410171000_20170410172000_N__T_0104_0000.nc + nc_fci_oca: file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCFileHandler - # TODO: Pattern based on the available test files, not compatible with MTG GFS definitions - file_patterns: ['FCI_SIM_OCA_2L_2KM_{creation_time:%Y%m%d}_1700.nc'] + file_patterns: ['FCI_SIM_OCA_2L_2KM_{creation_time:%Y%m%d}_1700.nc', + 'W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-OCA--{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc'] - # EUMETSAT MTG FCI L2 Cloud Mask files in NetCDF4 format nc_fci_clm: file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCFileHandler + file_patterns: ['FCI_SIM_CLM_2KM_{creation_time:%Y%m%d}_1700.nc', + 'W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-CLM--{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc'] + + nc_fci_ct: + file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCFileHandler + file_patterns: ['W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-CT--{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc'] + + nc_fci_cloud: + file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCFileHandler + file_patterns: ['W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-CTTH--{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc' ] + + nc_fci_asr: + file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCFileHandler + file_patterns: [ "W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-ASR--{temp_str}_C_EUMT_{processing_time:%Y%m%d%H%M%S}_L2PF_{platform_id}_{start_time:%Y%m%d%H%M%S}Z_{end_time:%Y%m%d%H%M%S}Z_N__T_0104_0000.nc"] + + + # W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-2-GII--FD------NC4E_C_EUMT_20201105030947_L2PF_DEV_20170410171000_20170410172000_N__T_0104_0000.nc + nc_fci_gii: + file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCSegmentFileHandler # TODO: Pattern based on the available test files, not compatible with MTG GFS definitions - file_patterns: ['FCI_SIM_CLM_2KM_{creation_time:%Y%m%d}_1700.nc'] + file_patterns: ["W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-GII--{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc"] datasets: @@ -166,3 +188,156 @@ datasets: standard_name: quality_index fill_value: -999 mask_value: 0 + + latitude: + name: latitude + file_key: 'latitude' +# resolution: + file_type: [nc_fci_gii] + standard_name: latitude + fill_value: -32767 + mask_value: -32767 + units: degree_north + + + longitude: + name: longitude + file_key: 'longitude' +# resolution: + file_type: [nc_fci_gii] + standard_name: longitude + fill_value: -32767 + mask_value: -32767 + units: degree_east + + + # GII + k_index: + name: k_index + file_type: nc_fci_gii + file_key: k_index + standard_name: k_index + fill_value: -32767 + mask_value: -32767 + coordinates: + - longitude + - latitude + + lifted_index: + name: lifted_index + file_type: nc_fci_gii + file_key: lifted_index + standard_name: lifted_index + fill_value: -32767 + mask_value: -32767 + coordinates: + - longitude + - latitude + + percent_cloud_free: + name: percent_cloud_free + file_type: nc_fci_gii + file_key: percent_cloud_free + standard_name: percent_cloud_free + fill_value: -127 + mask_value: -127 + coordinates: + - longitude + - latitude + + prec_water_high: + name: prec_water_high + file_type: nc_fci_gii + file_key: prec_water_high + standard_name: prec_water_high + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + prec_water_low: + name: prec_water_low + file_type: nc_fci_gii + file_key: prec_water_low + standard_name: prec_water_low + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + prec_water_mid: + name: prec_water_mid + file_type: nc_fci_gii + file_key: prec_water_mid + standard_name: prec_water_mid + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + prec_water_total: + name: prec_water_total + file_type: nc_fci_gii + file_key: prec_water_total + standard_name: prec_water_total + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + +# FCI CT L2 + cloud_phase: + name: cloud_phase + file_type: nc_fci_ct + file_key: cloud_phase +# standard_name: cloud_phase + fill_value: 0 + mask_value: 0 + + cloud_type: + name: cloud_type + file_type: nc_fci_ct + file_key: cloud_type +# standard_name: cloud_type + fill_value: 0 + mask_value: 0 + +# FCI CTTH Product + cloud_top_aviation_height: + name: cloud_top_aviation_height + file_type: nc_fci_cloud + file_key: cloud_top_aviation_height + fill_value: 0 + mask_value: 0 + + cloud_top_height: + name: cloud_top_height + file_type: nc_fci_cloud + file_key: cloud_top_height + fill_value: 0 + mask_value: 0 + + cloud_top_pressure: + name: cloud_top_pressure + file_type: nc_fci_th + file_key: cloud_top_pressure + fill_value: 0 + mask_value: 0 + + cloud_top_temperature: + name: cloud_top_temperature + file_type: nc_fci_cloud + file_key: cloud_top_temperature + fill_value: 0 + mask_value: 0 + + effective_cloudiness: + name: effective_cloudiness + file_type: nc_fci_cloud + file_key: effective_cloudiness + fill_value: 0 + mask_value: 0 diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index a03b807b28..d90ab5c684 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -32,6 +32,52 @@ PRODUCT_DATA_DURATION_MINUTES = 20 +SSP_DEFAULT = 0.0 + + +class CommonFunctions(object): + @staticmethod + def get_start_time(start_time_string): + """Get observation start time.""" + try: + start_time = datetime.strptime(start_time_string, '%Y%m%d%H%M%S') + except (ValueError, KeyError): + # TODO if the sensing_start_time_utc attribute is not valid, uses a hardcoded value + logger.warning("Start time cannot be obtained from file content, using default value instead") + start_time = datetime.strptime('20200101120000', '%Y%m%d%H%M%S') + + return start_time + + @staticmethod + def get_end_time(end_time_string, start_time): + """Get observation end time.""" + try: + end_time = datetime.strptime(end_time_string, '%Y%m%d%H%M%S') + except (ValueError, KeyError): + # TODO if the sensing_end_time_utc attribute is not valid, adds 20 minutes to the start time + end_time = start_time + timedelta(minutes=PRODUCT_DATA_DURATION_MINUTES) + return end_time + + @staticmethod + def get_spacecraft_name(attributes): + """Return spacecraft name.""" + try: + return attributes['platform'] + except KeyError: + # TODO if the platform attribute is not valid, return a default value + logger.warning("Spacecraft name cannot be obtained from file content, using default value instead") + return 'DEFAULT_MTG' + + @staticmethod + def get_sensor_name(attributes): + """Return instrument.""" + try: + return attributes['data_source'] + except KeyError: + # TODO if the data_source attribute is not valid, return a default value + logger.warning("Sensor cannot be obtained from file content, using default value instead") + return 'fci' + class FciL2NCFileHandler(BaseFileHandler): """Reader class for FCI L2 products in NetCDF4 format.""" @@ -61,44 +107,19 @@ def __init__(self, filename, filename_info, filetype_info): @property def start_time(self): - """Get observation start time.""" - try: - start_time = datetime.strptime(self.nc.attrs['time_coverage_start'], '%Y%m%d%H%M%S') - except (ValueError, KeyError): - # TODO if the sensing_start_time_utc attribute is not valid, uses a hardcoded value - logger.warning("Start time cannot be obtained from file content, using default value instead") - start_time = datetime.strptime('20200101120000', '%Y%m%d%H%M%S') - return start_time + return CommonFunctions.get_start_time(self.nc.attrs['time_coverage_start']) @property def end_time(self): - """Get observation end time.""" - try: - end_time = datetime.strptime(self.nc.attrs['time_coverage_end'], '%Y%m%d%H%M%S') - except (ValueError, KeyError): - # TODO if the sensing_end_time_utc attribute is not valid, adds 20 minutes to the start time - end_time = self.start_time + timedelta(minutes=PRODUCT_DATA_DURATION_MINUTES) - return end_time + return CommonFunctions.get_end_time(self.nc.attrs['time_coverage_end'], self.start_time) @property def spacecraft_name(self): - """Return spacecraft name.""" - try: - return self.nc.attrs['platform'] - except KeyError: - # TODO if the platform attribute is not valid, return a default value - logger.warning("Spacecraft name cannot be obtained from file content, using default value instead") - return 'DEFAULT_MTG' + return CommonFunctions.get_spacecraft_name(self.nc.attrs) @property def sensor(self): - """Return instrument.""" - try: - return self.nc.attrs['data_source'] - except KeyError: - # TODO if the data_source attribute is not valid, return a default value - logger.warning("Sensor cannot be obtained from file content, using default value instead") - return 'fci' + return CommonFunctions.get_sensor_name(self.nc.attrs) @property def ssp_lon(self): @@ -106,9 +127,8 @@ def ssp_lon(self): try: return float(self._projection.attrs['longitude_of_projection_origin']) except KeyError: - # TODO if the longitude_of_projection_origin attribute is not valid, return a default value logger.warning("ssp_lon cannot be obtained from file content, using default value instead") - return 0. + return SSP_DEFAULT def get_dataset(self, dataset_id, dataset_info): """Get dataset using the file_key in dataset_info.""" @@ -144,7 +164,9 @@ def get_dataset(self, dataset_id, dataset_info): variable = variable.sel(maximum_number_of_layers=layer) # Rename the dimensions as required by Satpy - variable = variable.rename({'number_of_rows': 'y', 'number_of_columns': 'x'}) +# self.col_str = "number_of_columns" +# self.row_str = "number_of_rows" + variable = variable.rename({"number_of_rows": 'y', "number_of_columns": 'x'}) # Manage the attributes of the dataset variable.attrs.setdefault('units', None) @@ -152,6 +174,7 @@ def get_dataset(self, dataset_id, dataset_info): variable.attrs.update(dataset_info) variable.attrs.update(self._get_global_attributes()) + print('dataset', variable) return variable def _get_global_attributes(self): @@ -244,3 +267,119 @@ def __del__(self): self.nc.close() except AttributeError: pass + +class FciL2NCSegmentFileHandler(BaseFileHandler): + """Reader class for FCI L2 Segmented products in NetCDF4 format.""" + + def __init__(self, filename, filename_info, filetype_info): + """Open the NetCDF file with xarray and prepare for dataset reading.""" + super().__init__(filename, filename_info, filetype_info) + + # Use xarray's default netcdf4 engine to open the file + self.nc = xr.open_dataset( + self.filename, + decode_cf=True, + mask_and_scale=True, + chunks={ + 'number_of_FoR_cols': CHUNK_SIZE, + 'number_of_FoR_rows': CHUNK_SIZE + } + ) + + # Read metadata which are common to all datasets + self.nlines = self.nc['number_of_FoR_rows'].size + self.ncols = self.nc['number_of_FoR_cols'].size + + @property + def start_time(self): + return CommonFunctions.get_start_time(self.nc.attrs['time_coverage_start']) + + @property + def end_time(self): + return CommonFunctions.get_end_time(self.nc.attrs['time_coverage_end'], self.start_time) + + @property + def spacecraft_name(self): + return CommonFunctions.get_spacecraft_name(self.nc.attrs) + + @property + def sensor(self): + return CommonFunctions.get_sensor_name(self.nc.attrs) + + def get_dataset(self, dataset_id, dataset_info): + """Get dataset using the file_key in dataset_info.""" + var_key = dataset_info['file_key'] + logger.debug('Reading in file to get dataset with key %s.', var_key) + + try: + variable = self.nc[var_key] + except KeyError: + logger.warning("Could not find key %s in NetCDF file, no valid Dataset created", var_key) + return None + + # TODO in some of the test files, invalid pixels contain the value defined as "fill_value" in the YAML file + # instead of being masked directly in the netCDF variable. + # therefore NaN is applied where such value is found or (0 if the array contains integer values) + # the next 11 lines have to be removed once the product files are correctly configured + try: + mask_value = dataset_info['mask_value'] + except KeyError: + mask_value = np.NaN + try: + fill_value = dataset_info['fill_value'] + except KeyError: + fill_value = np.NaN + float_variable = variable.where(variable != fill_value, mask_value).astype('float32', copy=False) + float_variable.attrs = variable.attrs + variable = float_variable + + # If the variable has 3 dimensions, select the required layer + if variable.ndim == 3: + layer = dataset_info.get('layer', 0) + logger.debug('Selecting the layer %d.', layer) + variable = variable.sel(maximum_number_of_layers=layer) + + # Rename the dimensions as required by Satpy + variable = variable.rename({"number_of_FoR_rows": 'y', "number_of_FoR_cols": 'x'}) + + # Manage the attributes of the dataset + variable.attrs.setdefault('units', None) + + variable.attrs.update(dataset_info) + variable.attrs.update(self._get_global_attributes()) + + return variable + + def _get_global_attributes(self): + """Create a dictionary of global attributes to be added to all datasets. + + Returns: + dict: A dictionary of global attributes. + filename: name of the product file + start_time: sensing start time from best available source + end_time: sensing end time from best available source + spacecraft_name: name of the spacecraft + ssp_lon: longitude of subsatellite point + sensor: name of sensor + creation_time: creation time of the product + platform_name: name of the platform + + """ + attributes = { + 'filename': self.filename, + 'start_time': self.start_time, + 'end_time': self.end_time, + 'spacecraft_name': self.spacecraft_name, + 'ssp_lon': SSP_DEFAULT, + 'sensor': self.sensor, + 'creation_time': self.filename_info['creation_time'], + 'platform_name': self.spacecraft_name, + } + return attributes + + def __del__(self): + """Close the NetCDF file that may still be open.""" + try: + self.nc.close() + except AttributeError: + pass From 0e21cec881ffb2a9b7ad1019fdafa1d7b4912d78 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Wed, 2 Dec 2020 17:14:04 +0100 Subject: [PATCH 02/13] Updates to FCI reader to include CT, CTTH, GII and the latest filename formats for FCI L2 PF products --- satpy/readers/fci_l2_nc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index d90ab5c684..2fc9a1856e 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -268,6 +268,7 @@ def __del__(self): except AttributeError: pass + class FciL2NCSegmentFileHandler(BaseFileHandler): """Reader class for FCI L2 Segmented products in NetCDF4 format.""" From 14b1b53dc195564bbff07439924746ae4e257367 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Thu, 3 Dec 2020 16:24:00 +0100 Subject: [PATCH 03/13] Added the ASR product and updated the code to use a Base class for common functions --- satpy/etc/readers/fci_l2_nc.yaml | 208 ++++++++++++++++++++++++++++++- satpy/readers/fci_l2_nc.py | 160 +++++++++--------------- 2 files changed, 262 insertions(+), 106 deletions(-) diff --git a/satpy/etc/readers/fci_l2_nc.yaml b/satpy/etc/readers/fci_l2_nc.yaml index 55a6f13b20..6c3faba843 100644 --- a/satpy/etc/readers/fci_l2_nc.yaml +++ b/satpy/etc/readers/fci_l2_nc.yaml @@ -7,7 +7,6 @@ reader: reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader file_types: - # EUMETSAT MTG FCI L2 Optimal Cloud Analysis files in NetCDF4 format # Filename examples # FCI_SIM_OCA_2L_2KM_{creation_time:%Y%m%d}_1700.nc # W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-2-ASR--FD------NC4E_C_EUMT_20201105031219_L2PF_DEV_20170410171000_20170410172000_N__T_0104_0000.nc @@ -31,8 +30,8 @@ file_types: file_patterns: ['W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-CTTH--{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc' ] nc_fci_asr: - file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCFileHandler - file_patterns: [ "W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-ASR--{temp_str}_C_EUMT_{processing_time:%Y%m%d%H%M%S}_L2PF_{platform_id}_{start_time:%Y%m%d%H%M%S}Z_{end_time:%Y%m%d%H%M%S}Z_N__T_0104_0000.nc"] + file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCSegmentFileHandler + file_patterns: [ "W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-ASR--{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc"] # W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-2-GII--FD------NC4E_C_EUMT_20201105030947_L2PF_DEV_20170410171000_20170410172000_N__T_0104_0000.nc @@ -193,7 +192,7 @@ datasets: name: latitude file_key: 'latitude' # resolution: - file_type: [nc_fci_gii] + file_type: [nc_fci_gii, nc_fci_asr] standard_name: latitude fill_value: -32767 mask_value: -32767 @@ -204,7 +203,7 @@ datasets: name: longitude file_key: 'longitude' # resolution: - file_type: [nc_fci_gii] + file_type: [nc_fci_gii, nc_fci_asr] standard_name: longitude fill_value: -32767 mask_value: -32767 @@ -341,3 +340,202 @@ datasets: file_key: effective_cloudiness fill_value: 0 mask_value: 0 + +# ASR + bt_max: + name: bt_max + file_type: nc_fci_asr + file_key: bt_max + standard_name: bt_max + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + bt_mean: + name: bt_mean + file_type: nc_fci_asr + file_key: bt_mean + standard_name: bt_mean + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + bt_min: + name: bt_min + file_type: nc_fci_asr + file_key: bt_min + standard_name: bt_min + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + bt_std: + name: bt_std + file_type: nc_fci_asr + file_key: bt_std + standard_name: bt_std + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + radiance_max: + name: radiance_max + file_type: nc_fci_asr + file_key: radiance_max + standard_name: radiance_max + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + radiance_mean: + name: radiance_mean + file_type: nc_fci_asr + file_key: radiance_mean + standard_name: radiance_mean + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + radiance_min: + name: radiance_min + file_type: nc_fci_asr + file_key: radiance_min + standard_name: radiance_min + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + radiance_std: + name: radiance_std + file_type: nc_fci_asr + file_key: radiance_std + standard_name: radiance_std + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + reflectance_max: + name: reflectance_max + file_type: nc_fci_asr + file_key: reflectance_max + standard_name: reflectance_max + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + reflectance_mean: + name: reflectance_mean + file_type: nc_fci_asr + file_key: reflectance_mean + standard_name: reflectance_mean + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + reflectance_min: + name: reflectance_min + file_type: nc_fci_asr + file_key: reflectance_min + standard_name: reflectance_min + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + reflectance_std: + name: reflectance_std + file_type: nc_fci_asr + file_key: reflectance_std + standard_name: reflectance_std + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + quality_bt: + name: quality_bt + file_type: nc_fci_asr + file_key: quality_bt + standard_name: quality_bt + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + quality_reflectance: + name: quality_reflectance + file_type: nc_fci_asr + file_key: quality_reflectance + standard_name: quality_reflectance + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + quality_radiance: + name: quality_radiance + file_type: nc_fci_asr + file_key: quality_radiance + standard_name: quality_radiance + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + land_pixel_percent: + name: land_pixel_percent + file_type: nc_fci_asr + file_key: land_pixel_percent + standard_name: land_pixel_percent + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + water_pixel_percent: + name: water_pixel_percent + file_type: nc_fci_asr + file_key: water_pixel_percent + standard_name: water_pixel_percent + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude + + pixel_percentage: + name: pixel_percentage + file_type: nc_fci_asr + file_key: pixel_percentage + standard_name: pixel_percentage + fill_value: 65535 + mask_value: 65535 + coordinates: + - longitude + - latitude \ No newline at end of file diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index 2fc9a1856e..d42711f0ac 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -35,51 +35,79 @@ SSP_DEFAULT = 0.0 -class CommonFunctions(object): - @staticmethod - def get_start_time(start_time_string): - """Get observation start time.""" +class FciL2BaseClass(object): + def _start_time(self): try: - start_time = datetime.strptime(start_time_string, '%Y%m%d%H%M%S') + start_time = datetime.strptime(self.nc.attrs['time_coverage_start'], '%Y%m%d%H%M%S') except (ValueError, KeyError): # TODO if the sensing_start_time_utc attribute is not valid, uses a hardcoded value logger.warning("Start time cannot be obtained from file content, using default value instead") start_time = datetime.strptime('20200101120000', '%Y%m%d%H%M%S') - return start_time - @staticmethod - def get_end_time(end_time_string, start_time): + def _end_time(self): """Get observation end time.""" try: - end_time = datetime.strptime(end_time_string, '%Y%m%d%H%M%S') + end_time = datetime.strptime(self.nc.attrs['time_coverage_end'], '%Y%m%d%H%M%S') except (ValueError, KeyError): # TODO if the sensing_end_time_utc attribute is not valid, adds 20 minutes to the start time - end_time = start_time + timedelta(minutes=PRODUCT_DATA_DURATION_MINUTES) + end_time = self.start_time + timedelta(minutes=PRODUCT_DATA_DURATION_MINUTES) return end_time - @staticmethod - def get_spacecraft_name(attributes): + def _spacecraft_name(self): """Return spacecraft name.""" try: - return attributes['platform'] + return self.nc.attrs['platform'] except KeyError: # TODO if the platform attribute is not valid, return a default value logger.warning("Spacecraft name cannot be obtained from file content, using default value instead") return 'DEFAULT_MTG' - @staticmethod - def get_sensor_name(attributes): + def _sensor_name(self): """Return instrument.""" try: - return attributes['data_source'] + return self.nc.attrs['data_source'] except KeyError: # TODO if the data_source attribute is not valid, return a default value logger.warning("Sensor cannot be obtained from file content, using default value instead") return 'fci' + def _get_global_attributes(self): + """Create a dictionary of global attributes to be added to all datasets. + + Returns: + dict: A dictionary of global attributes. + filename: name of the product file + start_time: sensing start time from best available source + end_time: sensing end time from best available source + spacecraft_name: name of the spacecraft + ssp_lon: longitude of subsatellite point + sensor: name of sensor + creation_time: creation time of the product + platform_name: name of the platform + + """ + attributes = { + 'filename': self.filename, + 'start_time': self.start_time, + 'end_time': self.end_time, + 'spacecraft_name': self.spacecraft_name, + 'ssp_lon': self.ssp_lon, + 'sensor': self.sensor, + 'creation_time': self.filename_info['creation_time'], + 'platform_name': self.spacecraft_name, + } + return attributes + + def __del__(self): + """Close the NetCDF file that may still be open.""" + try: + self.nc.close() + except AttributeError: + pass + -class FciL2NCFileHandler(BaseFileHandler): +class FciL2NCFileHandler(BaseFileHandler, FciL2BaseClass): """Reader class for FCI L2 products in NetCDF4 format.""" def __init__(self, filename, filename_info, filetype_info): @@ -107,19 +135,19 @@ def __init__(self, filename, filename_info, filetype_info): @property def start_time(self): - return CommonFunctions.get_start_time(self.nc.attrs['time_coverage_start']) + return self._start_time() @property def end_time(self): - return CommonFunctions.get_end_time(self.nc.attrs['time_coverage_end'], self.start_time) + return self._end_time() @property def spacecraft_name(self): - return CommonFunctions.get_spacecraft_name(self.nc.attrs) + return self._spacecraft_name() @property def sensor(self): - return CommonFunctions.get_sensor_name(self.nc.attrs) + return self._sensor_name() @property def ssp_lon(self): @@ -164,8 +192,6 @@ def get_dataset(self, dataset_id, dataset_info): variable = variable.sel(maximum_number_of_layers=layer) # Rename the dimensions as required by Satpy -# self.col_str = "number_of_columns" -# self.row_str = "number_of_rows" variable = variable.rename({"number_of_rows": 'y', "number_of_columns": 'x'}) # Manage the attributes of the dataset @@ -174,36 +200,8 @@ def get_dataset(self, dataset_id, dataset_info): variable.attrs.update(dataset_info) variable.attrs.update(self._get_global_attributes()) - print('dataset', variable) return variable - def _get_global_attributes(self): - """Create a dictionary of global attributes to be added to all datasets. - - Returns: - dict: A dictionary of global attributes. - filename: name of the product file - start_time: sensing start time from best available source - end_time: sensing end time from best available source - spacecraft_name: name of the spacecraft - ssp_lon: longitude of subsatellite point - sensor: name of sensor - creation_time: creation time of the product - platform_name: name of the platform - - """ - attributes = { - 'filename': self.filename, - 'start_time': self.start_time, - 'end_time': self.end_time, - 'spacecraft_name': self.spacecraft_name, - 'ssp_lon': self.ssp_lon, - 'sensor': self.sensor, - 'creation_time': self.filename_info['creation_time'], - 'platform_name': self.spacecraft_name, - } - return attributes - def get_area_def(self, key): """Return the area definition (common to all data in product).""" return self._area_def @@ -261,21 +259,13 @@ def _compute_area_def(self): return area_def - def __del__(self): - """Close the NetCDF file that may still be open.""" - try: - self.nc.close() - except AttributeError: - pass - -class FciL2NCSegmentFileHandler(BaseFileHandler): +class FciL2NCSegmentFileHandler(BaseFileHandler, FciL2BaseClass): """Reader class for FCI L2 Segmented products in NetCDF4 format.""" def __init__(self, filename, filename_info, filetype_info): """Open the NetCDF file with xarray and prepare for dataset reading.""" super().__init__(filename, filename_info, filetype_info) - # Use xarray's default netcdf4 engine to open the file self.nc = xr.open_dataset( self.filename, @@ -291,21 +281,23 @@ def __init__(self, filename, filename_info, filetype_info): self.nlines = self.nc['number_of_FoR_rows'].size self.ncols = self.nc['number_of_FoR_cols'].size + self.ssp_lon = SSP_DEFAULT + @property def start_time(self): - return CommonFunctions.get_start_time(self.nc.attrs['time_coverage_start']) + return self._start_time() @property def end_time(self): - return CommonFunctions.get_end_time(self.nc.attrs['time_coverage_end'], self.start_time) + return self._end_time() @property def spacecraft_name(self): - return CommonFunctions.get_spacecraft_name(self.nc.attrs) + return self._spacecraft_name() @property def sensor(self): - return CommonFunctions.get_sensor_name(self.nc.attrs) + return self._sensor_name() def get_dataset(self, dataset_id, dataset_info): """Get dataset using the file_key in dataset_info.""" @@ -335,52 +327,18 @@ def get_dataset(self, dataset_id, dataset_info): variable = float_variable # If the variable has 3 dimensions, select the required layer - if variable.ndim == 3: + # ASR parameters are to be ignored + if variable.ndim == 3 and dataset_info['file_type'] != 'nc_fci_asr': layer = dataset_info.get('layer', 0) logger.debug('Selecting the layer %d.', layer) variable = variable.sel(maximum_number_of_layers=layer) # Rename the dimensions as required by Satpy variable = variable.rename({"number_of_FoR_rows": 'y', "number_of_FoR_cols": 'x'}) - - # Manage the attributes of the dataset +# # Manage the attributes of the dataset variable.attrs.setdefault('units', None) variable.attrs.update(dataset_info) variable.attrs.update(self._get_global_attributes()) return variable - - def _get_global_attributes(self): - """Create a dictionary of global attributes to be added to all datasets. - - Returns: - dict: A dictionary of global attributes. - filename: name of the product file - start_time: sensing start time from best available source - end_time: sensing end time from best available source - spacecraft_name: name of the spacecraft - ssp_lon: longitude of subsatellite point - sensor: name of sensor - creation_time: creation time of the product - platform_name: name of the platform - - """ - attributes = { - 'filename': self.filename, - 'start_time': self.start_time, - 'end_time': self.end_time, - 'spacecraft_name': self.spacecraft_name, - 'ssp_lon': SSP_DEFAULT, - 'sensor': self.sensor, - 'creation_time': self.filename_info['creation_time'], - 'platform_name': self.spacecraft_name, - } - return attributes - - def __del__(self): - """Close the NetCDF file that may still be open.""" - try: - self.nc.close() - except AttributeError: - pass From 0ee35950ee843f38f01ffb6166dcf449f59cfca8 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Fri, 4 Dec 2020 11:56:03 +0100 Subject: [PATCH 04/13] Added tests for FCIL2SegmentFileHandler and removed some redundant code from FCI L2 PF reader --- satpy/readers/fci_l2_nc.py | 7 -- satpy/tests/reader_tests/test_fci_l2_nc.py | 116 ++++++++++++++++++++- 2 files changed, 115 insertions(+), 8 deletions(-) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index d42711f0ac..654d749fe5 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -326,13 +326,6 @@ def get_dataset(self, dataset_id, dataset_info): float_variable.attrs = variable.attrs variable = float_variable - # If the variable has 3 dimensions, select the required layer - # ASR parameters are to be ignored - if variable.ndim == 3 and dataset_info['file_type'] != 'nc_fci_asr': - layer = dataset_info.get('layer', 0) - logger.debug('Selecting the layer %d.', layer) - variable = variable.sel(maximum_number_of_layers=layer) - # Rename the dimensions as required by Satpy variable = variable.rename({"number_of_FoR_rows": 'y', "number_of_FoR_cols": 'x'}) # # Manage the attributes of the dataset diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index ef9660a72d..b3de0e2c19 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -23,7 +23,7 @@ import datetime from netCDF4 import Dataset -from satpy.readers.fci_l2_nc import FciL2NCFileHandler, PRODUCT_DATA_DURATION_MINUTES +from satpy.readers.fci_l2_nc import FciL2NCFileHandler, FciL2NCSegmentFileHandler, PRODUCT_DATA_DURATION_MINUTES import unittest @@ -34,6 +34,7 @@ TEST_FILE = 'test_file_fci_l2_nc.nc' +SEG_TEST_FILE = 'test_seg_file_fci_l2_nc.nc' class TestFciL2NCFileHandler(unittest.TestCase): @@ -192,3 +193,116 @@ def test_dataset(self): 'fill_value': -999, 'mask_value': 0}) # Checks that the function returns None self.assertEqual(invalid_dataset, None) + + +class TestFciL2NCSegmentFileHandler(unittest.TestCase): + """Test the FciL2NCFileHandler reader.""" + + def setUp(self): + """Set up the test by creating a test file and opening it with the reader.""" + # Easiest way to test the reader is to create a test netCDF file on the fly + with Dataset(SEG_TEST_FILE, 'w') as nc: + # Create dimensions + nc.createDimension('number_of_FoR_cols', 10) + nc.createDimension('number_of_FoR_rows', 100) + nc.createDimension('number_of_channels', 8) + nc.createDimension('number_of_categories', 6) + + # add global attributes + nc.data_source = 'test_fci_data_source' + nc.platform = 'test_fci_platform' + nc.time_coverage_start = '20170920173040' + nc.time_coverage_end = '20170920174117' + + # Add datasets + x = nc.createVariable('x', np.float32, dimensions=('number_of_FoR_cols',)) + x.standard_name = 'projection_x_coordinate' + x[:] = np.arange(10) + + y = nc.createVariable('y', np.float32, dimensions=('number_of_FoR_rows',)) + x.standard_name = 'projection_y_coordinate' + y[:] = np.arange(100) + + chans = nc.createVariable('channels', np.float32, dimensions=('number_of_channels',)) + chans.standard_name = 'fci_channels' + chans[:] = np.arange(8) + + cats = nc.createVariable('categories', np.float32, dimensions=('number_of_categories',)) + cats.standard_name = 'product_categories' + cats[:] = np.arange(6) + + test_dataset = nc.createVariable('test_values', np.float32, + dimensions=('number_of_FoR_rows', 'number_of_FoR_cols', + 'number_of_channels', 'number_of_categories')) + test_dataset[:] = np.ones((100, 10, 8, 6)) + test_dataset.test_attr = 'attr' + test_dataset.units = 'test_units' + + self.segment_reader = FciL2NCSegmentFileHandler( + filename=SEG_TEST_FILE, + filename_info={ + 'creation_time': datetime.datetime(year=2017, month=9, day=20, + hour=12, minute=30, second=30) + }, + filetype_info={} + ) + + def tearDown(self): + """Remove the previously created test file.""" + # First delete the reader, forcing the file to be closed if still open + del self.segment_reader + # Then can safely remove it from the system + try: + os.remove(SEG_TEST_FILE) + except OSError: + pass + + def test_all_basic(self): + """Test all basic functionalities.""" + self.assertEqual(PRODUCT_DATA_DURATION_MINUTES, 20) + + self.assertEqual(self.segment_reader.start_time, + datetime.datetime(year=2017, month=9, day=20, + hour=17, minute=30, second=40)) + + self.assertEqual(self.segment_reader.end_time, + datetime.datetime(year=2017, month=9, day=20, + hour=17, minute=41, second=17)) + + self.assertEqual(self.segment_reader.spacecraft_name, 'test_fci_platform') + self.assertEqual(self.segment_reader.sensor, 'test_fci_data_source') + self.assertEqual(self.segment_reader.ssp_lon, 0.0) + + global_attributes = self.segment_reader._get_global_attributes() + expected_global_attributes = { + 'filename': SEG_TEST_FILE, + 'start_time': datetime.datetime(year=2017, month=9, day=20, + hour=17, minute=30, second=40), + 'end_time': datetime.datetime(year=2017, month=9, day=20, + hour=17, minute=41, second=17), + 'spacecraft_name': 'test_fci_platform', + 'ssp_lon': 0.0, + 'sensor': 'test_fci_data_source', + 'creation_time': datetime.datetime(year=2017, month=9, day=20, + hour=12, minute=30, second=30), + 'platform_name': 'test_fci_platform' + } + self.assertEqual(global_attributes, expected_global_attributes) + + def test_dataset(self): + """Test the execution of the get_dataset function.""" + # Checks the correct execution of the get_dataset function with a valid file_key + dataset = self.segment_reader.get_dataset(None, + {'file_key': 'test_asr_values', + 'fill_value': -999, 'mask_value': 0}) + self.assertTrue(np.allclose(dataset.values, np.ones((100, 10, 8, 6)))) + self.assertEqual(dataset.attrs['test_attr'], 'attr') + self.assertEqual(dataset.attrs['units'], 'test_units') + self.assertEqual(dataset.attrs['fill_value'], -999) + + # Checks the correct execution of the get_dataset function with an invalid file_key + invalid_dataset = self.segment_reader.get_dataset(None, + {'file_key': 'test_invalid', + 'fill_value': -999, 'mask_value': 0}) + # Checks that the function returns None + self.assertEqual(invalid_dataset, None) From 3daf33f3168c5f1d1839228dfacace14ec075332 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Fri, 4 Dec 2020 12:21:13 +0100 Subject: [PATCH 05/13] Error in the test script, now fixed --- satpy/tests/reader_tests/test_fci_l2_nc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index b3de0e2c19..cf2bd8aca6 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -293,7 +293,7 @@ def test_dataset(self): """Test the execution of the get_dataset function.""" # Checks the correct execution of the get_dataset function with a valid file_key dataset = self.segment_reader.get_dataset(None, - {'file_key': 'test_asr_values', + {'file_key': 'test_values', 'fill_value': -999, 'mask_value': 0}) self.assertTrue(np.allclose(dataset.values, np.ones((100, 10, 8, 6)))) self.assertEqual(dataset.attrs['test_attr'], 'attr') From 09c4ca3710e7d9bc30f7396540a1704e1044a3a5 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Fri, 4 Dec 2020 15:26:30 +0100 Subject: [PATCH 06/13] Added tests for Exception Type handling --- satpy/tests/reader_tests/test_fci_l2_nc.py | 78 ++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index cf2bd8aca6..f67b422394 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -35,6 +35,7 @@ TEST_FILE = 'test_file_fci_l2_nc.nc' SEG_TEST_FILE = 'test_seg_file_fci_l2_nc.nc' +TEST_ERROR_FILE = 'test_error_file_fci_l2_nc.nc' class TestFciL2NCFileHandler(unittest.TestCase): @@ -274,6 +275,7 @@ def test_all_basic(self): self.assertEqual(self.segment_reader.ssp_lon, 0.0) global_attributes = self.segment_reader._get_global_attributes() + expected_global_attributes = { 'filename': SEG_TEST_FILE, 'start_time': datetime.datetime(year=2017, month=9, day=20, @@ -306,3 +308,79 @@ def test_dataset(self): 'fill_value': -999, 'mask_value': 0}) # Checks that the function returns None self.assertEqual(invalid_dataset, None) + + +class TestFciL2NCErrorFileHandler(unittest.TestCase): + """Test the FciL2NCFileHandler reader.""" + + def setUp(self): + """Set up the test by creating a test file and opening it with the reader.""" + # Easiest way to test the reader is to create a test netCDF file on the fly + + with Dataset(TEST_ERROR_FILE, 'w') as nc_err: + # Create dimensions + nc_err.createDimension('number_of_FoR_cols', 10) + nc_err.createDimension('number_of_FoR_rows', 100) + nc_err.createDimension('number_of_channels', 8) + nc_err.createDimension('number_of_categories', 6) + # add erroneous global attributes + nc_err.data_source = 'test_fci_data_source' # Error in key name + nc_err.platform_err = 'test_fci_platform' # Error in key name + nc_err.time_coverage_start = '2017092017304000' # Error in time format + nc_err.time_coverage_end_err = '20170920174117' # Error in key name + + # Add datasets + x = nc_err.createVariable('x', np.float32, dimensions=('number_of_FoR_cols',)) + x.standard_name = 'projection_x_coordinate' + x[:] = np.arange(10) + + y = nc_err.createVariable('y', np.float32, dimensions=('number_of_FoR_rows',)) + x.standard_name = 'projection_y_coordinate' + y[:] = np.arange(100) + + chans = nc_err.createVariable('channels', np.float32, dimensions=('number_of_channels',)) + chans.standard_name = 'fci_channels' + chans[:] = np.arange(8) + + cats = nc_err.createVariable('categories', np.float32, dimensions=('number_of_categories',)) + cats.standard_name = 'product_categories' + cats[:] = np.arange(6) + + test_dataset = nc_err.createVariable('test_values', np.float32, + dimensions=('number_of_FoR_rows', 'number_of_FoR_cols', + 'number_of_channels', 'number_of_categories')) + test_dataset[:] = np.ones((100, 10, 8, 6)) + test_dataset.test_attr = 'attr' + test_dataset.units = 'test_units' + + self.error_reader = FciL2NCSegmentFileHandler( + filename=TEST_ERROR_FILE, + filename_info={ + 'creation_time': datetime.datetime(year=2017, month=9, day=20, + hour=12, minute=30, second=30) + }, + filetype_info={} + ) + + def tearDown(self): + """Remove the previously created test file.""" + # First delete the reader, forcing the file to be closed if still open + del self.error_reader + # Then can safely remove it from the system + try: + os.remove(TEST_ERROR_FILE) + except OSError: + pass + + def test_errors(self): + self.assertRaises(TypeError, self.error_reader.start_time, + datetime.datetime(year=2017, month=9, day=20, + hour=17, minute=30, second=40)) + + self.assertRaises(TypeError, self.error_reader.end_time, + datetime.datetime(year=2017, month=9, day=20, + hour=17, minute=41, second=17)) + + self.assertRaises(TypeError, self.error_reader.spacecraft_name) + + self.assertRaises(TypeError, self.error_reader.sensor) From 8961e8daff9c6235d78df427698ab495ef0d7cb1 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Fri, 4 Dec 2020 15:59:16 +0100 Subject: [PATCH 07/13] Changed BaseClass to CommonFuntions --- satpy/readers/fci_l2_nc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index 654d749fe5..2dabb13753 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -35,7 +35,7 @@ SSP_DEFAULT = 0.0 -class FciL2BaseClass(object): +class FciL2CommonFunctions(object): def _start_time(self): try: start_time = datetime.strptime(self.nc.attrs['time_coverage_start'], '%Y%m%d%H%M%S') @@ -107,7 +107,7 @@ def __del__(self): pass -class FciL2NCFileHandler(BaseFileHandler, FciL2BaseClass): +class FciL2NCFileHandler(BaseFileHandler, FciL2CommonFunctions): """Reader class for FCI L2 products in NetCDF4 format.""" def __init__(self, filename, filename_info, filetype_info): @@ -260,7 +260,7 @@ def _compute_area_def(self): return area_def -class FciL2NCSegmentFileHandler(BaseFileHandler, FciL2BaseClass): +class FciL2NCSegmentFileHandler(BaseFileHandler, FciL2CommonFunctions): """Reader class for FCI L2 Segmented products in NetCDF4 format.""" def __init__(self, filename, filename_info, filetype_info): From 847ca3d97a9664fe5b2ffbfaa942f4ad61739614 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Fri, 4 Dec 2020 17:25:04 +0100 Subject: [PATCH 08/13] Individual class properites moved to CommFunctions class --- satpy/readers/fci_l2_nc.py | 48 +++++----------------- satpy/tests/reader_tests/test_fci_l2_nc.py | 27 ++++++------ 2 files changed, 24 insertions(+), 51 deletions(-) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index 2dabb13753..0455046d4e 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -36,6 +36,7 @@ class FciL2CommonFunctions(object): + @property def _start_time(self): try: start_time = datetime.strptime(self.nc.attrs['time_coverage_start'], '%Y%m%d%H%M%S') @@ -45,15 +46,17 @@ def _start_time(self): start_time = datetime.strptime('20200101120000', '%Y%m%d%H%M%S') return start_time + @property def _end_time(self): """Get observation end time.""" try: end_time = datetime.strptime(self.nc.attrs['time_coverage_end'], '%Y%m%d%H%M%S') except (ValueError, KeyError): # TODO if the sensing_end_time_utc attribute is not valid, adds 20 minutes to the start time - end_time = self.start_time + timedelta(minutes=PRODUCT_DATA_DURATION_MINUTES) + end_time = self._start_time + timedelta(minutes=PRODUCT_DATA_DURATION_MINUTES) return end_time + @property def _spacecraft_name(self): """Return spacecraft name.""" try: @@ -63,6 +66,7 @@ def _spacecraft_name(self): logger.warning("Spacecraft name cannot be obtained from file content, using default value instead") return 'DEFAULT_MTG' + @property def _sensor_name(self): """Return instrument.""" try: @@ -89,13 +93,13 @@ def _get_global_attributes(self): """ attributes = { 'filename': self.filename, - 'start_time': self.start_time, - 'end_time': self.end_time, - 'spacecraft_name': self.spacecraft_name, + 'start_time': self._start_time, + 'end_time': self._end_time, + 'spacecraft_name': self._spacecraft_name, 'ssp_lon': self.ssp_lon, - 'sensor': self.sensor, + 'sensor': self._sensor_name, 'creation_time': self.filename_info['creation_time'], - 'platform_name': self.spacecraft_name, + 'platform_name': self._spacecraft_name, } return attributes @@ -133,22 +137,6 @@ def __init__(self, filename, filename_info, filetype_info): # Compute the area definition self._area_def = self._compute_area_def() - @property - def start_time(self): - return self._start_time() - - @property - def end_time(self): - return self._end_time() - - @property - def spacecraft_name(self): - return self._spacecraft_name() - - @property - def sensor(self): - return self._sensor_name() - @property def ssp_lon(self): """Return subsatellite point longitude.""" @@ -283,22 +271,6 @@ def __init__(self, filename, filename_info, filetype_info): self.ssp_lon = SSP_DEFAULT - @property - def start_time(self): - return self._start_time() - - @property - def end_time(self): - return self._end_time() - - @property - def spacecraft_name(self): - return self._spacecraft_name() - - @property - def sensor(self): - return self._sensor_name() - def get_dataset(self, dataset_id, dataset_info): """Get dataset using the file_key in dataset_info.""" var_key = dataset_info['file_key'] diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index f67b422394..a2f5a15173 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -95,7 +95,7 @@ def setUp(self): def tearDown(self): """Remove the previously created test file.""" - # First delets the reader, forcing the file to be closed if still open + # First delete the reader, forcing the file to be closed if still open del self.reader # Then can safely remove it from the system try: @@ -106,17 +106,18 @@ def tearDown(self): def test_all_basic(self): """Test all basic functionalities.""" self.assertEqual(PRODUCT_DATA_DURATION_MINUTES, 20) + print('reader', self.reader) - self.assertEqual(self.reader.start_time, + self.assertEqual(self.reader._start_time, datetime.datetime(year=2017, month=9, day=20, hour=17, minute=30, second=40)) - self.assertEqual(self.reader.end_time, + self.assertEqual(self.reader._end_time, datetime.datetime(year=2017, month=9, day=20, hour=17, minute=41, second=17)) - self.assertEqual(self.reader.spacecraft_name, 'test_platform') - self.assertEqual(self.reader.sensor, 'test_data_source') + self.assertEqual(self.reader._spacecraft_name, 'test_platform') + self.assertEqual(self.reader._sensor_name, 'test_data_source') self.assertEqual(self.reader.ssp_lon, 10.0) global_attributes = self.reader._get_global_attributes() @@ -262,16 +263,16 @@ def test_all_basic(self): """Test all basic functionalities.""" self.assertEqual(PRODUCT_DATA_DURATION_MINUTES, 20) - self.assertEqual(self.segment_reader.start_time, + self.assertEqual(self.segment_reader._start_time, datetime.datetime(year=2017, month=9, day=20, hour=17, minute=30, second=40)) - self.assertEqual(self.segment_reader.end_time, + self.assertEqual(self.segment_reader._end_time, datetime.datetime(year=2017, month=9, day=20, hour=17, minute=41, second=17)) - self.assertEqual(self.segment_reader.spacecraft_name, 'test_fci_platform') - self.assertEqual(self.segment_reader.sensor, 'test_fci_data_source') + self.assertEqual(self.segment_reader._spacecraft_name, 'test_fci_platform') + self.assertEqual(self.segment_reader._sensor_name, 'test_fci_data_source') self.assertEqual(self.segment_reader.ssp_lon, 0.0) global_attributes = self.segment_reader._get_global_attributes() @@ -373,14 +374,14 @@ def tearDown(self): pass def test_errors(self): - self.assertRaises(TypeError, self.error_reader.start_time, + self.assertRaises(TypeError, self.error_reader._start_time, datetime.datetime(year=2017, month=9, day=20, hour=17, minute=30, second=40)) - self.assertRaises(TypeError, self.error_reader.end_time, + self.assertRaises(TypeError, self.error_reader._end_time, datetime.datetime(year=2017, month=9, day=20, hour=17, minute=41, second=17)) - self.assertRaises(TypeError, self.error_reader.spacecraft_name) + self.assertRaises(TypeError, self.error_reader._spacecraft_name) - self.assertRaises(TypeError, self.error_reader.sensor) + self.assertRaises(TypeError, self.error_reader._sensor_name) From c80eaf0b366c667cd197bf6f49be86dac7b5502b Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Tue, 8 Dec 2020 12:23:37 +0100 Subject: [PATCH 09/13] Added Cloud Mask Test product --- satpy/etc/readers/fci_l2_nc.yaml | 466 ++++++++++++++++++++- satpy/readers/fci_l2_nc.py | 14 +- satpy/tests/reader_tests/test_fci_l2_nc.py | 18 +- 3 files changed, 486 insertions(+), 12 deletions(-) diff --git a/satpy/etc/readers/fci_l2_nc.yaml b/satpy/etc/readers/fci_l2_nc.yaml index 6c3faba843..9a74b9b934 100644 --- a/satpy/etc/readers/fci_l2_nc.yaml +++ b/satpy/etc/readers/fci_l2_nc.yaml @@ -21,6 +21,10 @@ file_types: file_patterns: ['FCI_SIM_CLM_2KM_{creation_time:%Y%m%d}_1700.nc', 'W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-CLM--{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc'] + nc_fci_test_clm: + file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCFileHandler + file_patterns: [ 'W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-CLMTest-{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc' ] + nc_fci_ct: file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCFileHandler file_patterns: ['W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-CT--{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc'] @@ -33,8 +37,6 @@ file_types: file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCSegmentFileHandler file_patterns: [ "W_XX-EUMETSAT-{reception_location},{instrument},{long_platform_id}+{processing_location}-{level}-ASR--{temp_str}_C_EUMT_{creation_time:%Y%m%d%H%M%S}_L2PF_{env}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_N__T_{rep_cycle_in_day}_{rep_cycle_count}.nc"] - - # W_XX-EUMETSAT-Darmstadt,IMG+SAT,MTI1+FCI-2-GII--FD------NC4E_C_EUMT_20201105030947_L2PF_DEV_20170410171000_20170410172000_N__T_0104_0000.nc nc_fci_gii: file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCSegmentFileHandler # TODO: Pattern based on the available test files, not compatible with MTG GFS definitions @@ -188,6 +190,466 @@ datasets: fill_value: -999 mask_value: 0 +# CLM Test + cloud_test_sit1_flag: + name: cloud_test_sit1_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_sit1_flag + extract_byte: 0 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt1_flag: + name: cloud_test_cmt1_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt1_flag + extract_byte: 1 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt2_flag: + name: cloud_test_cmt2_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt2_flag + extract_byte: 2 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt3_flag: + name: cloud_test_cmt3_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt3_flag + extract_byte: 3 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt4_flag: + name: cloud_test_cmt4_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt4_flag + extract_byte: 4 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt5_flag: + name: cloud_test_cmt5_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt5_flag + extract_byte: 5 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt6_flag: + name: cloud_test_cmt6_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt6_flag + extract_byte: 6 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt7_flag: + name: cloud_test_cmt7_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt7_flag + extract_byte: 7 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt8_flag: + name: cloud_test_cmt8_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt8_flag + extract_byte: 8 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt9_flag: + name: cloud_test_cmt9_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt9_flag + extract_byte: 9 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt10_flag: + name: cloud_test_cmt10_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt0_flag + extract_byte: 10 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt11_flag: + name: cloud_test_cmt11_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt11_flag + extract_byte: 11 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt12_flag: + name: cloud_test_cmt12_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt12_flag + extract_byte: 12 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt13_flag: + name: cloud_test_cmt13_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt13_flag + extract_byte: 13 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt14_flag: + name: cloud_test_cmt14_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmt14_flag + extract_byte: 14 + fill_value: -999 + mask_value: 0 + + cloud_test_opqt_flag: + name: cloud_test_opqt_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_opqt_flag + extract_byte: 15 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt1_flag: + name: cloud_test_cmrt1_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmrt1_flag + extract_byte: 16 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt2_flag: + name: cloud_test_cmrt2_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmrt2_flag + extract_byte: 17 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt3_flag: + name: cloud_test_cmrt3_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmrt3_flag + extract_byte: 18 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt4_flag: + name: cloud_test_cmrt4_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmrt4_flag + extract_byte: 19 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt5_flag: + name: cloud_test_cmrt5_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmrt5_flag + extract_byte: 20 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt6_flag: + name: cloud_test_cmrt6_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_cmrt6_flag + extract_byte: 21 + fill_value: -999 + mask_value: 0 + + cloud_test_dust_flag: + name: cloud_test_dust_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_dust_flag + extract_byte: 22 + fill_value: -999 + mask_value: 0 + + cloud_test_ash_flag: + name: cloud_test_ash_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_ash_flag + extract_byte: 23 + fill_value: -999 + mask_value: 0 + + cloud_test_dust_ash_flag: + name: cloud_test_dust_ash_flag + file_type: nc_fci_test_clm + file_key: cloud_mask_test_flag + standard_name: cloud_mask_test_dust_ash_flag + extract_byte: 24 + fill_value: -999 + mask_value: 0 + + cloud_test_sit1: + name: cloud_test_sit1 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_sit1 + extract_byte: 0 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt1: + name: cloud_test_cmt1 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt1 + extract_byte: 1 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt2: + name: cloud_test_cmt2 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt2 + extract_byte: 2 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt3: + name: cloud_test_cmt3 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt3 + extract_byte: 3 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt4: + name: cloud_test_cmt4 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt4 + extract_byte: 4 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt5: + name: cloud_test_cmt5 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt5 + extract_byte: 5 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt6: + name: cloud_test_cmt6 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt6 + extract_byte: 6 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt7: + name: cloud_test_cmt7 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt7 + extract_byte: 7 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt8: + name: cloud_test_cmt8 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt8 + extract_byte: 8 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt9: + name: cloud_test_cmt9 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt9 + extract_byte: 9 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt10: + name: cloud_test_cmt10 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt10 + extract_byte: 10 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt11: + name: cloud_test_cmt11 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt11 + extract_byte: 11 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt12: + name: cloud_test_cmt12 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt12 + extract_byte: 12 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt13: + name: cloud_test_cmt13 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt13 + extract_byte: 13 + fill_value: -999 + mask_value: 0 + + cloud_test_cmt14: + name: cloud_test_cmt14 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmt14 + extract_byte: 14 + fill_value: -999 + mask_value: 0 + + cloud_test_opqt: + name: cloud_test_opqt + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_opqt + extract_byte: 15 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt1: + name: cloud_test_cmrt1 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmrt1 + extract_byte: 16 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt2: + name: cloud_test_cmrt2 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmrt2 + extract_byte: 17 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt3: + name: cloud_test_cmrt3 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmrt3 + extract_byte: 18 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt4: + name: cloud_test_cmrt4 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmrt4 + extract_byte: 19 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt5: + name: cloud_test_cmrt5 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmrt5 + extract_byte: 20 + fill_value: -999 + mask_value: 0 + + cloud_test_cmrt6: + name: cloud_test_cmrt6 + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_cmrt6 + extract_byte: 21 + fill_value: -999 + mask_value: 0 + + cloud_test_dust: + name: cloud_test_dust + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_dust + extract_byte: 22 + fill_value: -999 + mask_value: 0 + + cloud_test_ash: + name: cloud_test_ash + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_ash + extract_byte: 23 + fill_value: -999 + mask_value: 0 + + cloud_test_dust_ash: + name: cloud_test_dust_ash + file_type: nc_fci_test_clm + file_key: cloud_mask_test_result + standard_name: cloud_mask_test_dust_ash + extract_byte: 24 + fill_value: -999 + mask_value: 0 + + cloud_mask_cmrt6_result: + name: cloud_mask_cmrt6_result + file_type: nc_fci_test_clm + file_key: cloud_mask_cmrt6_test_result + standard_name: cloud_mask_cmrt6_result + extract_byte: 0 +# fill_value: -999 + mask_value: 0 + latitude: name: latitude file_key: 'latitude' diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index 0455046d4e..0ef15e1cfc 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -169,9 +169,14 @@ def get_dataset(self, dataset_id, dataset_info): fill_value = dataset_info['fill_value'] except KeyError: fill_value = np.NaN - float_variable = variable.where(variable != fill_value, mask_value).astype('float32', copy=False) - float_variable.attrs = variable.attrs - variable = float_variable + + if dataset_info['file_type'] == 'nc_fci_test_clm': + data_values = variable.where(variable != fill_value, mask_value).astype('uint32', copy=False) + else: + data_values = variable.where(variable != fill_value, mask_value).astype('float32', copy=False) + + data_values.attrs = variable.attrs + variable = data_values # If the variable has 3 dimensions, select the required layer if variable.ndim == 3: @@ -179,6 +184,9 @@ def get_dataset(self, dataset_id, dataset_info): logger.debug('Selecting the layer %d.', layer) variable = variable.sel(maximum_number_of_layers=layer) + if dataset_info['file_type'] == 'nc_fci_test_clm' and var_key != 'cloud_mask_cmrt6_test_result': + variable.values = (variable.values >> dataset_info['extract_byte'] << 31 >> 31) + # Rename the dimensions as required by Satpy variable = variable.rename({"number_of_rows": 'y', "number_of_columns": 'x'}) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index a2f5a15173..0ca4bdf97f 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -88,7 +88,7 @@ def setUp(self): filename=TEST_FILE, filename_info={ 'creation_time': datetime.datetime(year=2017, month=9, day=20, - hour=12, minute=30, second=30) + hour=12, minute=30, second=30), }, filetype_info={} ) @@ -175,7 +175,9 @@ def test_dataset(self): # Checks the correct execution of the get_dataset function with a valid file_key dataset = self.reader.get_dataset(None, {'file_key': 'test_one_layer', - 'fill_value': -999, 'mask_value': 0}) + 'fill_value': -999, 'mask_value': 0., + 'file_type': 'test_file_type'}) + self.assertTrue(np.allclose(dataset.values, np.ones((100, 10)))) self.assertEqual(dataset.attrs['test_attr'], 'attr') self.assertEqual(dataset.attrs['units'], 'test_units') @@ -184,7 +186,8 @@ def test_dataset(self): # Checks the correct execution of the get_dataset function with a valid file_key & layer dataset = self.reader.get_dataset(None, {'file_key': 'test_two_layers', 'layer': 1, - 'fill_value': -999, 'mask_value': 0}) + 'fill_value': -999, 'mask_value': 0, + 'file_type': 'test_file_type'}) self.assertTrue(np.allclose(dataset.values, 2 * np.ones((100, 10)))) self.assertEqual(dataset.attrs['units'], None) self.assertEqual(dataset.attrs['spacecraft_name'], 'test_platform') @@ -192,7 +195,8 @@ def test_dataset(self): # Checks the correct execution of the get_dataset function with an invalid file_key invalid_dataset = self.reader.get_dataset(None, {'file_key': 'test_invalid', - 'fill_value': -999, 'mask_value': 0}) + 'fill_value': -999, 'mask_value': 0, + 'file_type': 'test_file_type'}) # Checks that the function returns None self.assertEqual(invalid_dataset, None) @@ -244,7 +248,7 @@ def setUp(self): filename=SEG_TEST_FILE, filename_info={ 'creation_time': datetime.datetime(year=2017, month=9, day=20, - hour=12, minute=30, second=30) + hour=12, minute=30, second=30), }, filetype_info={} ) @@ -297,7 +301,7 @@ def test_dataset(self): # Checks the correct execution of the get_dataset function with a valid file_key dataset = self.segment_reader.get_dataset(None, {'file_key': 'test_values', - 'fill_value': -999, 'mask_value': 0}) + 'fill_value': -999, 'mask_value': 0, }) self.assertTrue(np.allclose(dataset.values, np.ones((100, 10, 8, 6)))) self.assertEqual(dataset.attrs['test_attr'], 'attr') self.assertEqual(dataset.attrs['units'], 'test_units') @@ -358,7 +362,7 @@ def setUp(self): filename=TEST_ERROR_FILE, filename_info={ 'creation_time': datetime.datetime(year=2017, month=9, day=20, - hour=12, minute=30, second=30) + hour=12, minute=30, second=30), }, filetype_info={} ) From 9716b6afa24a311ab5197a4a8d98a319642a3974 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Thu, 17 Dec 2020 10:32:46 +0000 Subject: [PATCH 10/13] New test added that extracts but values for the Cloud Test flags --- satpy/readers/fci_l2_nc.py | 17 ++-- satpy/tests/reader_tests/test_fci_l2_nc.py | 92 +++++++++++++++++++--- 2 files changed, 87 insertions(+), 22 deletions(-) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index 0ef15e1cfc..9aced692ff 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -27,6 +27,7 @@ from satpy.readers.file_handlers import BaseFileHandler from satpy.readers._geos_area import get_area_definition, make_ext from satpy import CHUNK_SIZE +from contextlib import suppress logger = logging.getLogger(__name__) @@ -105,10 +106,8 @@ def _get_global_attributes(self): def __del__(self): """Close the NetCDF file that may still be open.""" - try: + with suppress(OSError): self.nc.close() - except AttributeError: - pass class FciL2NCFileHandler(BaseFileHandler, FciL2CommonFunctions): @@ -294,14 +293,10 @@ def get_dataset(self, dataset_id, dataset_info): # instead of being masked directly in the netCDF variable. # therefore NaN is applied where such value is found or (0 if the array contains integer values) # the next 11 lines have to be removed once the product files are correctly configured - try: - mask_value = dataset_info['mask_value'] - except KeyError: - mask_value = np.NaN - try: - fill_value = dataset_info['fill_value'] - except KeyError: - fill_value = np.NaN + + mask_value = dataset_info.get('mask_value', np.NaN) + fill_value = dataset_info.get('fill_value', np.NaN) + float_variable = variable.where(variable != fill_value, mask_value).astype('float32', copy=False) float_variable.attrs = variable.attrs variable = float_variable diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index 0ca4bdf97f..b34d9da93a 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -27,6 +27,8 @@ import unittest +from contextlib import suppress + try: from unittest import mock except ImportError: @@ -36,6 +38,7 @@ TEST_FILE = 'test_file_fci_l2_nc.nc' SEG_TEST_FILE = 'test_seg_file_fci_l2_nc.nc' TEST_ERROR_FILE = 'test_error_file_fci_l2_nc.nc' +TEST_BYTE_FILE = 'test_byte_file_fci_l2_nc.nc' class TestFciL2NCFileHandler(unittest.TestCase): @@ -98,15 +101,12 @@ def tearDown(self): # First delete the reader, forcing the file to be closed if still open del self.reader # Then can safely remove it from the system - try: + with suppress(OSError): os.remove(TEST_FILE) - except OSError: - pass def test_all_basic(self): """Test all basic functionalities.""" self.assertEqual(PRODUCT_DATA_DURATION_MINUTES, 20) - print('reader', self.reader) self.assertEqual(self.reader._start_time, datetime.datetime(year=2017, month=9, day=20, @@ -202,7 +202,7 @@ def test_dataset(self): class TestFciL2NCSegmentFileHandler(unittest.TestCase): - """Test the FciL2NCFileHandler reader.""" + """Test the FciL2NCSegmentFileHandler reader.""" def setUp(self): """Set up the test by creating a test file and opening it with the reader.""" @@ -258,10 +258,8 @@ def tearDown(self): # First delete the reader, forcing the file to be closed if still open del self.segment_reader # Then can safely remove it from the system - try: + with suppress(OSError): os.remove(SEG_TEST_FILE) - except OSError: - pass def test_all_basic(self): """Test all basic functionalities.""" @@ -372,10 +370,8 @@ def tearDown(self): # First delete the reader, forcing the file to be closed if still open del self.error_reader # Then can safely remove it from the system - try: + with suppress(OSError): os.remove(TEST_ERROR_FILE) - except OSError: - pass def test_errors(self): self.assertRaises(TypeError, self.error_reader._start_time, @@ -389,3 +385,77 @@ def test_errors(self): self.assertRaises(TypeError, self.error_reader._spacecraft_name) self.assertRaises(TypeError, self.error_reader._sensor_name) + + +class TestFciL2NCReadingByteData(unittest.TestCase): + """Test the FciL2NCFileHandler when reading and extracting byte data.""" + + def setUp(self): + """Set up the test by creating a test file and opening it with the reader.""" + # Easiest way to test the reader is to create a test netCDF file on the fly + + with Dataset(TEST_BYTE_FILE, 'w') as nc_byte: + # Create dimensions + nc_byte.createDimension('number_of_columns', 1) + nc_byte.createDimension('number_of_rows', 1) + + # Add datasets + x = nc_byte.createVariable('x', np.float32, dimensions=('number_of_columns',)) + x.standard_name = 'projection_x_coordinate' + x[:] = np.arange(1) + + y = nc_byte.createVariable('y', np.float32, dimensions=('number_of_rows',)) + x.standard_name = 'projection_y_coordinate' + y[:] = np.arange(1) + + mtg_geos_projection = nc_byte.createVariable('mtg_geos_projection', np.int, dimensions=()) + mtg_geos_projection.longitude_of_projection_origin = 10.0 + mtg_geos_projection.semi_major_axis = 6378137. + mtg_geos_projection.semi_minor_axis = 6356752. + mtg_geos_projection.perspective_point_height = 35786400. + + test_dataset = nc_byte.createVariable('cloud_mask_test_flag', np.float32, + dimensions=('number_of_rows', 'number_of_columns',)) + + # This number was chosen as we know the expected byte values + test_dataset[:] = 4544767 + + self.byte_reader = FciL2NCFileHandler( + filename=TEST_BYTE_FILE, + filename_info={ + 'creation_time': datetime.datetime(year=2017, month=9, day=20, + hour=12, minute=30, second=30), + }, + filetype_info={} + ) + + def tearDown(self): + """Remove the previously created test file.""" + # First delete the reader, forcing the file to be closed if still open + del self.byte_reader + # Then can safely remove it from the system + with suppress(OSError): + os.remove(TEST_BYTE_FILE) + + def test_byte_extraction(self): + """Test the execution of the get_dataset function.""" + + # Value of 1 is expected to be returned for this test + dataset = self.byte_reader.get_dataset(None, + {'file_key': 'cloud_mask_test_flag', + 'fill_value': -999, 'mask_value': 0., + 'file_type': 'nc_fci_test_clm', + 'extract_byte': 1, + }) + + self.assertEquals(dataset.values, 1) + + # Value of 0 is expected fto be returned or this test + dataset = self.byte_reader.get_dataset(None, + {'file_key': 'cloud_mask_test_flag', + 'fill_value': -999, 'mask_value': 0., + 'file_type': 'nc_fci_test_clm', + 'extract_byte': 23, + }) + + self.assertEquals(dataset.values, 0) From dfc2972c2f9fb42ea7ca0688f027311b50ccb962 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Thu, 17 Dec 2020 13:14:50 +0000 Subject: [PATCH 11/13] fci l2 reader and test script had their imports optimised --- satpy/readers/fci_l2_nc.py | 10 +++++----- satpy/tests/reader_tests/test_fci_l2_nc.py | 9 ++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index 9aced692ff..1c6a0d18b1 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -19,15 +19,15 @@ """Reader for the FCI L2 products in NetCDF4 format.""" import logging +from contextlib import suppress +from datetime import datetime, timedelta + import numpy as np import xarray as xr -from datetime import datetime, timedelta - -from satpy.readers.file_handlers import BaseFileHandler -from satpy.readers._geos_area import get_area_definition, make_ext from satpy import CHUNK_SIZE -from contextlib import suppress +from satpy.readers._geos_area import get_area_definition, make_ext +from satpy.readers.file_handlers import BaseFileHandler logger = logging.getLogger(__name__) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index b34d9da93a..17a1a2cc11 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -18,17 +18,16 @@ """The fci_cld_l2_nc reader tests package.""" +import datetime import os +import unittest +from contextlib import suppress + import numpy as np -import datetime from netCDF4 import Dataset from satpy.readers.fci_l2_nc import FciL2NCFileHandler, FciL2NCSegmentFileHandler, PRODUCT_DATA_DURATION_MINUTES -import unittest - -from contextlib import suppress - try: from unittest import mock except ImportError: From 9f37c164414cb1b31395b2253f9a5f5fd1db28d9 Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Thu, 17 Dec 2020 13:32:37 +0000 Subject: [PATCH 12/13] fci l2 reader and test script had their imports optimised --- satpy/tests/reader_tests/test_fci_l2_nc.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index 17a1a2cc11..699eec226a 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -22,18 +22,13 @@ import os import unittest from contextlib import suppress +from unittest import mock import numpy as np from netCDF4 import Dataset from satpy.readers.fci_l2_nc import FciL2NCFileHandler, FciL2NCSegmentFileHandler, PRODUCT_DATA_DURATION_MINUTES -try: - from unittest import mock -except ImportError: - import mock - - TEST_FILE = 'test_file_fci_l2_nc.nc' SEG_TEST_FILE = 'test_seg_file_fci_l2_nc.nc' TEST_ERROR_FILE = 'test_error_file_fci_l2_nc.nc' From 104db01ea119b0f4f0178f09ff7f022c81c7754d Mon Sep 17 00:00:00 2001 From: "Colin.Duff@eumetsat.int" Date: Thu, 17 Dec 2020 13:37:21 +0000 Subject: [PATCH 13/13] changed assertequals to assertequal --- satpy/tests/reader_tests/test_fci_l2_nc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index 699eec226a..c61139b8ef 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -442,7 +442,7 @@ def test_byte_extraction(self): 'extract_byte': 1, }) - self.assertEquals(dataset.values, 1) + self.assertEqual(dataset.values, 1) # Value of 0 is expected fto be returned or this test dataset = self.byte_reader.get_dataset(None, @@ -452,4 +452,4 @@ def test_byte_extraction(self): 'extract_byte': 23, }) - self.assertEquals(dataset.values, 0) + self.assertEqual(dataset.values, 0)