Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix issue with reading MSG GRIB products from the eumetsat datastore #1503

Merged
merged 4 commits into from
Jan 22, 2021
Merged

Fix issue with reading MSG GRIB products from the eumetsat datastore #1503

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cacf4fd
Code updated to fix issue with reading MSG GRIB products from the eum…
Jan 5, 2021
07961ef
Moved import loggin statement
Jan 5, 2021
9f2370d
added variable identifiers for 0100 string as this can vary
Jan 7, 2021
3ea7001
Updated reader and yaml file in line with reviewers comments
Jan 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
183 changes: 169 additions & 14 deletions satpy/etc/readers/seviri_l2_grib.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,59 @@ file_types:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'CLMEncProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGCLMK-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGCLMK-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGCLMK-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGCLMK-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGCLMK-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

# EUMETSAT MSG SEVIRI L2 Optimal Cloud Analysis files in GRIB format
grib_seviri_oca:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'OCAEncProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGOCAE-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGOCAE-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGOCAE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGOCAE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGOCAE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

# EUMETSAT MSG SEVIRI L2 Active Fire Monitoring files in GRIB format
grib_seviri_fir:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'FIREncProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGFIRG-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGFIRG-0100-0100-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGFIRG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGFIRG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGFIRG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

grib_seviri_aes:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'AESGRIBProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGAESE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGAESE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGAESE-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

grib_seviri_cth:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'CTHEncProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGCLTH-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGCLTH-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGCLTH-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

grib_seviri_crm:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'CRMEncProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGCRMN-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGCRMN-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGCRMN-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

grib_seviri_mpe:
file_reader: !!python/name:satpy.readers.seviri_l2_grib.SeviriL2GribFileHandler
file_patterns:
- 'MPEGRIBProd_{start_time:%Y%m%d%H%M%S}Z_00_{server:8s}_{spacecraft:5s}_{scan_mode:3s}_{sub_sat:5s}'
- '{spacecraft:4s}-SEVI-MSGMPEG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}'
- '{spacecraft:4s}-SEVI-MSGMPEG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-{product_creation_time:%Y%m%d%H%M%S}-{ord_num:7s}.grb'
- '{spacecraft:4s}-SEVI-MSGMPEG-{id1:4s}-{id2:4s}-{start_time:%Y%m%d%H%M%S}.000000000Z-NA.grb'

datasets:

Expand All @@ -38,31 +73,31 @@ datasets:
resolution: 3000
file_type: grib_seviri_clm
parameter_number: 7
units: ""
units: "1"
long_name: cloud_mask_classification

pixel_scene_type:
name: pixel_scene_type
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 8
units: ""
units: "1"
long_name: scene_classification

measurement_cost:
name: measurement_cost
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 30
units: ""
units: "1"
long_name: cost_function

upper_layer_cloud_optical_depth:
name: upper_layer_cloud_optical_depth
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 31
units: ""
units: "1"
long_name: cloud_optical_depth

upper_layer_cloud_top_pressure:
Expand All @@ -86,7 +121,7 @@ datasets:
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 34
units: ""
units: "1"
long_name: cloud_optical_depth

error_in_upper_layer_cloud_top_pressure:
Expand All @@ -110,7 +145,7 @@ datasets:
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 37
units: ""
units: "1"
long_name: cloud_optical_depth

lower_layer_cloud_top_pressure:
Expand All @@ -126,7 +161,7 @@ datasets:
resolution: 3000
file_type: grib_seviri_oca
parameter_number: 39
units: ""
units: "1"
long_name: cloud_optical_depth_standard_error

error_in_lower_layer_cloud_top_pressure:
Expand All @@ -150,5 +185,125 @@ datasets:
resolution: 3000
file_type: grib_seviri_fir
parameter_number: 9
units: ""
units: "1"
long_name: active_fire_classification

aerosol_optical_thickness_vis06:
name: aerosol_optical_thickness_vis06
resolution: 3000
file_type: grib_seviri_aes
parameter_number: 20
units: "um"
long_name: aerosol_optical_thickness_vis06

aerosol_optical_thickness_vis08:
name: aerosol_optical_thickness_vis08
resolution: 3000
file_type: grib_seviri_aes
parameter_number: 21
units: "um"
long_name: aerosol_optical_thickness_vis08

aerosol_optical_thickness_vis16:
name: aerosol_optical_thickness_vis16
resolution: 3000
file_type: grib_seviri_aes
parameter_number: 22
units: "um"
long_name: aerosol_optical_thickness_vis06

angstroem_coefficient:
name: angstroem_coefficient
resolution: 3000
file_type: grib_seviri_aes
parameter_number: 23
units: "1"
long_name: angstroem_coefficient

aes_quality:
name: aes_quality
resolution: 3000
file_type: grib_seviri_aes
parameter_number: 192
units: "1"
long_name: aes_quality

cloud_top_height:
name: cloud_top_height
resolution: 3000
file_type: grib_seviri_cth
parameter_number: 2
units: Pa
long_name: cloud_top_height

cloud_top_quality:
name: cloud_top_quality
resolution: 3000
file_type: grib_seviri_cth
parameter_number: 3
units: "1"
long_name: cloud_top_quality

vis_refl_06:
name: vis_refl_06
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 9
units: "%"
long_name: vis_refl_06

vis_refl_08:
name: vis_refl_08
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 10
units: "%"
long_name: vis_refl_08

vis_refl_16:
name: vis_refl_16
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 11
units: "%"
long_name: vis_refl_16

nir_refl_39:
name: nir_refl_39
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 12
units: "%"
long_name: nir_refl_39

num_accumulations:
name: num_accumulations
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 6
units: "1"
long_name: num_accumulations

azimuth_angle:
name: azimuth_angle
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 7
units: degrees
long_name: azimuth_angle

relative_azimuth_angle:
name: relative_azimuth_angle
resolution: 3000
file_type: grib_seviri_crm
parameter_number: 8
units: degrees
long_name: relative_azimuth_angle

instantaneous_rain_rate:
name: instantaneous_rain_rate
resolution: 3000
file_type: grib_seviri_mpe
parameter_number: 1
units: "kg m-2 s-1"
long_name: instantaneous_rain_rate
11 changes: 6 additions & 5 deletions satpy/readers/seviri_l2_grib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,17 @@

"""
import logging
import dask.array as da
import numpy as np
import xarray as xr
import dask.array as da

from datetime import timedelta

from satpy.readers.file_handlers import BaseFileHandler
from satpy import CHUNK_SIZE
from satpy.readers._geos_area import get_area_definition
from satpy.readers.file_handlers import BaseFileHandler
from satpy.readers.seviri_base import (calculate_area_extent,
PLATFORM_DICT,
REPEAT_CYCLE_DURATION)
from satpy import CHUNK_SIZE

try:
import eccodes as ec
Expand Down Expand Up @@ -143,6 +142,7 @@ def get_dataset(self, dataset_id, dataset_info):

# Check if the parameter number in the GRIB message corresponds to the required key
parameter_number = self._get_from_msg(gid, 'parameterNumber')

if parameter_number != dataset_info['parameter_number']:
# The parameter number is not the correct one, skip to next message
ec.codes_release(gid)
Expand All @@ -153,7 +153,8 @@ def get_dataset(self, dataset_id, dataset_info):

# Retrieve values and metadata from the GRIB message, masking the values equal to missing_value
xarr = self._get_xarray_from_msg(gid)
xarr.where(xarr.data == missing_value, np.NaN)

xarr.data = da.where(xarr.data == missing_value, np.nan, xarr.data)

ec.codes_release(gid)

Expand Down