Skip to content

Commit

Permalink
Merge pull request #1770 from joleenf/clavrx
Browse files Browse the repository at this point in the history 
Fix CLAVR-x reader and 'awips_tiled' writer to produce AWIPS-compatible output
  • Loading branch information
@djhoese
djhoese committed Jul 29, 2021
2 parents 13d61e8 + 155ce5e commit a76c17e
Show file tree
Hide file tree
Showing 6 changed files with 270 additions and 362 deletions.
15 changes: 15 additions & 0 deletions satpy/etc/writers/awips_tiled.yaml
View file
Expand Up @@ -66,9 +66,12 @@ templates:
# value: "${ORGANIZATION}"
awips_id: {}
# value: "{awips_id}" # special variable created by awips_tiled.py
physical_element: {}
# value: "{physical_element}" #special variable created by awips_tiled.py
satellite_id:
value: "{platform_name!u}-{sensor!u}"
sector_id: {} # special handler in awips_tiled.py

coordinates:
x:
attributes:
Expand Down Expand Up @@ -117,6 +120,7 @@ templates:
reader: clavrx
var_name: data
attributes:
units: {}
physical_element:
value: 'CLAVR-x {name}'
clavrx_cloud_type:
Expand All @@ -126,74 +130,85 @@ templates:
attributes:
physical_element:
raw_value: CLAVR-x Cloud Type
units: {}
clavrx_cld_temp_acha:
reader: clavrx
name: cld_temp_acha
var_name: data
attributes:
units: {}
physical_element:
raw_value: CLAVR-x Cloud Top Temperature (ACHA)
clavrx_cld_height_acha:
reader: clavrx
name: cld_height_acha
var_name: data
attributes:
units: {}
physical_element:
raw_value: CLAVR-x Cloud Top Height (ACHA)
clavrx_cloud_phase:
reader: clavrx
name: cloud_phase
var_name: data
attributes:
units: {}
physical_element:
raw_value: CLAVR-x Cloud Phase
clavrx_cld_opd_dcomp:
reader: clavrx
name: cld_opd_dcomp
var_name: data
attributes:
units: {}
physical_element:
raw_value: CLAVR-x Cloud Optical Depth (dcomp)
clavrx_clld_opd_nlcomp:
reader: clavrx
name: cloud_opd_nlcomp
var_name: data
attributes:
units: {}
physical_element:
raw_value: CLAVR-x Cloud Optical Depth (nlcomp)
clavrx_cld_reff_dcomp:
reader: clavrx
name: cld_reff_dcomp
var_name: data
attributes:
units: {}
physical_element:
raw_value: CLAVR-x Cloud Effective Radius (dcomp)
clavrx_cld_reff_nlcomp:
reader: clavrx
name: cld_reff_nlcomp
var_name: data
attributes:
units: {}
physical_element:
raw_value: CLAVR-x Cloud Effective Radius (nlcomp)
clavrx_cld_emiss_acha:
reader: clavrx
name: cld_emiss_acha
var_name: data
attributes:
units: {}
physical_element:
raw_value: CLAVR-x Cloud Emissivity (ACHA)
clavrx_refl_lunar_dnb_nom:
reader: clavrx
name: refl_lunar_dnb_nom
var_name: data
attributes:
units: {}
physical_element:
raw_value: CLAVR-x Cloud Lunar Reflectance
clavrx_rain_rate:
reader: clavrx
name: rain_rate
var_name: data
attributes:
units: {}
physical_element:
raw_value: CLAVR-x Rain Rate

Expand Down
127 changes: 58 additions & 69 deletions satpy/readers/clavrx.py
View file
Expand Up @@ -96,7 +96,7 @@ class _CLAVRxHelper:
def _remove_attributes(attrs: dict) -> dict:
"""Remove attributes that described data before scaling."""
old_attrs = ['unscaled_missing', 'SCALED_MIN', 'SCALED_MAX',
'SCALED_MISSING', 'actual_missing']
'SCALED_MISSING']

for attr_key in old_attrs:
attrs.pop(attr_key, None)
Expand All @@ -105,31 +105,39 @@ def _remove_attributes(attrs: dict) -> dict:
@staticmethod
def _scale_data(data_arr: xr.DataArray, scale_factor: float, add_offset: float) -> xr.DataArray:
"""Scale data, if needed."""
scaling_needed = not (scale_factor == 1 and add_offset == 0)
scaling_needed = not (scale_factor == 1.0 and add_offset == 0.0)
if scaling_needed:
data_arr = data_arr * scale_factor + add_offset
return data_arr

@staticmethod
def _get_data(data: xr.DataArray, dataset_id: dict, ds_info: dict) -> xr.DataArray:
def _get_data(data: xr.DataArray, dataset_id: dict) -> xr.DataArray:
"""Get a dataset."""
if dataset_id.get('resolution'):
data.attrs['resolution'] = dataset_id['resolution']

attrs = data.attrs.copy()
fill = attrs.pop('_FillValue', None)
factor = attrs.pop('scale_factor', 1.0)
offset = attrs.pop('add_offset', 0.0)
valid_range = attrs.pop('valid_range', None)

data = data.where(data != fill)
data = _CLAVRxHelper._scale_data(data, factor, offset)

if valid_range is not None:
fill = attrs.get('_FillValue')
factor = attrs.pop('scale_factor', (np.ones(1, dtype=data.dtype))[0])
offset = attrs.pop('add_offset', (np.zeros(1, dtype=data.dtype))[0])
valid_range = attrs.get('valid_range', [None])
if isinstance(valid_range, np.ndarray):
attrs["valid_range"] = valid_range.tolist()

flags = not data.attrs.get("SCALED", 1) and any(data.attrs.get("flag_values", [None]))
if not flags:
data = data.where(data != fill)
data = _CLAVRxHelper._scale_data(data, factor, offset)
# don't need _FillValue if it has been applied.
attrs.pop('_FillValue', None)
fill = _CLAVRxHelper._scale_data(fill, factor, offset)

if all(valid_range):
valid_min = _CLAVRxHelper._scale_data(valid_range[0], factor, offset)
valid_max = _CLAVRxHelper._scale_data(valid_range[1], factor, offset)
data = data.where((data >= valid_min) & (data <= valid_max))
data.attrs['valid_min'], data.attrs['valid_max'] = valid_min, valid_max
attrs['valid_range'] = [valid_min, valid_max]

data.attrs = _CLAVRxHelper._remove_attributes(attrs)

Expand Down Expand Up @@ -237,6 +245,32 @@ def _read_axi_fixed_grid(filename: str, l1b_attr) -> geometry.AreaDefinition:

return area

@staticmethod
def get_metadata(sensor, platform, attrs: dict, ds_info: dict) -> dict:
"""Get metadata."""
i = {}
i.update(attrs)
i.update(ds_info)

flag_meanings = i.get('flag_meanings', None)
if not i.get('SCALED', 1) and not flag_meanings:
i['flag_meanings'] = '<flag_meanings_unknown>'
i.setdefault('flag_values', [None])
u = i.get('units')
if u in CF_UNITS:
# CF compliance
i['units'] = CF_UNITS[u]
if u.lower() == "none":
i['units'] = "1"
i['sensor'] = sensor
i['platform_name'] = platform
rps = _get_rows_per_scan(sensor)
if rps:
i['rows_per_scan'] = rps
i['reader'] = 'clavrx'

return i


class CLAVRXHDF4FileHandler(HDF4FileHandler, _CLAVRxHelper):
"""A file handler for CLAVRx files."""
Expand All @@ -257,36 +291,13 @@ def end_time(self):
"""Get the end time."""
return self.filename_info.get('end_time', self.start_time)

def get_metadata(self, attrs: dict, ds_info: dict) -> dict:
"""Get metadata."""
i = {}
i.update(attrs)
i.update(ds_info)

flag_meanings = i.get('flag_meanings', None)
if not i.get('SCALED', 1) and not flag_meanings:
i['flag_meanings'] = '<flag_meanings_unknown>'
i.setdefault('flag_values', [None])
u = i.get('units')
if u in CF_UNITS:
# CF compliance
i['units'] = CF_UNITS[u]

i['sensor'] = self.sensor
i['platform'] = i['platform_name'] = self.platform
rps = _get_rows_per_scan(self.sensor)
if rps:
i['rows_per_scan'] = rps
i['reader'] = 'clavrx'

return i

def get_dataset(self, dataset_id, ds_info):
"""Get a dataset."""
var_name = ds_info.get('file_key', dataset_id['name'])
data = self[var_name]
data = _CLAVRxHelper._get_data(data, dataset_id, ds_info)
data.attrs = self.get_metadata(data.attrs, ds_info)
data = _CLAVRxHelper._get_data(data, dataset_id)
data.attrs = _CLAVRxHelper.get_metadata(self.sensor, self.platform,
data.attrs, ds_info)
return data

def get_nadir_resolution(self, sensor):
Expand Down Expand Up @@ -380,21 +391,24 @@ def __init__(self, filename, filename_info, filetype_info):
decode_cf=True,
mask_and_scale=False,
decode_coords=True,
chunks={'pixel_elements_along_scan_direction': CHUNK_SIZE,
'scan_lines_along_track_direction': CHUNK_SIZE})
chunks=CHUNK_SIZE)
# y,x is used in satpy, bands rather than channel using in xrimage
self.nc = self.nc.rename_dims({'scan_lines_along_track_direction': "y",
'pixel_elements_along_scan_direction': "x"})

self.platform = _get_platform(
self.filename_info.get('platform_shortname', None))
self.sensor = self.nc.attrs.get('sensor', None)
# coordinates need scaling and valid_range (mask_and_scale won't work on valid_range)
self.nc.coords["latitude"] = _CLAVRxHelper._get_data(self.nc.coords["latitude"],
{"name": "latitude"})
self.nc.coords["longitude"] = _CLAVRxHelper._get_data(self.nc.coords["longitude"],
{"name": "longitude"})

def _get_ds_info_for_data_arr(self, var_name):
ds_info = {
'file_type': self.filetype_info['file_type'],
'name': var_name,
'coordinates': ["longitude", "latitude"]
}
return ds_info

Expand Down Expand Up @@ -453,38 +467,13 @@ def get_area_def(self, key):
l1b_att = str(self.nc.attrs.get('L1B', None))
return _CLAVRxHelper._read_axi_fixed_grid(self.filename, l1b_att)

def get_metadata(self, attrs, ds_info):
"""Get metadata."""
i = {}
i.update(attrs)
i.update(ds_info)

flag_meanings = i.get('flag_meanings', None)
if not i.get('SCALED', 1) and not flag_meanings:
i['flag_meanings'] = '<flag_meanings_unknown>'
i.setdefault('flag_values', [None])

u = i.get('units')
if u in CF_UNITS:
# CF compliance
i['units'] = CF_UNITS[u]

i['sensor'] = self.sensor
i['platform'] = i['platform_name'] = self.platform
rps = _get_rows_per_scan(self.sensor)
if rps:
i['rows_per_scan'] = rps
i['reader'] = 'clavrx'

return i

def get_dataset(self, dataset_id, ds_info):
"""Get a dataset."""
var_name = ds_info.get('name', dataset_id['name'])
data = self[var_name]
data = _CLAVRxHelper._get_data(data, dataset_id, ds_info)
data.attrs = self.get_metadata(data.attrs, ds_info)

data = _CLAVRxHelper._get_data(data, dataset_id)
data.attrs = _CLAVRxHelper.get_metadata(self.sensor, self.platform,
data.attrs, ds_info)
return data

def __getitem__(self, item):
Expand Down
15 changes: 9 additions & 6 deletions satpy/tests/reader_tests/test_clavrx.py
View file
Expand Up @@ -98,7 +98,7 @@ def get_test_content(self, filename, filename_info, filetype_info):
'_FillValue': -128,
'flag_meanings': 'clear water supercooled mixed ice unknown',
'flag_values': [0, 1, 2, 3, 4, 5],
'units': '1',
'units': 'none',
})
file_content['variable3/shape'] = DEFAULT_FILE_SHAPE

Expand Down Expand Up @@ -217,9 +217,8 @@ def test_load_all(self):
'variable3'])
self.assertEqual(len(datasets), 3)
for v in datasets.values():
assert 'calibration' not in v.attrs
self.assertEqual(v.attrs['units'], '1')
self.assertEqual(v.attrs['platform'], 'npp')
self.assertEqual(v.attrs['platform_name'], 'npp')
self.assertEqual(v.attrs['sensor'], 'viirs')
self.assertIsInstance(v.attrs['area'], SwathDefinition)
self.assertEqual(v.attrs['area'].lons.attrs['rows_per_scan'], 16)
Expand Down Expand Up @@ -373,9 +372,13 @@ def test_load_all_old_donor(self):
datasets = r.load(['variable1', 'variable2', 'variable3'])
self.assertEqual(len(datasets), 3)
for v in datasets.values():
assert 'calibration' not in v.attrs
self.assertNotIn('calibration', v.attrs)
self.assertEqual(v.attrs['units'], '1')
self.assertIsInstance(v.attrs['area'], AreaDefinition)
if v.attrs["name"] == 'variable1':
self.assertIsInstance(v.attrs["valid_range"], list)
else:
self.assertNotIn('valid_range', v.attrs)
self.assertIsNotNone(datasets['variable3'].attrs.get('flag_meanings'))

def test_load_all_new_donor(self):
Expand Down Expand Up @@ -406,10 +409,10 @@ def test_load_all_new_donor(self):
datasets = r.load(['variable1', 'variable2', 'variable3'])
self.assertEqual(len(datasets), 3)
for v in datasets.values():
assert 'calibration' not in v.attrs
self.assertNotIn('calibration', v.attrs)
self.assertEqual(v.attrs['units'], '1')
self.assertIsInstance(v.attrs['area'], AreaDefinition)
self.assertTrue(v.attrs['area'].is_geostationary)
self.assertEqual(v.attrs['platform'], 'himawari8')
self.assertEqual(v.attrs['platform_name'], 'himawari8')
self.assertEqual(v.attrs['sensor'], 'ahi')
self.assertIsNotNone(datasets['variable3'].attrs.get('flag_meanings'))
8 changes: 5 additions & 3 deletions satpy/tests/reader_tests/test_clavrx_nc.py
View file
Expand Up @@ -73,7 +73,7 @@ def fake_test_content(filename, **kwargs):
'scale_factor': 1.,
'add_offset': 0.,
'units': '1',
'valid_range': (-32767, 32767),
'valid_range': [-32767, 32767],
})

# data with fill values
Expand All @@ -84,7 +84,7 @@ def fake_test_content(filename, **kwargs):
'scale_factor': 1.,
'add_offset': 0.,
'units': '1',
'valid_range': (-32767, 32767),
'valid_range': [-32767, 32767],
})
variable2 = variable2.where(variable2 % 2 != 0)

Expand Down Expand Up @@ -188,7 +188,9 @@ def test_load_all_new_donor(self, filenames, loadable_ids):
assert 'calibration' not in v.attrs
assert v.attrs['units'] == '1'
assert isinstance(v.attrs['area'], AreaDefinition)
assert v.attrs['platform'] == 'himawari8'
assert v.attrs['platform_name'] == 'himawari8'
assert v.attrs['sensor'] == 'AHI'
assert 'rows_per_scan' not in v.coords.get('longitude').attrs
if v.attrs["name"] in ["variable1", "variable2"]:
assert isinstance(v.attrs["valid_range"], list)
assert (datasets['variable3'].attrs.get('flag_meanings')) is not None

0 comments on commit a76c17e

Please sign in to comment.