Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement generic set_orientation method for geostationary datasets #1194

Merged
merged 23 commits into from Jul 23, 2020
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
6c48075
implement first working prototype of set_orientation method
ameraner May 8, 2020
5fd71dc
undo flake8/Pycharm changes to other code parts
ameraner May 8, 2020
2623dd3
make datasets list optional and check for non-geographical Datasets
ameraner May 8, 2020
3f60be5
implement support of StackedAreaDefinitions
ameraner May 8, 2020
423921f
fix the swapping of numpy-arrayed area extents columns
ameraner May 8, 2020
3b7cc43
make _change_area_extent_of_areadef method a function
ameraner May 8, 2020
06d9e14
move set_orientation to yaml_reader and create new GEOFlippableFileYA…
ameraner Jun 4, 2020
7c16d51
improve handling of kwargs
ameraner Jun 4, 2020
7ca2e95
add support for segmented data
ameraner Jun 4, 2020
be5b1b8
change yamlreader class to other SEVIRI/FCI L1 readers
ameraner Jun 4, 2020
02f527b
improve handling of None datasets
ameraner Jun 5, 2020
e79126b
fix target orientation check, improve area_extent update
ameraner Jun 5, 2020
fd0d46a
improve kwargs handling, make upper_right_corner explicit
ameraner Jun 5, 2020
8a32c9b
use crs check instead of proj
ameraner Jun 5, 2020
207371c
Merge branch 'master' into implement_orientation_method
ameraner Jun 29, 2020
f269be0
add tests and do corrections to logging output
ameraner Jul 21, 2020
4cb98c9
add kwargs for ancillary variables loading
ameraner Jul 21, 2020
96579c8
rearrange code in smaller methods and add name of dataset in logging …
ameraner Jul 22, 2020
de70774
make stickler happy on over-indented line
ameraner Jul 22, 2020
88e1c1e
add documentation
ameraner Jul 22, 2020
ccd1728
add acceped values in documentation
ameraner Jul 22, 2020
5c31b6a
Merge branch 'master' into implement_orientation_method
ameraner Jul 22, 2020
7b2419f
fix the tests by replacing the hack with patch.object
ameraner Jul 23, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion satpy/etc/readers/fci_l1c_fdhsi.yaml
Expand Up @@ -6,7 +6,7 @@ reader:
Reader for FCI FDSHI data in NetCDF4 format.
Used to read Meteosat Third Generation (MTG) Flexible
Combined Imager (FCI) Full Disk High Spectral Imagery (FDHSI) data.
reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader
reader: !!python/name:satpy.readers.yaml_reader.GEOFlippableFileYAMLReader
sensors: [fci]

datasets:
Expand Down
2 changes: 1 addition & 1 deletion satpy/etc/readers/seviri_l1b_native.yaml
Expand Up @@ -6,7 +6,7 @@ reader:
Reader for EUMETSAT MSG SEVIRI Level 1b native format files.
sensors: [seviri]
default_channels: [HRV, IR_016, IR_039, IR_087, IR_097, IR_108, IR_120, IR_134, VIS006, VIS008, WV_062, WV_073]
reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader
reader: !!python/name:satpy.readers.yaml_reader.GEOFlippableFileYAMLReader

file_types:
native_msg:
Expand Down
2 changes: 1 addition & 1 deletion satpy/etc/readers/seviri_l1b_nc.yaml
Expand Up @@ -5,7 +5,7 @@ reader:
description: >
NetCDF4 reader for EUMETSAT MSG SEVIRI Level 1b files.
sensors: [seviri]
reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader
reader: !!python/name:satpy.readers.yaml_reader.GEOFlippableFileYAMLReader
group_keys: ["processing_time", "satid"]

file_types:
Expand Down
113 changes: 110 additions & 3 deletions satpy/readers/yaml_reader.py
Expand Up @@ -870,7 +870,114 @@ def _load_area_def(dsid, file_handlers):
return final_area.squeeze()


class GEOSegmentYAMLReader(FileYAMLReader):
def _set_orientation(dataset, upper_right_corner='native', **kwargs):
"""Set the orientation of datasets.

Allows to flip geostationary datasets without having to resample.

Args:
datasets (iterable): Limit the flipping to this list of names (str), wavelengths (float), or
DatasetID objects.
By default all currently loaded datasets are flipped.
upper_right_corner (str): Direction of the upper right corner of the image.
Possible options are 'NW', 'NE', 'SW', 'SE', or 'native'.
The common upright image orientation corresponds to 'NW'.
Defaults to 'native' (no flipping is applied).

"""
def _change_area_extent_of_areadef(areadef, new_extent):
"""Change the area extent of an area definition and regenerate it."""
# does this need to be moved somewhere else?
return AreaDefinition(
ameraner marked this conversation as resolved.
Show resolved Hide resolved
areadef.area_id,
areadef.description,
areadef.proj_id,
areadef.proj_dict,
areadef.width,
areadef.height,
new_extent
)

if upper_right_corner == 'native':
logger.debug("Requested Dataset orientation is 'native'. No flipping is applied.")
return
if upper_right_corner not in ['NW', 'NE', 'SE', 'SW', 'native']:
raise ValueError("Origin corner not recognized. Should be 'NW', 'NE', 'SW', 'SE' or 'native.")

# get the target orientation
target_northup = False
target_eastright = False
if upper_right_corner in ['NW', 'NE']:
target_northup = True
if upper_right_corner in ['NW', 'SW']:
ameraner marked this conversation as resolved.
Show resolved Hide resolved
target_eastright = True

if 'area' not in dataset.attrs:
logger.info("Dataset is not a geographical dataset and cannot be flipped.")
return dataset

if dataset.attrs['area'].proj_dict['proj'] != 'geos':
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is is possible to check the area's CRS here instead of the proj_dict?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess so yes. What is the advantage of doing so?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we're slowly migrating to using crs more instead of proj dicts

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also this here doesn't cover the projections defined by epsg code

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Understood. So I changed to:

if dataset.attrs['area'].crs.coordinate_operation.method_name not in ['Geostationary Satellite (Sweep Y)',
                                                                                                               'Geostationary Satellite (Sweep X)']:

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 @mraspaud Can we be sure that the area def has a crs property or do we need to fallback to the proj_dict?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Iirc the crs is a property of the areadef so always there

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

crs should always exist if they are using a new-ish version of pyproj (2.0+).

logger.info("Dataset is not in geos projection and cannot be flipped.")
return dataset

# get the current dataset orientation
if isinstance(dataset.attrs['area'], StackedAreaDefinition):
# array of area extents if the Area is a StackedAreaDefinition
ds_area_extents = np.asarray([list(adef.area_extent) for adef in dataset.attrs['area'].defs])
else:
# array with a single item if Area is in one piece
ds_area_extents = np.asarray([list(dataset.attrs['area'].area_extent)])

# assumes all AreaDefinitions inside a StackedAreaDefinition have the same orientation
current_northup = ds_area_extents[0, 3] - ds_area_extents[0, 1] > 0
current_eastright = ds_area_extents[0, 2] - ds_area_extents[0, 0] > 0

# if current and target orientations mismatch, then flip data and switch area extent elements
if target_northup == current_northup and target_eastright == current_eastright:
logger.info("Dataset is already in the target orientation.")
return dataset

if target_northup != current_northup:
logger.info("Flipping Dataset upside-down.")
dataset.data = dataset.data[::-1, :]
ds_area_extents[:, [1, 3]] = ds_area_extents[:, [3, 1]]

if target_eastright != current_eastright:
logger.info("Flipping Dataset left-to-right.")
dataset.data = dataset.data[:, ::-1]
ds_area_extents[:, [0, 2]] = ds_area_extents[:, [2, 0]]

# update the dataset area extent
# keeping the same id, description and proj_id, but should probably be changed to reflect the flipping
if len(ds_area_extents) == 1:
new_area_def = _change_area_extent_of_areadef(
dataset.attrs['area'], ds_area_extents[0])
else:
new_area_defs_to_stack = []
for n_area_def, area_def in enumerate(dataset.attrs['area'].defs):
new_area_defs_to_stack.append(_change_area_extent_of_areadef(
area_def, ds_area_extents[n_area_def]))

# flip the order of stacking if the area is upside down
if target_northup != current_northup:
new_area_defs_to_stack = new_area_defs_to_stack[::-1]
new_area_def = StackedAreaDefinition(*new_area_defs_to_stack)

dataset.attrs['area'] = new_area_def

return dataset


class GEOFlippableFileYAMLReader(FileYAMLReader):
"""Reader for flippable geostationary data."""

def _load_dataset_with_area(self, dsid, coords, **kwargs):
ds = super()._load_dataset_with_area(dsid, coords, **kwargs)
ds = _set_orientation(ds, **kwargs)
return ds


class GEOSegmentYAMLReader(GEOFlippableFileYAMLReader):
"""Reader for segmented geostationary data.

This reader pads the data to full geostationary disk if necessary.
Expand Down Expand Up @@ -904,7 +1011,7 @@ def create_filehandlers(self, filenames, fh_kwargs=None):
return created_fhs

@staticmethod
def _load_dataset(dsid, ds_info, file_handlers, dim='y', pad_data=True):
def _load_dataset(dsid, ds_info, file_handlers, dim='y', pad_data=True, **kwargs):
ameraner marked this conversation as resolved.
Show resolved Hide resolved
"""Load only a piece of the dataset."""
if not pad_data:
return FileYAMLReader._load_dataset(dsid, ds_info,
Expand Down Expand Up @@ -936,7 +1043,7 @@ def _load_dataset(dsid, ds_info, file_handlers, dim='y', pad_data=True):
res.attrs = combined_info
return res

def _load_area_def(self, dsid, file_handlers, pad_data=True):
def _load_area_def(self, dsid, file_handlers, pad_data=True, **kwargs):
"""Load the area definition of *dsid* with padding."""
if not pad_data:
return _load_area_def(dsid, file_handlers)
Expand Down