Merge pull request #301 from djhoese/feature-satpy-dataids

Add support for Satpy 0.23+ and PROJ 6.0+

.travis.yml

@@ -75,7 +75,7 @@ script:
 - python build_conda_pack.py -j -1 $oflag
 - ls -l
 after_success:
-- if [ $PYTHON_VERSION == "3.7" ] && [ "${TRAVIS_OS_NAME}" != "windows" ]; then coveralls;
+- if [ $PYTHON_VERSION == "3.7" ] && [ "${TRAVIS_OS_NAME}" != "windows" ]; then coveralls; fi
 - if [[ $TRAVIS_TAG == "" ]]; then
     odir="experimental/";
   else

README.md

@@ -2,7 +2,7 @@
 
 [![Coverage Status](https://coveralls.io/repos/github/ssec/sift/badge.svg)](https://coveralls.io/github/ssec/sift)
 [![PyPI version](https://badge.fury.io/py/uwsift.svg)](https://badge.fury.io/py/uwsift)
-[![Build Status](https://travis-ci.org/ssec/sift.svg?branch=master)](https://travis-ci.org/ssec/sift)
+[![Build Status](https://travis-ci.com/github/ssec/sift.svg?branch=master)](https://travis-ci.com/github/ssec/sift)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2587907.svg)](https://doi.org/10.5281/zenodo.2587907)
 [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/gitterHQ/gitter)
 

uwsift/satpy_compat.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Wrappers around Satpy to handle changes between versions."""
+
+try:
+    from satpy import DataID
+
+    def get_id_value(id_obj, key, default=None):
+        return id_obj.get(key, default)
+
+    def get_id_items(id_obj):
+        return id_obj.items()
+
+    def id_from_attrs(attrs):
+        return attrs['_satpy_id']
+except ImportError:
+    import warnings
+    warnings.warn("Satpy <0.23.0 will not be supported in future versions. "
+                  "Please update your version of Satpy.", DeprecationWarning)
+    from satpy import DatasetID as DataID
+
+    def get_id_value(id_obj, key, default=None):
+        return getattr(id_obj, key, default)
+
+    def get_id_items(id_obj):
+        return id_obj._asdict().items()
+
+    def id_from_attrs(attrs):
+        return DataID.from_dict(attrs)

uwsift/tests/workspace/test_importer.py

@@ -11,7 +11,7 @@
 from satpy import DatasetID, Scene
 from pyresample.geometry import AreaDefinition
 from uwsift.workspace.importer import available_satpy_readers, SatpyImporter
-from uwsift.common import Info
+from uwsift.common import Info, Kind
 
 
 def test_available_satpy_readers_defaults():
@@ -121,3 +121,57 @@ def test_satpy_importer_basic(tmpdir, monkeypatch, mocker):
     assert len(products) == 1
     assert products[0].info[Info.CENTRAL_WAVELENGTH] == 2.0
     assert products[0].info[Info.STANDARD_NAME] == 'toa_bidirectional_reflectance'
+
+
+def test_satpy_importer_contour_0_360(tmpdir, monkeypatch, mocker):
+    """Test import of grib contour data using Satpy."""
+    db_sess = mocker.MagicMock()
+    attrs = {
+        'name': 'gh',
+        'level': 125,
+        'area': AreaDefinition(
+            'test', 'test', 'test',
+            {
+                'proj': 'eqc',
+                'lon_0': 0,
+                'pm': 180,
+                'R': 6371229,
+            }, 240, 120,
+            (-20015806.220738243, -10007903.110369122, 20015806.220738243, 10007903.110369122)
+        ),
+        'start_time': datetime(2018, 9, 10, 17, 0, 31, 100000),
+        'end_time': datetime(2018, 9, 10, 17, 11, 7, 800000),
+        'model_time': datetime(2018, 9, 10, 17, 11, 7, 800000),
+        'standard_name': 'geopotential_height',
+    }
+    data_arr = xr.DataArray(da.from_array(np.random.random((120, 240)).astype(np.float64), chunks='auto'),
+                            attrs=attrs)
+    scn = Scene()
+    scn['gh'] = data_arr
+    scn.load = mocker.MagicMock()  # don't do anything on load
+
+    imp = SatpyImporter(['/test/file.nc'], tmpdir, db_sess,
+                        scene=scn,
+                        reader='grib',
+                        dataset_ids=[DatasetID(name='gh', level=125)])
+    imp.merge_resources()
+    assert imp.num_products == 1
+    products = list(imp.merge_products())
+    assert len(products) == 1
+    assert products[0].info[Info.STANDARD_NAME] == 'geopotential_height'
+    assert products[0].info[Info.KIND] == Kind.CONTOUR
+
+    query_mock = mocker.MagicMock(name='query')
+    filter1_mock = mocker.MagicMock(name='filter1')
+    filter2_mock = mocker.MagicMock(name='filter2')
+    db_sess.query.return_value = query_mock
+    query_mock.filter.return_value = filter1_mock
+    filter1_mock.filter.return_value = filter2_mock
+    filter2_mock.all.return_value = products
+    import_gen = imp.begin_import_products()
+    content_progresses = list(import_gen)
+    # image and contour content
+    assert len(content_progresses) == 2
+    # make sure data was swapped to -180/180 space
+    assert (content_progresses[0].data[:, :120] == data_arr.data[:, 120:].astype(np.float32)).all()
+    assert (content_progresses[0].data[:, 120:] == data_arr.data[:, :120].astype(np.float32)).all()

uwsift/view/open_file_wizard.py

@@ -17,13 +17,15 @@
 
 import os
 import logging
+from enum import Enum
 from PyQt5 import QtCore, QtWidgets, QtGui
 from PyQt5.QtCore import QPoint
 from PyQt5.QtWidgets import QMenu
 from collections import OrderedDict
 from typing import Generator, Tuple, Union
 
-from satpy import Scene, DatasetID
+from uwsift.satpy_compat import DataID, get_id_value
+from satpy import Scene
 from satpy.readers import group_files
 
 from uwsift import config
@@ -128,18 +130,12 @@ def select_all_products_state(self, checked: bool):
     def select_all_products(self, select=True, prop_key: Union[str, None] = None,
                             prop_val: Union[str, None] = None):
         """Select products based on a specific property."""
-        if prop_key is not None:
-            prop_column = self.config['id_components'].index(prop_key)
-        else:
-            prop_column = 0
-
         for row_idx in range(self.ui.selectIDTable.rowCount()):
             # our check state goes on the name item (always)
             name_item = self.ui.selectIDTable.item(row_idx, 0)
             if prop_key is not None:
-                prop_item = self.ui.selectIDTable.item(row_idx, prop_column)
-                item_val = prop_item.data(QtCore.Qt.UserRole)
-                if item_val != prop_val:
+                item_id = name_item.data(QtCore.Qt.UserRole)
+                if get_id_value(item_id, prop_key) != get_id_value(prop_val, prop_key):
                     continue
             check_state = self._get_checked(select)
             name_item.setCheckState(check_state)
@@ -148,25 +144,25 @@ def _product_context_menu(self, position: QPoint):
         item = self.ui.selectIDTable.itemAt(position)
         col = item.column()
         id_comp = self.config['id_components'][col]
-        id_val = item.data(QtCore.Qt.UserRole)
+        # first column always has DataID
+        id_data = self.ui.selectIDTable.item(item.row(), 0).data(QtCore.Qt.UserRole)
         menu = QMenu()
         select_action = menu.addAction("Select all by '{}'".format(id_comp))
         deselect_action = menu.addAction("Deselect all by '{}'".format(id_comp))
         action = menu.exec_(self.ui.selectIDTable.mapToGlobal(position))
         if action == select_action or action == deselect_action:
             select = action == select_action
-            self.select_all_products(select=select, prop_key=id_comp, prop_val=id_val)
+            self.select_all_products(select=select, prop_key=id_comp, prop_val=id_data)
 
     def collect_selected_ids(self):
         selected_ids = []
+        prime_key = self.config['id_components'][0]
         for item_idx in range(self.ui.selectIDTable.rowCount()):
             id_items = OrderedDict((key, self.ui.selectIDTable.item(item_idx, id_idx))
                                    for id_idx, key in enumerate(self.config['id_components']))
-            if id_items['name'].checkState():
-                id_dict = {key: id_item.data(QtCore.Qt.UserRole)
-                           for key, id_item in id_items.items() if id_item is not None}
-                id_dict['modifiers'] = None
-                selected_ids.append(DatasetID(**id_dict))
+            if id_items[prime_key].checkState():
+                data_id = id_items[prime_key]
+                selected_ids.append(data_id.data(QtCore.Qt.UserRole))
         return selected_ids
 
     def initializePage(self, p_int):
@@ -268,10 +264,10 @@ def _init_file_page(self):
             if self.config['default_reader'] == reader_name:
                 self.ui.readerComboBox.setCurrentIndex(idx)
 
-    def _pretty_identifiers(self, ds_id: DatasetID) -> Generator[Tuple[str, object, str], None, None]:
+    def _pretty_identifiers(self, data_id: DataID) -> Generator[Tuple[str, object, str], None, None]:
         """Determine pretty version of each identifier."""
         for key in self.config['id_components']:
-            value = getattr(ds_id, key, None)
+            value = get_id_value(data_id, key)
             if value is None:
                 pretty_val = "N/A"
             elif key == 'wavelength':
@@ -280,6 +276,10 @@ def _pretty_identifiers(self, ds_id: DatasetID) -> Generator[Tuple[str, object,
                 pretty_val = "{:d} hPa".format(int(value))
             elif key == 'resolution':
                 pretty_val = "{:d}m".format(int(value))
+            elif key == 'calibration' and isinstance(value, Enum):
+                # calibration is an enum in newer Satpy version
+                pretty_val = value.name
+                value = value.name
             else:
                 pretty_val = value
 
@@ -301,7 +301,10 @@ def _init_product_select_page(self):
 
                 self.ui.selectIDTable.setRowCount(idx + 1)
                 item = QtWidgets.QTableWidgetItem(pretty_val)
-                item.setData(QtCore.Qt.UserRole, id_val)
+                if col_idx == 0:
+                    item.setData(QtCore.Qt.UserRole, ds_id)
+                else:
+                    item.setData(QtCore.Qt.UserRole, id_val)
                 item.setFlags((item.flags() ^ QtCore.Qt.ItemIsEditable) | QtCore.Qt.ItemIsUserCheckable)
                 if id_key == 'name':
                     item.setCheckState(QtCore.Qt.Checked)

uwsift/view/transform.py

@@ -559,6 +559,13 @@ def latlong_init(proj_dict):
     }
     """)
 
+# handle prime meridian shifts
+pm_func_str = """
+    float adjlon(float lon) {{
+        return lon + radians({pm});
+    }}
+"""
+
 pj_msfn = Function("""
     float pj_msfn(float sinphi, float cosphi, float es) {
         return (cosphi / sqrt (1. - es * sinphi * sinphi));
@@ -662,7 +669,11 @@ def __init__(self, proj4_str, inverse=False):
         proj_init = proj_funcs[0]
         proj_args = proj_init(proj_dict)
 
-        if proj_args.get('over'):
+        if 'pm' in proj_args:
+            # force to float
+            proj_args['pm'] = float(proj_args['pm'])
+            proj_args['over'] = 'lambda = adjlon(lambda);'
+        elif proj_args.get('over'):
             proj_args['over'] = ''
         else:
             proj_args['over'] = 'lambda = adjlon(lambda);'
@@ -693,7 +704,11 @@ def __init__(self, proj4_str, inverse=False):
             self._shader_map._add_dep(d)
             self._shader_imap._add_dep(d)
 
-        if proj_args['over']:
+        if 'pm' in proj_args:
+            pm_func = Function(pm_func_str.format(**proj_args))
+            self._shader_map._add_dep(pm_func)
+            self._shader_imap._add_dep(pm_func)
+        elif proj_args['over']:
             self._shader_map._add_dep(adjlon_func)
             self._shader_imap._add_dep(adjlon_func)
 
@@ -726,6 +741,10 @@ def create_proj_dict(self, proj_str):
         d['proj4_str'] = proj_str
 
         # if they haven't provided a radius then they must have provided a datum or ellps
+        if 'R' in d:
+            # spheroid
+            d.setdefault('a', d['R'])
+            d.setdefault('b', d['R'])
         if 'a' not in d:
             if 'datum' not in d:
                 d.setdefault('ellps', d.setdefault('datum', 'WGS84'))