Merge pull request #488 from mraspaud/feature-check-satpy

Add the check_satpy function to find missing dependencies

.github/ISSUE_TEMPLATE/bug_report.md

@@ -27,7 +27,8 @@ If applicable, add screenshots to help explain your problem.
 **Environment Info:**
  - OS: [e.g. OSX, Windows, Linux]
  - SatPy Version: [e.g. 0.9.0]
- - PyResample Version: 
+ - PyResample Version:
+ - Readers and writers dependencies (when relevant): [run `from satpy.config import check_satpy; check_satpy()`]
 
 **Additional context**
 Add any other context about the problem here.

doc/source/quickstart.rst

@@ -219,6 +219,16 @@ on saving datasets and customizing enhancements see the documentation on
 Troubleshooting
 ===============
 
+When something goes wrong, a first step to take is check that the latest Version
+of satpy and its dependencies are installed. Satpy drags in a few packages as
+dependencies per default, but each reader and writer has it's own dependencies
+which can be unfortunately easy to miss when just doing a regular `pip install`.
+To check the missing dependencies for the readers and writers, a utility
+function called `check_satpy` can be used:
+
+  >>> from satpy.config import check_satpy
+  >>> check_satpy()
+
 Due to the way SatPy works, producing as many datasets as possible, there are
 times that behavior can be unexpected but with no exceptions raised. To help
 troubleshoot these situations log messages can be turned on. To do this run

satpy/config.py

@@ -28,9 +28,10 @@
 import glob
 import logging
 import os
-from collections import Mapping
+from collections import Mapping, OrderedDict
 
 from six.moves import configparser
+import yaml
 
 LOG = logging.getLogger(__name__)
 
@@ -52,8 +53,7 @@ def get_environ_ancpath(default='.'):
 
 
 def runtime_import(object_path):
-    """Import at runtime
-    """
+    """Import at runtime."""
     obj_module, obj_element = object_path.rsplit(".", 1)
     loader = __import__(obj_module, globals(), locals(), [str(obj_element)])
     return getattr(loader, obj_element)
@@ -69,17 +69,18 @@ def config_search_paths(filename, *search_dirs, **kwargs):
     # FUTURE: Remove CONFIG_PATH because it should be included as a search_dir
     paths += [os.path.join(CONFIG_PATH, filename),
               os.path.join(PACKAGE_CONFIG_PATH, filename)]
+    paths = [os.path.abspath(path) for path in paths]
 
     if kwargs.get("check_exists", True):
         paths = [x for x in paths if os.path.isfile(x)]
 
+    paths = list(OrderedDict.fromkeys(paths))
     # flip the order of the list so builtins are loaded first
     return paths[::-1]
 
 
 def get_config(filename, *search_dirs, **kwargs):
-    """Blends the different configs, from package defaults to .
-    """
+    """Blends the different configs, from package defaults to ."""
     config = kwargs.get("config_reader_class", configparser.ConfigParser)()
 
     paths = config_search_paths(filename, *search_dirs)
@@ -106,8 +107,7 @@ def glob_config(pattern, *search_dirs):
 
 
 def get_config_path(filename, *search_dirs):
-    """Get the appropriate path for a filename, in that order: filename, ., PPP_CONFIG_DIR, package's etc dir.
-    """
+    """Get the appropriate path for a filename, in that order: filename, ., PPP_CONFIG_DIR, package's etc dir."""
     paths = config_search_paths(filename, *search_dirs)
 
     for path in paths[::-1]:
@@ -116,7 +116,7 @@ def get_config_path(filename, *search_dirs):
 
 
 def recursive_dict_update(d, u):
-    """Recursive dictionary update using
+    """Recursive dictionary update.
 
     Copied from:
 
@@ -130,3 +130,50 @@ def recursive_dict_update(d, u):
         else:
             d[k] = u[k]
     return d
+
+
+def check_yaml_configs(configs, key, hdr_len):
+    """Get a diagnostic for the yaml *configs*.
+
+    *key* is the section to look for to get a name for the config at hand.
+    *hdr_len* is the number of lines that can be safely read from the config to
+    get a name.
+    """
+    diagnostic = {}
+    for i in configs:
+        for fname in i:
+            with open(fname) as stream:
+                try:
+                    res = yaml.load(stream)
+                    try:
+                        diagnostic[res[key]['name']] = 'ok'
+                    except Exception:
+                        continue
+                except yaml.YAMLError as err:
+                    stream.seek(0)
+                    lines = ''.join(stream.readline() for line in range(hdr_len))
+                    res = yaml.load(lines)
+                    if err.context == 'while constructing a Python object':
+                        problem = err.problem
+                    else:
+                        problem = 'error'
+                    try:
+                        diagnostic[res[key]['name']] = problem
+                    except Exception:
+                        continue
+    return diagnostic
+
+
+def check_satpy():
+    """Check the satpy readers and writers for correct installation."""
+    from satpy.readers import configs_for_reader
+    from satpy.writers import configs_for_writer
+    print('Readers')
+    print('=======')
+    for reader, res in sorted(check_yaml_configs(configs_for_reader(), 'reader', 5).items()):
+        print(reader + ': ' + res)
+    print()
+    print('Writers')
+    print('=======')
+    for writer, res in sorted(check_yaml_configs(configs_for_writer(), 'writer', 3).items()):
+        print(writer + ': ' + res)

satpy/tests/test_writers.py

@@ -212,7 +212,7 @@ def test_enhance_empty_config(self):
         self.assertIsNotNone(e.enhancement_tree)
         get_enhanced_image(ds, enhance=e)
         self.assertSetEqual(set(e.sensor_enhancement_configs),
-                            {self.ENH_FN3})
+                            {os.path.abspath(self.ENH_FN3)})
 
     def test_enhance_with_sensor_no_entry(self):
         """Test enhancing an image that has no configuration sections."""
@@ -225,7 +225,8 @@ def test_enhance_with_sensor_no_entry(self):
         self.assertIsNotNone(e.enhancement_tree)
         get_enhanced_image(ds, enhance=e)
         self.assertSetEqual(set(e.sensor_enhancement_configs),
-                            {self.ENH_FN2, self.ENH_ENH_FN2})
+                            {os.path.abspath(self.ENH_FN2),
+                             os.path.abspath(self.ENH_ENH_FN2)})
 
     def test_deprecated_enhance_with_file_specified(self):
         """Test enhancing an image when config file is specified."""
@@ -259,7 +260,8 @@ def test_enhance_with_sensor_entry(self):
         img = get_enhanced_image(ds, enhance=e)
         self.assertSetEqual(
             set(e.sensor_enhancement_configs),
-            {self.ENH_FN, self.ENH_ENH_FN})
+            {os.path.abspath(self.ENH_FN),
+             os.path.abspath(self.ENH_ENH_FN)})
         np.testing.assert_almost_equal(img.data.isel(bands=0).max().values,
                                        1.)
 
@@ -270,7 +272,8 @@ def test_enhance_with_sensor_entry(self):
         self.assertIsNotNone(e.enhancement_tree)
         img = get_enhanced_image(ds, enhance=e)
         self.assertSetEqual(set(e.sensor_enhancement_configs),
-                            {self.ENH_FN, self.ENH_ENH_FN})
+                            {os.path.abspath(self.ENH_FN),
+                             os.path.abspath(self.ENH_ENH_FN)})
         np.testing.assert_almost_equal(img.data.isel(bands=0).max().values, 1.)
 
     def test_enhance_with_sensor_entry2(self):
@@ -285,7 +288,8 @@ def test_enhance_with_sensor_entry2(self):
         self.assertIsNotNone(e.enhancement_tree)
         img = get_enhanced_image(ds, enhance=e)
         self.assertSetEqual(set(e.sensor_enhancement_configs),
-                            {self.ENH_FN, self.ENH_ENH_FN})
+                            {os.path.abspath(self.ENH_FN),
+                             os.path.abspath(self.ENH_ENH_FN)})
         np.testing.assert_almost_equal(img.data.isel(bands=0).max().values, 0.5)