Change writer configs from INI (.cfg) to YAML (#63)

* Change writer configs from INI (.cfg) to YAML

* Add very simple writer tests and fix writer load from Scene

.travis.yml

@@ -7,6 +7,8 @@ python:
 - '3.6'
 os:
 - linux
+before_install:
+- source travis/linux_install.sh
 install:
 - pip install -U pip
 - pip install -U setuptools
@@ -18,10 +20,12 @@ install:
 - pip install h5netcdf
 - pip install python-hdf4
 - "pip install --no-binary :all: netCDF4"
+- pip install gdal==1.10.0
 addons:
   apt_packages:
+    - libgdal-dev
     - libhdf5-serial-dev
-    - libhdf4-dev
+    - libhdf4-alt-dev
     - netcdf-bin
     - libnetcdf-dev
     - cython
@@ -37,7 +41,7 @@ deploy:
   on:
     tags: true
     repo: pytroll/satpy
-sudo: false
+sudo: true
 notifications:
   slack:
     rooms:

satpy/etc/writers/cf.yaml

@@ -0,0 +1,7 @@
+writer:
+  name: cf
+  description: Generic netCDF4/CF Writer
+  writer: !!python/name:satpy.writers.cf_writer.CFWriter
+  file_pattern: '{name}_{start_time:%Y%m%d_%H%M%S}.nc'
+  compress: DEFLATE
+  zlevel: 6

satpy/etc/writers/geotiff.yaml

@@ -0,0 +1,7 @@
+writer:
+  name: geotiff
+  description: Generic GeoTIFF Writer
+  writer: !!python/name:satpy.writers.geotiff.GeoTIFFWriter
+  file_pattern: '{name}_{start_time:%Y%m%d_%H%M%S}.tif'
+  compress: DEFLATE
+  zlevel: 6

satpy/etc/writers/ninjotiff.yaml

@@ -0,0 +1,7 @@
+writer:
+  name: ninjotiff
+  description: NinjoTIFF Writer
+  writer: !!python/name:satpy.writers.ninjotiff.NinjoTIFFWriter
+  file_pattern: '{name}_{start_time:%Y%m%d_%H%M%S}.tif'
+  compress: DEFLATE
+  zlevel: 6

satpy/etc/writers/simple_image.yaml

@@ -0,0 +1,5 @@
+writer:
+  name: simple_image
+  description: Generic Image Writer
+  writer: !!python/name:satpy.writers.simple_image.PillowWriter
+  file_pattern: '{name}_{start_time:%Y%m%d_%H%M%S}.png'

satpy/plugin_base.py

@@ -24,12 +24,13 @@
 
 import logging
 import os
+import yaml
 
-from satpy.config import config_search_paths, get_environ_config_dir
+from satpy.config import config_search_paths, get_environ_config_dir, recursive_dict_update
 
 try:
     import configparser
-except:
+except ImportError:
     from six.moves import configparser
 
 LOG = logging.getLogger(__name__)
@@ -56,29 +57,11 @@ def __init__(self,
         if not isinstance(self.config_files, (list, tuple)):
             self.config_files = [self.config_files]
 
+        self.config = {}
         if self.config_files:
-            conf = configparser.RawConfigParser()
-            conf.read(self.config_files)
-            self.load_config(conf)
-
-    # FIXME: why is this a static method, and not a function ?
-    @staticmethod
-    def _runtime_import(object_path):
-        """Import at runtime
-        """
-        obj_module, obj_element = object_path.rsplit(".", 1)
-        loader = __import__(obj_module, globals(), locals(), [obj_element])
-        return getattr(loader, obj_element)
-
-    def get_section_type(self, section_name):
-        return section_name.split(":")[0]
-
-    def load_config(self, conf):
-        # XXX: Need to load specific object section first if we want to do name-based section filtering
-        # Assumes only one section with "reader:" prefix
-        for section_name in conf.sections():
-            section_type = self.get_section_type(section_name)
-            load_func = "load_section_%s" % (section_type, )
-            if hasattr(self, load_func):
-                getattr(self, load_func)(section_name,
-                                         dict(conf.items(section_name)))
+            for config_file in self.config_files:
+                self.load_yaml_config(config_file)
+
+    def load_yaml_config(self, conf):
+        with open(conf) as fd:
+            self.config = recursive_dict_update(self.config, yaml.load(fd))

satpy/scene.py

@@ -26,10 +26,11 @@
 
 import logging
 import os
+import yaml
 
 from satpy.composites import CompositorLoader, IncompatibleAreas
 from satpy.config import (config_search_paths, get_environ_config_dir,
-                          runtime_import)
+                          runtime_import, recursive_dict_update)
 from satpy.dataset import Dataset, DatasetID, InfoObject
 from satpy.node import DependencyTree
 from satpy.readers import DatasetDict, ReaderFinder
@@ -595,21 +596,15 @@ def images(self):
                 yield projectable.to_image()
 
     def load_writer_config(self, config_files, **kwargs):
-        conf = configparser.RawConfigParser()
-        successes = conf.read(config_files)
-        if not successes:
-            raise IOError("Writer configuration files do not exist: %s" %
-                          (config_files, ))
-
-        for section_name in conf.sections():
-            if section_name.startswith("writer:"):
-                options = dict(conf.items(section_name))
-                writer_class_name = options["writer"]
-                writer_class = runtime_import(writer_class_name)
-                writer = writer_class(ppp_config_dir=self.ppp_config_dir,
-                                      config_files=config_files,
-                                      **kwargs)
-                return writer
+        conf = {}
+        for conf_fn in config_files:
+            with open(conf_fn) as fd:
+                conf = recursive_dict_update(conf, yaml.load(fd))
+        writer_class = conf['writer']['writer']
+        writer = writer_class(ppp_config_dir=self.ppp_config_dir,
+                              config_files=config_files,
+                              **kwargs)
+        return writer
 
     def save_dataset(self, dataset_id, filename=None, writer=None, overlay=None, **kwargs):
         """Save the *dataset_id* to file using *writer* (geotiff by default).
@@ -637,7 +632,7 @@ def save_datasets(self, writer="geotiff", datasets=None, **kwargs):
         writer.save_datasets(datasets, **kwargs)
 
     def get_writer(self, writer="geotiff", **kwargs):
-        config_fn = writer + ".cfg" if "." not in writer else writer
+        config_fn = writer + ".yaml" if "." not in writer else writer
         config_files = config_search_paths(
             os.path.join("writers", config_fn), self.ppp_config_dir)
         kwargs.setdefault("config_files", config_files)

satpy/tests/__init__.py

@@ -28,7 +28,7 @@
 from satpy.tests import (reader_tests, test_dataset, test_file_handlers,
                          test_helper_functions, test_readers, test_resample,
                          test_scene, test_utils, test_writers,
-                         test_yaml_reader)
+                         test_yaml_reader, writer_tests)
 
 if sys.version_info < (2, 7):
     import unittest2 as unittest
@@ -50,6 +50,7 @@ def suite():
     mysuite.addTests(test_yaml_reader.suite())
     mysuite.addTests(test_helper_functions.suite())
     mysuite.addTests(reader_tests.suite())
+    mysuite.addTests(writer_tests.suite())
     mysuite.addTests(test_file_handlers.suite())
     mysuite.addTests(test_utils.suite())
 

satpy/tests/writer_tests/__init__.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2017 David Hoese
+#
+# Author(s):
+#
+#   David Hoese <david.hoese@ssec.wisc.edu>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""The writer tests package.
+"""
+
+import sys
+
+from satpy.tests.writer_tests import (test_cf, test_geotiff,
+                                      test_ninjotiff, test_simple_image)
+
+if sys.version_info < (2, 7):
+    import unittest2 as unittest
+else:
+    import unittest
+
+
+def suite():
+    """Test suite for all writer tests"""
+    mysuite = unittest.TestSuite()
+    # mysuite.addTests(test_cf.suite())
+    mysuite.addTests(test_geotiff.suite())
+    # mysuite.addTests(test_ninjotiff.suite())
+    mysuite.addTests(test_simple_image.suite())
+    return mysuite

satpy/tests/writer_tests/test_cf.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2017 David Hoese
+#
+# Author(s):
+#
+#   David Hoese <david.hoese@ssec.wisc.edu>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Tests for the CF writer.
+"""
+import os
+import sys
+
+import numpy as np
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+if sys.version_info < (2, 7):
+    import unittest2 as unittest
+else:
+    import unittest
+
+
+class TestCFWriter(unittest.TestCase):
+    def test_init(self):
+        from satpy.writers.cf_writer import CFWriter
+        w = CFWriter()
+
+
+def suite():
+    """The test suite for this writer's tests.
+    """
+    loader = unittest.TestLoader()
+    mysuite = unittest.TestSuite()
+    mysuite.addTest(loader.loadTestsFromTestCase(TestCFWriter))
+    return mysuite

satpy/tests/writer_tests/test_geotiff.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2017 David Hoese
+#
+# Author(s):
+#
+#   David Hoese <david.hoese@ssec.wisc.edu>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Tests for the CF writer.
+"""
+import os
+import sys
+
+import numpy as np
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+if sys.version_info < (2, 7):
+    import unittest2 as unittest
+else:
+    import unittest
+
+
+class TestGeoTIFFWriter(unittest.TestCase):
+    def test_init(self):
+        from satpy.writers.geotiff import GeoTIFFWriter
+        w = GeoTIFFWriter()
+
+
+def suite():
+    """The test suite for this writer's tests.
+    """
+    loader = unittest.TestLoader()
+    mysuite = unittest.TestSuite()
+    mysuite.addTest(loader.loadTestsFromTestCase(TestGeoTIFFWriter))
+    return mysuite