From 182a55027258e955d30b389d092d9ba66832b018 Mon Sep 17 00:00:00 2001
From: davidh-ssec <david.hoese@ssec.wisc.edu>
Date: Tue, 16 Jul 2013 22:49:52 -0500
Subject: [PATCH 1/7] added role for wildcard capable CSVConfigReader

---
 py/polar2grid_core/polar2grid/core/roles.py | 145 +++++++++++++++++++-
 py/polar2grid_core/polar2grid/core/test.py  |  84 ++++++++++++
 2 files changed, 227 insertions(+), 2 deletions(-)
 create mode 100644 py/polar2grid_core/polar2grid/core/test.py

diff --git a/py/polar2grid_core/polar2grid/core/roles.py b/py/polar2grid_core/polar2grid/core/roles.py
index 63b26b3a..c38cbe42 100644
--- a/py/polar2grid_core/polar2grid/core/roles.py
+++ b/py/polar2grid_core/polar2grid/core/roles.py
@@ -47,6 +47,7 @@
 import os
 import sys
 import logging
+import re
 from abc import ABCMeta,abstractmethod,abstractproperty
 
 try:
@@ -100,6 +101,148 @@ def __init__(self, callable):
 
 ### End of Copy ###
 
+class CSVConfigReader(object):
+    """Base class for CSV configuration file readers.
+    The format of the file is 1 configuration entry per line. Where the first
+    N elements 'identify' the line in the internal storage of this object.
+
+    Configuration files can be passed into the ``__init__`` method. One or more
+    configuration files can be passed at a time. Individual configuration
+    files can be added via the ``load_config_file`` method. Provided
+    configuration files can be filepaths as strings or a file-like object.
+
+    Wildcard identifying entries are supported by the '*' character. When
+    requesting an entry, the loaded configuration files are searched in the
+    order they were entered and in top-down line order. It is recommended
+    that wildcards go near the bottom of files to avoid conflict.
+
+    Class attributes:
+        NUM_ID_ELEMENTS (default 6): Number of elements (starting from the
+            first) that 'identify' the configuration entry.
+        COMMENT_CHARACTER (default '#'): First character in a line
+            representing a comment. Inline comments are not supported.
+    """
+    __metaclass__ = ABCMeta
+    NUM_ID_ELEMENTS = 6
+    COMMENT_CHARACTER = '#'
+
+    def __init__(self, *config_files, **kwargs):
+        """Initialize configuration reader.
+        Provided configuration files can be filepaths or file-like objects.
+
+        :keyword ignore_bad_lines: Ignore bad configuration lines if encountered
+        :keyword min_num_elements: Minimum number of elements allowed in a config line
+        """
+        self.config_storage = []
+        self.ignore_bad_lines = kwargs.get("ignore_bad_lines", False)
+        self.min_num_elements = kwargs.get("min_num_elements", self.NUM_ID_ELEMENTS)
+        for config_file in config_files:
+            self.load_config_file(config_file)
+
+    def load_config_file(self, config_file):
+        """Load one configuration file into internal storage.
+        """
+        # If we were provided a string filepath then open the file
+        if isinstance(config_file, str):
+            config_file = open(config_file, 'r')
+
+        # Read in each line
+        for line in config_file:
+            # Clean the line
+            line = line.strip()
+            # Ignore comments and blank lines
+            if line.startswith(self.COMMENT_CHARACTER) or line == "":
+                continue
+            # Get each element
+            parts = tuple( x.strip() for x in line.split(",") )
+            # Parse the line
+            self.parse_config_parts(parts)
+
+    def parse_config_parts(self, parts):
+        if len(parts) < self.min_num_elements:
+            log.error("Line does not have correct number of elements: '%s'" % (str(parts),))
+            if self.ignore_bad_lines: return
+            raise ValueError("Line does not have correct number of elements: '%s'" % (str(parts),))
+
+        # Separate the parts into identifying vs configuration parts
+        id_parts = parts[:self.NUM_ID_ELEMENTS]
+        entry_parts = parts[self.NUM_ID_ELEMENTS:]
+
+        # Handle each part separately
+        try:
+            id_regex_obj = self.parse_id_parts(id_parts)
+            entry_info = self.parse_entry_parts(entry_parts)
+        except StandardError:
+            if self.ignore_bad_lines: return
+            raise ValueError("Bad configuration line: '%s'" % (str(parts),))
+
+        self.config_storage.append((id_regex_obj,entry_info))
+
+    def parse_id_parts(self, id_parts):
+        parsed_parts = []
+        for part in id_parts:
+            if part == "None" or part == "none" or part == "":
+                part = ''
+            # If there is a '*' anywhere in the entry, make it a wildcard
+            part = part.replace("*", r'.*')
+            parsed_parts.append(part)
+
+        this_regex = "_".join(parsed_parts)
+
+        try:
+            this_regex_obj = re.compile(this_regex)
+        except re.error:
+            log.error("Invalid configuration identifying information (not valid regular expression): '%s'" % (str(id_parts),))
+            raise ValueError("Invalid configuration identifying information (not valid regular expression): '%s'" % (str(id_parts),))
+
+        return this_regex_obj
+
+    def parse_entry_parts(self, entry_parts):
+        """Method called when loading a configuration entry.
+        The argument passed is a tuple of each configuration entries
+        information loaded from the file.
+
+        This is where a user could convert a tuple to a dictionary with
+        specific key names.
+        """
+        return entry_parts
+
+    def prepare_config_entry(self, entry_info, id_info):
+        """Method called when retrieving a configuration entry to prepare
+        configuration information for the use.
+        The second argument is a tuple of the identifying elements provided
+        during the search. The first argument is whatever object was returned
+        by ``parse_entry_parts`` during configuration loading.
+        
+        This method is used during the retrieval process in
+        case the structure of the entry is based on the specifics of the
+        match. For example, if a wildcard is matched, the configuration
+        information might take on a different meaning based on what matched.
+
+        This is where a user could convert a tuple to a dictionary with
+        specific key names.
+        """
+        return entry_info
+
+    def get_config_entry(self, *args, **kwargs):
+        """Retrieve configuration information.
+        Passed arguments will be matched against loaded configuration
+        entry identities, therefore there must be the same number of elements
+        as ``NUM_ID_ELEMENTS``.
+        """
+        if len(args) != self.NUM_ID_ELEMENTS:
+            log.error("Incorrect number of identifying elements when searching configuration")
+            raise ValueError("Incorrect number of identifying elements when searching configuration")
+
+        search_id = "_".join(args)
+        for regex_pattern,entry_info in self.config_storage:
+            m = regex_pattern.match(search_id)
+            if m is None: continue
+
+            return self.prepare_config_entry(entry_info, args)
+
+        raise ValueError("No config entry found matching: '%s'" % (search_id,))
+
 class RescalerRole(object):
     __metaclass__ = ABCMeta
 
@@ -553,9 +696,7 @@ def remove_all(self):
 def main():
     """Run some tests on the interfaces/roles
     """
-    # TODO
     import doctest
-    print "Running doctests"
     return doctest.testmod()
 
 if __name__ == "__main__":
diff --git a/py/polar2grid_core/polar2grid/core/test.py b/py/polar2grid_core/polar2grid/core/test.py
new file mode 100644
index 00000000..45cf10e0
--- /dev/null
+++ b/py/polar2grid_core/polar2grid/core/test.py
@@ -0,0 +1,84 @@
+"""Module for running unit tests from the polar2grid.core package.
+
+:author: David Hoese, SSEC
+"""
+
+import os
+import sys
+import unittest
+from StringIO import StringIO
+
+from . import roles
+
+
+### Role Tests ###
+
+class TestCSVConfig(unittest.TestCase):
+    string_1 = """# This is a comment
+this,is,item,1,id,id,stuff1,stuff1
+this,  is,   item,  2, id, id, stuff2, stuff2
+"""
+    string_1_diff = """# This is a comment
+this,is,item,1,id,id,stuff1_diff,stuff1_diff
+this,  is,   item,  2, id, id, stuff2_diff, stuff2_diff
+"""
+    string_2 = """# This is a comment
+this,is,item,3,id,id,stuff3,stuff3
+this,  is,   item,  4, id, id, stuff4, stuff4
+# Another comment
+"""
+    string_3 = """# This is a comment
+this,  is,   item,  4, id, id, stuff4, stuff4
+this,is,item,*,id,id,stuffwild,stuffwild
+# Another comment
+"""
+    def test_basic_1(self):
+        """Test that config files can be loaded.
+        """
+        file_obj = StringIO(self.string_1)
+        reader = roles.CSVConfigReader(file_obj)
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "1", "id", "id"), ("stuff1", "stuff1"))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "2", "id", "id"), ("stuff2", "stuff2"))
+
+    def test_basic_2(self):
+        """Test that a config file can be loaded after initialization.
+        """
+        file_obj = StringIO(self.string_1)
+        reader = roles.CSVConfigReader(file_obj)
+        reader.load_config_file(StringIO(self.string_2))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "1", "id", "id"), ("stuff1", "stuff1"))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "2", "id", "id"), ("stuff2", "stuff2"))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "3", "id", "id"), ("stuff3", "stuff3"))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "4", "id", "id"), ("stuff4", "stuff4"))
+
+    def test_repeat_1(self):
+        """Test that config files are handled in order.
+        """
+        reader = roles.CSVConfigReader()
+        reader.load_config_file(StringIO(self.string_1))
+        reader.load_config_file(StringIO(self.string_1_diff))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "1", "id", "id"), ("stuff1", "stuff1"))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "2", "id", "id"), ("stuff2", "stuff2"))
+
+        reader = roles.CSVConfigReader()
+        reader.load_config_file(StringIO(self.string_1_diff))
+        reader.load_config_file(StringIO(self.string_1))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "1", "id", "id"), ("stuff1_diff", "stuff1_diff"))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "2", "id", "id"), ("stuff2_diff", "stuff2_diff"))
+
+    def test_wildcard_1(self):
+        """Test that wild cards work.
+        """
+        reader = roles.CSVConfigReader()
+        reader.load_config_file(StringIO(self.string_3))
+        reader.load_config_file(StringIO(self.string_1))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "1", "id", "id"), ("stuffwild", "stuffwild"))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "2", "id", "id"), ("stuffwild", "stuffwild"))
+        self.assertEqual(reader.get_config_entry("this", "is", "item", "4", "id", "id"), ("stuff4", "stuff4"))
+        pass
+
+def main():
+    return unittest.main()
+
+if __name__ == "__main__":
+    sys.exit(main())

From b86047f3c4751aa15e88034e7f342e279deee2f6 Mon Sep 17 00:00:00 2001
From: davidh-ssec <david.hoese@ssec.wisc.edu>
Date: Wed, 17 Jul 2013 15:25:54 -0500
Subject: [PATCH 2/7] rescale role class now inherits from CSVConfigReader

---
 .../polar2grid/awips/awips_netcdf.py          |   2 +-
 py/polar2grid/polar2grid/binary.py            |   2 +-
 py/polar2grid/polar2grid/gtiff_backend.py     |   2 +-
 .../polar2grid/ninjo/ninjo_backend.py         |   2 +-
 py/polar2grid_core/polar2grid/core/rescale.py |  19 +--
 .../core/rescale_configs/rescale.16bit.conf   |  69 +++-----
 .../core/rescale_configs/rescale.8bit.conf    |  68 +++-----
 .../rescale_configs/rescale_inc.16bit.conf    |  69 +++-----
 .../rescale_configs/rescale_inc.8bit.conf     |  69 +++-----
 py/polar2grid_core/polar2grid/core/roles.py   | 158 ++++++------------
 10 files changed, 156 insertions(+), 304 deletions(-)

diff --git a/py/polar2grid/polar2grid/awips/awips_netcdf.py b/py/polar2grid/polar2grid/awips/awips_netcdf.py
index a967fa78..414d0dd3 100644
--- a/py/polar2grid/polar2grid/awips/awips_netcdf.py
+++ b/py/polar2grid/polar2grid/awips/awips_netcdf.py
@@ -160,7 +160,7 @@ def __init__(self, backend_config=None, rescale_config=None, fill_value=DEFAULT_
         self.rescale_config = rescale_config
         self.fill_in = fill_value
         self.fill_out = DEFAULT_FILL_VALUE
-        self.rescaler = Rescaler(config=self.rescale_config, fill_in=self.fill_in, fill_out=self.fill_out)
+        self.rescaler = Rescaler(self.rescale_config, fill_in=self.fill_in, fill_out=self.fill_out)
 
     def can_handle_inputs(self, sat, instrument, nav_set_uid, kind, band, data_kind):
         """Function for backend-calling script to ask if the backend will be
diff --git a/py/polar2grid/polar2grid/binary.py b/py/polar2grid/polar2grid/binary.py
index 67e74a2b..339f7ec4 100644
--- a/py/polar2grid/polar2grid/binary.py
+++ b/py/polar2grid/polar2grid/binary.py
@@ -92,7 +92,7 @@ def __init__(self, output_pattern=None,
         if rescale_config is None:
             self.rescaler = None
         else:
-            self.rescaler = Rescaler(config=self.rescale_config,
+            self.rescaler = Rescaler(self.rescale_config,
                     fill_in=self.fill_in, fill_out=self.fill_out,
                     inc_by_one=inc_by_one
                     )
diff --git a/py/polar2grid/polar2grid/gtiff_backend.py b/py/polar2grid/polar2grid/gtiff_backend.py
index d2c241f0..3d3aadd5 100644
--- a/py/polar2grid/polar2grid/gtiff_backend.py
+++ b/py/polar2grid/polar2grid/gtiff_backend.py
@@ -192,7 +192,7 @@ def __init__(self, output_pattern=None,
         # Instantiate the rescaler
         self.fill_in = fill_value
         self.fill_out = DEFAULT_FILL_VALUE
-        self.rescaler = Rescaler(config=self.rescale_config,
+        self.rescaler = Rescaler(self.rescale_config,
                 fill_in=self.fill_in, fill_out=self.fill_out,
                 inc_by_one=inc_by_one
                 )
diff --git a/py/polar2grid/polar2grid/ninjo/ninjo_backend.py b/py/polar2grid/polar2grid/ninjo/ninjo_backend.py
index 20e0fbf2..11c5c393 100644
--- a/py/polar2grid/polar2grid/ninjo/ninjo_backend.py
+++ b/py/polar2grid/polar2grid/ninjo/ninjo_backend.py
@@ -479,7 +479,7 @@ def __init__(self,
         # Instatiate the rescaler
         self.fill_in  = fill_value
         self.fill_out = DEFAULT_FILL_VALUE
-        self.rescaler = Rescaler(config=self.rescale_config,
+        self.rescaler = Rescaler(self.rescale_config,
                 fill_in=self.fill_in, fill_out=self.fill_out
                 )
 
diff --git a/py/polar2grid_core/polar2grid/core/rescale.py b/py/polar2grid_core/polar2grid/core/rescale.py
index 0042711e..7fd297a7 100644
--- a/py/polar2grid_core/polar2grid/core/rescale.py
+++ b/py/polar2grid_core/polar2grid/core/rescale.py
@@ -360,22 +360,15 @@ def __call__(self, sat, instrument, nav_set_uid, kind, band, data_kind, data,
         add 1 to the scaled data excluding the invalid values.
         """
         log_level = logging.getLogger('').handlers[0].level or 0
-        band_id = self._create_config_id(sat, instrument, nav_set_uid, kind, band, data_kind)
         fill_in = fill_in or self.fill_in
         fill_out = fill_out or self.fill_out
 
-        if self.config is None or band_id not in self.config:
-            # Run the default scaling functions
-            log.debug("Config ID '%s' was not found in '%r'" % (band_id,self.config.keys()))
-            log.info("Running default rescaling method for kind: %s, band: %s" % (kind,band))
-            if data_kind not in RESCALE_FOR_KIND:
-                log.error("No default rescaling is set for data of kind %s" % data_kind)
-                raise ValueError("No default rescaling is set for data of kind %s" % data_kind)
-            rescale_func,rescale_args = RESCALE_FOR_KIND[data_kind]
-        else:
-            # We know how to rescale using the onfiguration file
-            log.info("'%s' was found in the rescaling configuration" % (band_id))
-            rescale_func,rescale_args = self.config[band_id]
+        try:
+            rescale_func,rescale_args = self.get_config_entry(sat, instrument, nav_set_uid, kind, band, data_kind)
+            log.info("'%r' was found in the rescaling configuration" % ((sat, instrument, nav_set_uid, kind, band, data_kind),))
+        except StandardError:
+            log.error("'%r' was not found in rescaling configuration file" % ((sat, instrument, nav_set_uid, kind, band, data_kind),))
+            raise
 
         # Only perform this calculation if it will be shown, its very time consuming
         if log_level <= logging.DEBUG:
diff --git a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.16bit.conf b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.16bit.conf
index cdf65401..24bac516 100644
--- a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.16bit.conf
+++ b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.16bit.conf
@@ -30,44 +30,26 @@ npp,   viirs,      m_nav,       m,    15,   btemp,       btemp,        242.0,
 npp,   viirs,      m_nav,       m,    16,   btemp,       btemp,        242.0,   169620,  514, 107426, 257
 
 # modis related data; most of these handle similarly to viirs
-# aqua data
-aqua,  modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 6553.5
-aqua,  modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 6553.5
-aqua,  modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 6553.5
-aqua,  modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 6553.5
-aqua,  modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 6553.5
-aqua,  modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 6553.5
-aqua,  modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 169620, 514, 107426, 257
-aqua,  modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 169620, 514, 107426, 257
-aqua,  modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 169620, 514, 107426, 257
-aqua,  modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  1542.0, 35.0
-aqua,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 62965
-aqua,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 62965
-aqua,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    12593.0, 51400, 12850, -1.0, 1.0, 0.0, 65535.0
-aqua,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 64250
-aqua,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  7710.0, 10280.0
-
-# terra data
-terra, modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 6553.5
-terra, modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 6553.5
-terra, modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 6553.5
-terra, modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 6553.5
-terra, modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 6553.5
-terra, modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 6553.5
-terra, modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 169620, 514, 107426, 257
-terra, modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 169620, 514, 107426, 257
-terra, modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 169620, 514, 107426, 257
-terra, modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  1542.0, 8995.0
-terra, modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 62965
-terra, modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 62965
-terra, modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    12593.0, 51400, 12850, -1.0, 1.0, 0.0, 65535.0
-terra, modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 64250
-terra, modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  7710.0, 10280.0
+*,  modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 6553.5
+*,  modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 6553.5
+*,  modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 6553.5
+*,  modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 6553.5
+*,  modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 6553.5
+*,  modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 6553.5
+*,  modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 169620, 514, 107426, 257
+*,  modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 169620, 514, 107426, 257
+*,  modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 169620, 514, 107426, 257
+*,  modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  1542.0, 35.0
+*,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 62965
+*,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 62965
+*,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    12593.0, 51400, 12850, -1.0, 1.0, 0.0, 65535.0
+*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 64250
+*,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  7710.0, 10280.0
 
 # TODO, this is guesswork
-terra, modis,      geo_1000m_nav,     inversion_strength,        none,  btemp,             btemp_c, 242.0, 169620, 514, 107426, 257
-terra, modis,      geo_1000m_nav,     inversion_depth,           none,  distance,          raw
-terra, modis,      geo_1000m_nav,     ice_concentration,         none,  percent,           raw
+*, modis,      geo_1000m_nav,     inversion_strength,        none,  btemp,             btemp_c, 242.0, 169620, 514, 107426, 257
+*, modis,      geo_1000m_nav,     inversion_depth,           none,  distance,          raw
+*, modis,      geo_1000m_nav,     ice_concentration,         none,  percent,           raw
 
 # crefl (applied over range -0.01 to 1.1)
 npp,   viirs,      m_nav,   crefl,    01,   corrected_reflectance, lookup, 59040.54, 590.4054
@@ -75,13 +57,8 @@ npp,   viirs,      m_nav,   crefl,    03,   corrected_reflectance, lookup, 59040
 npp,   viirs,      m_nav,   crefl,    04,   corrected_reflectance, lookup, 59040.54, 590.4054
 npp,   viirs,      i_nav,   crefl,    08,   corrected_reflectance, lookup, 59040.54, 590.4054
 npp,   viirs,      m_nav,   true_color_crefl,    none,   true_color_crefl, lookup, 59040.54, 590.4054
-aqua,  modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 59040.54, 590.4054
-aqua,  modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 59040.54, 590.4054
-aqua,  modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 59040.54, 590.4054
-aqua,  modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 59040.54, 590.4054
-aqua,  modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 59040.54, 590.4054
-terra,  modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 59040.54, 590.4054
-terra,  modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 59040.54, 590.4054
-terra,  modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 59040.54, 590.4054
-terra,  modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 59040.54, 590.4054
-terra,  modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 59040.54, 590.4054
+*,  modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 59040.54, 590.4054
+*,  modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 59040.54, 590.4054
+*,  modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 59040.54, 590.4054
+*,  modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 59040.54, 590.4054
+*,  modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 59040.54, 590.4054
diff --git a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf
index 7f572e28..b8fb6985 100644
--- a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf
+++ b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf
@@ -31,43 +31,26 @@ npp,   viirs,      m_nav,       m,    16,   btemp,       btemp,        242.0, 66
 
 # modis related data; most of these handle similarly to viirs
 # aqua data
-aqua,  modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 25.5
-aqua,  modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 25.5
-aqua,  modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 25.5
-aqua,  modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 25.5
-aqua,  modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 25.5
-aqua,  modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 25.5
-aqua,  modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 660, 2, 418, 1
-aqua,  modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 660, 2, 418, 1
-aqua,  modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 660, 2, 418, 1
-aqua,  modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  6.0, 35.0
-aqua,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 245
-aqua,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 245
-aqua,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    49.0, 200.0, 50.0, -1.0, 1.0, 0.0, 255.0
-aqua,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 250
-aqua,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  30.0, 40.0
-
-# terra data
-terra, modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 25.5
-terra, modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 25.5
-terra, modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 25.5
-terra, modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 25.5
-terra, modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 25.5
-terra, modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 25.5
-terra, modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 660, 2, 418, 1
-terra, modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 660, 2, 418, 1
-terra, modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 660, 2, 418, 1
-terra, modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  6.0, 35.0
-terra, modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 245
-terra, modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 245
-terra, modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    49.0, 200.0, 50.0, -1.0, 1.0, 0.0, 255.0
-terra, modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 250
-terra, modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  30.0, 40.0
+*,  modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 25.5
+*,  modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 25.5
+*,  modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 25.5
+*,  modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 25.5
+*,  modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 25.5
+*,  modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 25.5
+*,  modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 660, 2, 418, 1
+*,  modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 660, 2, 418, 1
+*,  modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 660, 2, 418, 1
+*,  modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  6.0, 35.0
+*,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 245
+*,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 245
+*,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    49.0, 200.0, 50.0, -1.0, 1.0, 0.0, 255.0
+*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 250
+*,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  30.0, 40.0
 
 # TODO, this is guesswork
-terra, modis,      geo_1000m_nav,     inversion_strength,        none,  btemp,             btemp_c, 242.0, 660, 2, 418, 1
-terra, modis,      geo_1000m_nav,     inversion_depth,           none,  distance,          raw
-terra, modis,      geo_1000m_nav,     ice_concentration,         none,  percent,           raw
+*, modis,      geo_1000m_nav,     inversion_strength,        none,  btemp,             btemp_c, 242.0, 660, 2, 418, 1
+*, modis,      geo_1000m_nav,     inversion_depth,           none,  distance,          raw
+*, modis,      geo_1000m_nav,     ice_concentration,         none,  percent,           raw
 
 # crefl (applied over range -0.01 to 1.1)
 npp,   viirs,      m_nav,   crefl,    01,   corrected_reflectance, lookup, 229.73, 2.2973
@@ -75,13 +58,8 @@ npp,   viirs,      m_nav,   crefl,    03,   corrected_reflectance, lookup, 229.7
 npp,   viirs,      m_nav,   crefl,    04,   corrected_reflectance, lookup, 229.73, 2.2973
 npp,   viirs,      i_nav,   crefl,    08,   corrected_reflectance, lookup, 229.73, 2.2973
 npp,   viirs,      m_nav,   true_color_crefl,    none,   true_color_crefl, lookup, 229.73, 2.2973
-aqua,  modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 229.83, 2.2893
-aqua,  modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 229.83, 2.2893
-aqua,  modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 229.83, 2.2893
-aqua,  modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 229.83, 2.2893
-aqua,  modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 229.83, 2.2983
-terra, modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 229.83, 2.2893
-terra, modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 229.83, 2.2893
-terra, modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 229.83, 2.2893
-terra, modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 229.83, 2.2893
-terra, modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 229.83, 2.2983
+*,  modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 229.83, 2.2893
+*,  modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 229.83, 2.2893
+*,  modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 229.83, 2.2893
+*,  modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 229.83, 2.2893
+*,  modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 229.83, 2.2983
diff --git a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.16bit.conf b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.16bit.conf
index bbf6b238..4f1dc6cd 100644
--- a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.16bit.conf
+++ b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.16bit.conf
@@ -30,44 +30,26 @@ npp,   viirs,      m_nav,       m,    15,   btemp,       btemp,        242.0,
 npp,   viirs,      m_nav,       m,    16,   btemp,       btemp,        242.0,   169620,  514, 107422.854, 256.987
 
 # modis related data; most of these handle similarly to viirs
-# aqua data
-aqua,  modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 6553.4
-aqua,  modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 6553.4
-aqua,  modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 6553.4
-aqua,  modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 6553.4
-aqua,  modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 6553.4
-aqua,  modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 6553.4
-aqua,  modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 169620, 514, 107422.854, 256.987
-aqua,  modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 169620, 514, 107422.854, 256.987
-aqua,  modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 169620, 514, 107422.854, 256.987
-aqua,  modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  1542.0, 35.0
-aqua,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 62965
-aqua,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 62965
-aqua,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    12593.0, 51400, 12850, -1.0, 1.0, 0.0, 65534.0
-aqua,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 64250
-aqua,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  7710.0, 10280.0
-
-# terra data
-terra, modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 6553.4
-terra, modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 6553.4
-terra, modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 6553.4
-terra, modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 6553.4
-terra, modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 6553.4
-terra, modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 6553.4
-terra, modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 169620, 514, 107422.854, 256.987
-terra, modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 169620, 514, 107422.854, 256.987
-terra, modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 169620, 514, 107422.854, 256.987
-terra, modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  1542.0, 8995.0
-terra, modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 62965
-terra, modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 62965
-terra, modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    12593.0, 51400, 12850, -1.0, 1.0, 0.0, 65534.0
-terra, modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 64250
-terra, modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  7710.0, 10280.0
+*,  modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 6553.4
+*,  modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 6553.4
+*,  modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 6553.4
+*,  modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 6553.4
+*,  modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 6553.4
+*,  modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 6553.4
+*,  modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 169620, 514, 107422.854, 256.987
+*,  modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 169620, 514, 107422.854, 256.987
+*,  modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 169620, 514, 107422.854, 256.987
+*,  modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  1542.0, 35.0
+*,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 62965
+*,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 62965
+*,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    12593.0, 51400, 12850, -1.0, 1.0, 0.0, 65534.0
+*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 64250
+*,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  7710.0, 10280.0
 
 # TODO, this is guesswork
-terra, modis,      geo_1000m_nav,     inversion_strength,        none,  btemp,             btemp_c, 242.0, 169620, 514, 107426, 257
-terra, modis,      geo_1000m_nav,     inversion_depth,           none,  distance,          raw
-terra, modis,      geo_1000m_nav,     ice_concentration,         none,  percent,           raw
+*, modis,      geo_1000m_nav,     inversion_strength,        none,  btemp,             btemp_c, 242.0, 169620, 514, 107426, 257
+*, modis,      geo_1000m_nav,     inversion_depth,           none,  distance,          raw
+*, modis,      geo_1000m_nav,     ice_concentration,         none,  percent,           raw
 
 # crefl (applied over range -0.01 to 1.1)
 npp,   viirs,      m_nav,   crefl,    01,   corrected_reflectance, lookup, 59039.64, 590.3964
@@ -75,13 +57,8 @@ npp,   viirs,      m_nav,   crefl,    03,   corrected_reflectance, lookup, 59039
 npp,   viirs,      m_nav,   crefl,    04,   corrected_reflectance, lookup, 59039.64, 590.3964
 npp,   viirs,      i_nav,   crefl,    08,   corrected_reflectance, lookup, 59039.64, 590.3964
 npp,   viirs,      m_nav,   true_color_crefl,    none,   true_color_crefl, lookup, 59039.64, 590.3964
-aqua,  modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 59039.64, 590.3964
-aqua,  modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 59039.64, 590.3964
-aqua,  modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 59039.64, 590.3964
-aqua,  modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 59039.64, 590.3964
-aqua,  modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 59039.64, 590.3964
-terra, modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 59039.64, 590.3964
-terra, modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 59039.64, 590.3964
-terra, modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 59039.64, 590.3964
-terra, modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 59039.64, 590.3964
-terra, modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 59039.64, 590.3964
+*,  modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 59039.64, 590.3964
+*,  modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 59039.64, 590.3964
+*,  modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 59039.64, 590.3964
+*,  modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 59039.64, 590.3964
+*,  modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 59039.64, 590.3964
diff --git a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.8bit.conf b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.8bit.conf
index 25837e30..2d5d3877 100644
--- a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.8bit.conf
+++ b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.8bit.conf
@@ -30,44 +30,26 @@ npp,   viirs,      m_nav,       m,    15,   btemp,       btemp,        242.0,660
 npp,   viirs,      m_nav,       m,    16,   btemp,       btemp,        242.0,660,2,414.854,0.987
 
 # modis related data; most of these handle similarly to viirs
-# aqua data
-aqua,  modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 25.4
-aqua,  modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 25.4
-aqua,  modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 25.4
-aqua,  modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 25.4
-aqua,  modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 25.4
-aqua,  modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 25.4
-aqua,  modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 660, 2, 414.854, 0.987
-aqua,  modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 660, 2, 414.854, 0.987
-aqua,  modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 660, 2, 414.854, 0.987
-aqua,  modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  6.0, 35.0
-aqua,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 245
-aqua,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 245
-aqua,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    49.0, 200.0, 50.0, -1.0, 1.0, 0.0, 255.0
-aqua,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 250
-aqua,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  30.0, 40.0
-
-# terra data
-terra, modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 25.4
-terra, modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 25.4
-terra, modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 25.4
-terra, modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 25.4
-terra, modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 25.4
-terra, modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 25.4
-terra, modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 660, 2, 414.854, 0.987
-terra, modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 660, 2, 414.854, 0.987
-terra, modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 660, 2, 414.854, 0.987
-terra, modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  6.0, 35.0
-terra, modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 245
-terra, modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 245
-terra, modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    49.0, 200.0, 50.0, -1.0, 1.0, 0.0, 255.0
-terra, modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 250
-terra, modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  30.0, 40.0
+*,  modis,      geo_250m_nav, visible,                   01,    reflectance,       sqrt,    100.0, 25.4
+*,  modis,      geo_250m_nav, visible,                   02,    reflectance,       sqrt,    100.0, 25.4
+*,  modis,      geo_1000m_nav,     visible,                   01,    reflectance,       sqrt,    100.0, 25.4
+*,  modis,      geo_1000m_nav,     visible,                   02,    reflectance,       sqrt,    100.0, 25.4
+*,  modis,      geo_1000m_nav,     visible,                   07,    reflectance,       sqrt,    100.0, 25.4
+*,  modis,      geo_1000m_nav,     visible,                   26,    reflectance,       sqrt,    100.0, 25.4
+*,  modis,      geo_1000m_nav,     infrared,                  20,    btemp,             btemp,   242.0, 660, 2, 414.854, 0.987
+*,  modis,      geo_1000m_nav,     infrared,                  27,    btemp,             btemp,   242.0, 660, 2, 414.854, 0.987
+*,  modis,      geo_1000m_nav,     infrared,                  31,    btemp,             btemp,   242.0, 660, 2, 414.854, 0.987
+*,  modis,      geo_1000m_nav,     sea_surface_temp,          none,  btemp,             linear,  6.0, 35.0
+*,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 245
+*,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 245
+*,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    49.0, 200.0, 50.0, -1.0, 1.0, 0.0, 255.0
+*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 250
+*,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  30.0, 40.0
 
 # TODO, this is guesswork
-terra, modis,      geo_1000m_nav,     inversion_strength,        none,  btemp,             btemp_c, 242.0, 660, 2, 414.854, 0.987
-terra, modis,      geo_1000m_nav,     inversion_depth,           none,  distance,          raw
-terra, modis,      geo_1000m_nav,     ice_concentration,         none,  percent,           raw
+*, modis,      geo_1000m_nav,     inversion_strength,        none,  btemp,             btemp_c, 242.0, 660, 2, 414.854, 0.987
+*, modis,      geo_1000m_nav,     inversion_depth,           none,  distance,          raw
+*, modis,      geo_1000m_nav,     ice_concentration,         none,  percent,           raw
 
 # crefl (applied over range -0.01 to 1.1)
 npp,   viirs,      m_nav,   crefl,    01,   corrected_reflectance, lookup, 228.83, 2.2883
@@ -75,13 +57,8 @@ npp,   viirs,      m_nav,   crefl,    03,   corrected_reflectance, lookup, 228.8
 npp,   viirs,      m_nav,   crefl,    04,   corrected_reflectance, lookup, 228.83, 2.2883
 npp,   viirs,      i_nav,   crefl,    08,   corrected_reflectance, lookup, 228.83, 2.2883
 npp,   viirs,      m_nav,   true_color_crefl,    none,   true_color_crefl, lookup, 228.83, 2.2883
-aqua,  modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 228.83, 2.2883
-aqua,  modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 228.83, 2.2883
-aqua,  modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 228.83, 2.2883
-aqua,  modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 228.83, 2.2883
-aqua,  modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 228.83, 2.2883
-terra, modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 228.83, 2.2883
-terra, modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 228.83, 2.2883
-terra, modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 228.83, 2.2883
-terra, modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 228.83, 2.2883
-terra, modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 228.83, 2.2883
+*,  modis, geo_1000m_nav,crefl,    01,   corrected_reflectance, lookup, 228.83, 2.2883
+*,  modis, geo_250m_nav, crefl,    01,   corrected_reflectance, lookup, 228.83, 2.2883
+*,  modis, geo_1000m_nav,crefl,    03,   corrected_reflectance, lookup, 228.83, 2.2883
+*,  modis, geo_1000m_nav,crefl,    04,   corrected_reflectance, lookup, 228.83, 2.2883
+*,  modis, geo_1000m_nav,true_color_crefl,    none,   true_color_crefl, lookup, 228.83, 2.2883
diff --git a/py/polar2grid_core/polar2grid/core/roles.py b/py/polar2grid_core/polar2grid/core/roles.py
index c38cbe42..7497a748 100644
--- a/py/polar2grid_core/polar2grid/core/roles.py
+++ b/py/polar2grid_core/polar2grid/core/roles.py
@@ -48,6 +48,7 @@
 import sys
 import logging
 import re
+from StringIO import StringIO
 from abc import ABCMeta,abstractmethod,abstractproperty
 
 try:
@@ -141,10 +142,30 @@ def __init__(self, *config_files, **kwargs):
 
     def load_config_file(self, config_file):
         """Load one configuration file into internal storage.
+
+        If the config_file is a relative path string and can't be found it
+        will be loaded from a package relative location. If it can't be found
+        in the package an exception is raised.
         """
         # If we were provided a string filepath then open the file
         if isinstance(config_file, str):
-            config_file = open(config_file, 'r')
+            if not os.path.isabs(config_file):
+                # Its not an absolute path, lets see if its relative path
+                cwd_config = os.path.join(os.path.curdir, config_file)
+                if os.path.exists(cwd_config):
+                    config_file = cwd_config
+                    config_file = open(config_file, 'r')
+                else:
+                    # they have specified a package provided file
+                    log.info("Loading package provided rescale config: '%s'" % (config_file,))
+                    try:
+                        config_str = get_resource_string(self.__module__, config_file)
+                    except StandardError:
+                        log.error("Rescale config '%s' was not found" % (config_file,))
+                        raise
+                    config_file = StringIO(config_str)
+            else:
+                config_file = open(config_file, 'r')
 
         # Read in each line
         for line in config_file:
@@ -174,7 +195,8 @@ def parse_config_parts(self, parts):
             entry_info = self.parse_entry_parts(entry_parts)
         except StandardError:
             if self.ignore_bad_lines: return
-            raise ValueError("Bad configuration line: '%s'" % (str(parts),))
+            log.error("Bad configuration line: '%s'" % (str(parts),))
+            raise
 
         self.config_storage.append((id_regex_obj,entry_info))
 
@@ -234,7 +256,7 @@ def get_config_entry(self, *args, **kwargs):
             log.error("Incorrect number of identifying elements when searching configuration")
             raise ValueError("Incorrect number of identifying elements when searching configuration")
 
-        search_id = "_".join(args)
+        search_id = "_".join([ (x is not None and x) or "" for x in args ])
         for regex_pattern,entry_info in self.config_storage:
             m = regex_pattern.match(search_id)
             if m is None: continue
@@ -243,16 +265,14 @@ def get_config_entry(self, *args, **kwargs):
 
         raise ValueError("No config entry found matching: '%s'" % (search_id,))
 
-class RescalerRole(object):
+class RescalerRole(CSVConfigReader):
     __metaclass__ = ABCMeta
 
     # Fill values in the input and to set in the output
     DEFAULT_FILL_IN  = DEFAULT_FILL_VALUE
     DEFAULT_FILL_OUT = DEFAULT_FILL_VALUE
 
-    # Dictionary mapping of data identifier to rescaling function and its
-    # arguments
-    config = {}
+    NUM_ID_ELEMENTS = 6
 
     @abstractproperty
     def default_config_dir(self):
@@ -299,110 +319,40 @@ def known_data_kinds(self):
         # Used in configuration reader
         return self._known_data_kinds
 
-    def __init__(self, config=None, fill_in=None, fill_out=None):
+    def __init__(self, *config_files, **kwargs):
         """Load the initial configuration file and any other information
         needed for later rescaling.
         """
-        if config is not None:
-            self.load_config(config)
-
-        self.fill_in = fill_in or self.DEFAULT_FILL_IN
-        self.fill_out = fill_out or self.DEFAULT_FILL_OUT
-
-    def _create_config_id(self, sat, instrument, nav_set_uid, kind, band, data_kind):
-        return "_".join([sat.lower(), instrument.lower(), (nav_set_uid or "").lower(), kind.lower(), (band or "").lower(), data_kind.lower()])
-
-    def load_config_str(self, config_str):
-        """Just in case I want to have a config file stored as a string in
-        the future.
+        self.fill_in = kwargs.pop("fill_in", self.DEFAULT_FILL_IN)
+        self.fill_out = kwargs.pop("fill_out", self.DEFAULT_FILL_OUT)
+        kwargs["min_num_elements"] = kwargs.get("min_num_elements", self.NUM_ID_ELEMENTS+1) # ID + scaling func
+        super(RescalerRole, self).__init__(*config_files, **kwargs)
+
+    def parse_id_parts(self, parts):
+        # Make sure we know the data_kind
+        if parts[5] not in SET_DKINDS:
+            if parts[5] in self.known_data_kinds:
+                parts[5] = self.known_data_kinds[parts[5]]
+            else:
+                log.warning("Rescaling doesn't know the data kind '%s'" % parts[5])
+        parts = super(RescalerRole, self).parse_id_parts(parts)
+        return parts
 
-        """
-        # Get rid of trailing new lines and commas
-        config_lines = [ line.strip(",\n") for line in config_str.split("\n") ]
-        # Get rid of comment lines and blank lines
-        config_lines = [ line for line in config_lines if line and not line.startswith("#") and not line.startswith("\n") ]
-        # Check if we have any useful lines
-        if not config_lines:
-            log.warning("No non-comment lines were found in rescaling configuration")
-            return False
+    def parse_entry_parts(self, parts):
+        # Make sure we know the scale kind
+        if parts[0] not in self.known_rescale_kinds:
+            log.error("Rescaling doesn't know the rescaling kind '%s'" % parts[0])
+            if self.ignore_bad_lines:
+                return
+            raise ValueError("Rescaling doesn't know the rescaling kind '%s'" % parts[0])
+        parts = list(parts)
+        parts[0] = self.known_rescale_kinds[parts[0]]
 
-        try:
-            # Parse config lines
-            for line in config_lines:
-                parts = [ part.strip() for part in line.split(",") ]
-                if len(parts) < 7:
-                    log.error("Rescale config line needs at least 6 columns : '%s'" % (line))
-                    raise ValueError("Rescale config line needs at least 6 columns : '%s'" % (line))
-
-                # Verify that each identifying portion is valid
-                for i in range(7):
-                    assert parts[i],"Field %d can not be empty" % i
-                    # polar2grid demands lowercase fields
-                    parts[i] = parts[i].lower()
-
-                # Convert band if none
-                if parts[2] == '' or parts[2] == "none":
-                    parts[2] = NOT_APPLICABLE
-                if parts[4] == '' or parts[4] == "none":
-                    parts[4] = NOT_APPLICABLE
-                # Make sure we know the data_kind
-                if parts[5] not in SET_DKINDS:
-                    if parts[5] in self.known_data_kinds:
-                        parts[5] = self.known_data_kinds[parts[5]]
-                    else:
-                        log.warning("Rescaling doesn't know the data kind '%s'" % parts[5])
-
-                # Make sure we know the scale kind
-                if parts[6] not in self.known_rescale_kinds:
-                    log.error("Rescaling doesn't know the rescaling kind '%s'" % parts[6])
-                    raise ValueError("Rescaling doesn't know the rescaling kind '%s'" % parts[6])
-                parts[6] = self.known_rescale_kinds[parts[6]]
-                # TODO: Check argument lengths and maybe values per rescale kind 
-
-                # Enter the information into the configs dict
-                line_id = self._create_config_id(*parts[:6])
-                config_entry = (parts[6], tuple(float(x) for x in parts[7:]))
-                self.config[line_id] = config_entry
-        except StandardError:
-            # Clear out the bad config
-            log.warning("Rescaling configuration file could be in a corrupt state")
-            raise
+        # FUTURE: Check argument lengths and maybe values per rescale kind 
 
-        return True
+        parts = super(RescalerRole, self).parse_entry_parts(parts)
 
-    def load_config(self, config_filename):
-        """
-        Load a rescaling configuration file for later use by the `__call__`
-        function.
-
-        If the config isn't an absolute path, it checks the current directory,
-        and if the config can't be found there it is assumed to be relative to
-        the package structure. So entering just the filename will look in the
-        default rescaling configuration location (the package root) for the
-        filename provided.
-        """
-        if not os.path.isabs(config_filename):
-            # Its not an absolute path, lets see if its relative path
-            cwd_config = os.path.join(os.path.curdir, config_filename)
-            if os.path.exists(cwd_config):
-                config_filename = cwd_config
-            else:
-                # they have specified a package provided file
-                log.info("Loading package provided rescale config: '%s'" % (config_filename,))
-                try:
-                    config_str = get_resource_string(self.__module__, config_filename)
-                except StandardError:
-                    log.error("Rescale config '%s' was not found" % (config_filename,))
-                    raise
-                return self.load_config_str(config_str)
-
-        config = os.path.realpath(config_filename)
-
-        log.debug("Using rescaling configuration '%s'" % (config,))
-
-        config_file = open(config, 'r')
-        config_str = config_file.read()
-        return self.load_config_str(config_str)
+        return (parts[0], tuple(float(x) for x in parts[1:]))
 
     @abstractmethod
     def __call__(self, sat, instrument, nav_set_uid, kind, band, data_kind, data):

From 83df37154a85cccd9171b704cc0419d12f550553 Mon Sep 17 00:00:00 2001
From: davidh-ssec <david.hoese@ssec.wisc.edu>
Date: Thu, 18 Jul 2013 16:36:09 -0500
Subject: [PATCH 3/7] removed rescaling defaults from code and added flexible
 linear scaling function

---
 py/polar2grid_core/polar2grid/core/rescale.py | 86 +++++++++++--------
 .../core/rescale_configs/rescale.16bit.conf   |  2 +-
 .../core/rescale_configs/rescale.8bit.conf    |  2 +-
 .../rescale_configs/rescale_inc.16bit.conf    |  2 +-
 .../rescale_configs/rescale_inc.8bit.conf     |  2 +-
 py/polar2grid_core/polar2grid/core/roles.py   | 38 +++-----
 6 files changed, 65 insertions(+), 67 deletions(-)

diff --git a/py/polar2grid_core/polar2grid/core/rescale.py b/py/polar2grid_core/polar2grid/core/rescale.py
index 7fd297a7..422a8a2a 100644
--- a/py/polar2grid_core/polar2grid/core/rescale.py
+++ b/py/polar2grid_core/polar2grid/core/rescale.py
@@ -105,6 +105,50 @@ def passive_scale(img, fill_in=DEFAULT_FILL_IN, fill_out=DEFAULT_FILL_OUT):
     log.debug("Running 'passive_scale'...")
     return img
 
+def linear_flexible_scale(img, min_out, max_out, min_in=None, max_in=None, clip=0, fill_in=DEFAULT_FILL_IN, fill_out=DEFAULT_FILL_OUT):
+    """Flexible linear scaling by specifying what you want output, not the parameters of the linear equation.
+
+    This scaling function stops humans from doing math...let the computers do it.
+
+    - If you aren't sure what the valid limits of your data are, only specify 
+        the min and max output values. The input minimum and maximum will be
+        computed. Note that this could add a considerable amount of time to
+        the calculation.
+    - If you know the limits, specify the output and input ranges.
+    - If you want to flip the data range (ex. -16 to 40 data becomes 237 to 0
+        data) then specify the ranges as needed (ex. min_out=237, max_out=0,
+        min_in=-16, max_in=40). The flip happens automatically.
+    - If the data needs to be clipped to the output range, specify 1 or 0 for
+        the "clip" keyword. Note that most backends will do this to fit the
+        data type of the output format.
+    """
+    log.debug("Running 'linear_flexible_scale' with (min_out: %f, max_out: %f..." % (min_out,max_out))
+    fill_mask = img == fill_in
+
+    min_in = numpy.nanmin(img[~fill_mask]) if min_in is None else min_in
+    max_in = numpy.nanmax(img[~fill_mask]) if max_in is None else max_in
+    if min_in == max_in:
+        # Data doesn't differ...at all
+        log.warning("Data does not differ (min/max are the same), can not scale properly")
+        max_in = min_in + 1.0
+    log.debug("Input minimum: %f, Input maximum: %f" % (min_in,max_in))
+
+    m = (max_out - min_out) / (max_in - min_in)
+    b = min_out - m * min_in
+
+    numpy.multiply(img, m, img)
+    numpy.add(img, b, img)
+
+    if clip:
+        if min_out < max_out:
+            numpy.clip(img, min_out, max_out, out=img)
+        else:
+            numpy.clip(img, max_out, min_out, out=img)
+
+    img[fill_mask] = fill_out
+
+    return img
+
 def sqrt_scale(img, inner_mult, outer_mult, fill_in=DEFAULT_FILL_IN, fill_out=DEFAULT_FILL_OUT):
     log.debug("Running 'sqrt_scale'...")
     mask = img == fill_in
@@ -173,6 +217,7 @@ def bt_scale(img, threshold, high_max, high_mult, low_max, low_mult, fill_in=DEF
     return img
 
 # this method is intended to work on brightness temperatures in Kelvin
+### DEPRECATED ###
 def bt_scale_linear(image,
                     max_in,      min_in,
                     min_out=1.0, max_out=255.0,
@@ -183,31 +228,10 @@ def bt_scale_linear(image,
     in the original image will be set to fill_out in the final image.
     """
     log.debug("Running 'bt_scale_linear'...")
-    
-    # make a mask of where the fill values are for later
-    fill_mask = image == fill_in
-    
-    # set values beyond the bounds to the bounds
-    image[image < min_in] = min_in
-    image[image > max_in] = max_in
-    
-    # shift and scale the values
-    old_range = max_in  - min_in
-    new_range = max_out - min_out
-    # shift the bottom down to zero
-    image     -= min_in
-    # scale from the size of the old range to the new
-    image     *= (new_range / old_range)
-    # reverse the range
-    image     *= -1
-    image     += new_range
-    # shift the bottom back up to the new bottom
-    image     += min_out
-    
-    # set all the fill values to the outgoing fill value
-    image[fill_mask] = fill_out
-    
-    return image
+    log.warning("DEPRECATION: Please use 'linear_flex' instead of 'bt_linear' for rescaling")
+    log.warning("Arguments for bt_linear (A,B,C,D) become (C,D,A,B) for linear_flex")
+
+    return linear_flexible_scale(min_out, max_out, max_in, min_in, clip=1)
 
 def fog_scale(img, m, b, floor, floor_val, ceil, ceil_val, fill_in=DEFAULT_FILL_IN, fill_out=DEFAULT_FILL_OUT):
     """Scale data linearly. Then clip the data to `floor` and `ceil`,
@@ -301,15 +325,6 @@ def ndvi_scale (data,
     
     return data
 
-# DEFAULTS
-RESCALE_FOR_KIND = {
-        DKIND_RADIANCE    : (linear_scale, (255.0,0)),
-        DKIND_REFLECTANCE : (sqrt_scale,   (100.0, 25.5)),
-        DKIND_BTEMP       : (bt_scale,     (242.0,660.0,2,418.0,1)),
-        DKIND_FOG         : (fog_scale,    (10.0,105.0,5,4,205,206))
-        # TODO, add defaults for category, angle, distance, percent, and contiguous index
-        }
-
 class Rescaler(roles.RescalerRole):
     DEFAULT_FILL_IN = DEFAULT_FILL_IN
     DEFAULT_FILL_OUT = DEFAULT_FILL_OUT
@@ -331,12 +346,13 @@ def default_config_dir(self):
                 'btemp'    :  bt_scale,
                 'fog'      :  fog_scale,
                 'btemp_c'  :  bt_scale_c,
-                'btemp_lin':  bt_scale_linear,
+                'btemp_lin':  bt_scale_linear, # DEPRECATED: Use 'linear_flex'
                 'lst'      :  lst_scale,
                 'ndvi'     :  ndvi_scale,
                 'distance' : passive_scale, # TODO, this is wrong... but we'll sort it out later?
                 'percent'  : passive_scale, # TODO, this is wrong, find out what it should be
                 'lookup'   : lookup_scale,
+                'linear_flex' : linear_flexible_scale,
                 }
     @property
     def known_rescale_kinds(self):
diff --git a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.16bit.conf b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.16bit.conf
index 24bac516..86ac8803 100644
--- a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.16bit.conf
+++ b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.16bit.conf
@@ -43,7 +43,7 @@ npp,   viirs,      m_nav,       m,    16,   btemp,       btemp,        242.0,
 *,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 62965
 *,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 62965
 *,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    12593.0, 51400, 12850, -1.0, 1.0, 0.0, 65535.0
-*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 64250
+*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             linear_flex, 10, 64250, 300.26, 173.16, 1
 *,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  7710.0, 10280.0
 
 # TODO, this is guesswork
diff --git a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf
index b8fb6985..f8ec9c24 100644
--- a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf
+++ b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale.8bit.conf
@@ -44,7 +44,7 @@ npp,   viirs,      m_nav,       m,    16,   btemp,       btemp,        242.0, 66
 *,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 245
 *,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 245
 *,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    49.0, 200.0, 50.0, -1.0, 1.0, 0.0, 255.0
-*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 250
+*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             linear_flex, 10, 250, 300.26, 173.16, 1
 *,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  30.0, 40.0
 
 # TODO, this is guesswork
diff --git a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.16bit.conf b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.16bit.conf
index 4f1dc6cd..58f555f2 100644
--- a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.16bit.conf
+++ b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.16bit.conf
@@ -43,7 +43,7 @@ npp,   viirs,      m_nav,       m,    16,   btemp,       btemp,        242.0,
 *,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 62965
 *,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 62965
 *,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    12593.0, 51400, 12850, -1.0, 1.0, 0.0, 65534.0
-*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 64250
+*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             linear_flex, 10, 64250, 300.26, 173.16, 1
 *,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  7710.0, 10280.0
 
 # TODO, this is guesswork
diff --git a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.8bit.conf b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.8bit.conf
index 2d5d3877..84921bf9 100644
--- a/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.8bit.conf
+++ b/py/polar2grid_core/polar2grid/core/rescale_configs/rescale_inc.8bit.conf
@@ -43,7 +43,7 @@ npp,   viirs,      m_nav,       m,    16,   btemp,       btemp,        242.0,660
 *,  modis,      geo_1000m_nav,     land_surface_temp,         none,  btemp,             lst,     233.2, 322.0, 5, 245
 *,  modis,      geo_1000m_nav,     summer_land_surface_temp,  none,  btemp,             lst,     255.4, 344.3, 5, 245
 *,  modis,      geo_1000m_nav,     ndvi,                      none,  contiguous_index,  ndvi,    49.0, 200.0, 50.0, -1.0, 1.0, 0.0, 255.0
-*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             btemp_lin, 300.26, 173.16, 10, 250
+*,  modis,      mod06_nav,   cloud_top_temperature,     none,  btemp,             linear_flex, 10, 250, 300.26, 173.16, 1
 *,  modis,      mod07_nav,   total_precipitable_water,  none,  distance,          linear,  30.0, 40.0
 
 # TODO, this is guesswork
diff --git a/py/polar2grid_core/polar2grid/core/roles.py b/py/polar2grid_core/polar2grid/core/roles.py
index 7497a748..69a8198e 100644
--- a/py/polar2grid_core/polar2grid/core/roles.py
+++ b/py/polar2grid_core/polar2grid/core/roles.py
@@ -289,17 +289,9 @@ def default_config_dir(self):
 
     @abstractproperty
     def known_rescale_kinds(self):
-        """Return a dictionary mapping data_kind constants to scaling
-        function.  This will be used by the configuration file parser
-        to decide what scaling function goes with each data_kind.
-
-        The dictionary should at least have a key for each data_kind constant,
-        but it may also have some common data_kind equivalents that may appear
-        in a configuration file.  For instance, brightness temperatures may be
-        written in the configuration file as 'btemp', 'brightness temperature',
-        'btemperature', etc.  So the dictionary could have one key for each of
-        these variations as well as the constant DKIND_BTEMP, all mapping to
-        the same scaling function.
+        """Return a dictionary mapping rescaling kind to scaling
+        function.  This will be used during configuration file parsing
+        to decide if the line is valid.
 
         A good strategy is to define the dictionary outside this
         function/property and return a pointer to that class attribute.
@@ -308,17 +300,6 @@ def known_rescale_kinds(self):
         """
         return {}
 
-    # Define the dictionary once so it doesn't have to be
-    # allocated/instantiated every time it's used
-    _known_data_kinds = {
-                            'brightnesstemperature': DKIND_BTEMP,
-                        }
-
-    @property
-    def known_data_kinds(self):
-        # Used in configuration reader
-        return self._known_data_kinds
-
     def __init__(self, *config_files, **kwargs):
         """Load the initial configuration file and any other information
         needed for later rescaling.
@@ -330,11 +311,6 @@ def __init__(self, *config_files, **kwargs):
 
     def parse_id_parts(self, parts):
         # Make sure we know the data_kind
-        if parts[5] not in SET_DKINDS:
-            if parts[5] in self.known_data_kinds:
-                parts[5] = self.known_data_kinds[parts[5]]
-            else:
-                log.warning("Rescaling doesn't know the data kind '%s'" % parts[5])
         parts = super(RescalerRole, self).parse_id_parts(parts)
         return parts
 
@@ -351,8 +327,14 @@ def parse_entry_parts(self, parts):
         # FUTURE: Check argument lengths and maybe values per rescale kind 
 
         parts = super(RescalerRole, self).parse_entry_parts(parts)
+        args = []
+        for x in parts[1:]:
+            if x == "none" or x == "None":
+                args.append(None)
+            else:
+                args.append(float(x))
 
-        return (parts[0], tuple(float(x) for x in parts[1:]))
+        return (parts[0], args)
 
     @abstractmethod
     def __call__(self, sat, instrument, nav_set_uid, kind, band, data_kind, data):

From 623daf28958ef0efbefb80ff8ab5aed019f48653 Mon Sep 17 00:00:00 2001
From: davidh-ssec <david.hoese@ssec.wisc.edu>
Date: Thu, 18 Jul 2013 21:33:34 -0500
Subject: [PATCH 4/7] add lower limit clipping to square-root enhancement

---
 py/polar2grid_core/polar2grid/core/rescale.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/py/polar2grid_core/polar2grid/core/rescale.py b/py/polar2grid_core/polar2grid/core/rescale.py
index 422a8a2a..e0fe5e18 100644
--- a/py/polar2grid_core/polar2grid/core/rescale.py
+++ b/py/polar2grid_core/polar2grid/core/rescale.py
@@ -150,9 +150,13 @@ def linear_flexible_scale(img, min_out, max_out, min_in=None, max_in=None, clip=
     return img
 
 def sqrt_scale(img, inner_mult, outer_mult, fill_in=DEFAULT_FILL_IN, fill_out=DEFAULT_FILL_OUT):
+    """Square root enhancement
+
+    Note that any values below zero are clipped to zero before calculations.
+    """
     log.debug("Running 'sqrt_scale'...")
     mask = img == fill_in
-    img[mask] = 0 # For invalids because < 0 cant be sqrted
+    img[ img < 0 ] = 0 # because < 0 cant be sqrted
     numpy.multiply(img, inner_mult, img)
     numpy.sqrt(img, out=img)
     numpy.multiply(img, outer_mult, img)

From e3f83a84dd65282457f2326e856cdf012547e769 Mon Sep 17 00:00:00 2001
From: davidh-ssec <david.hoese@ssec.wisc.edu>
Date: Wed, 24 Jul 2013 17:20:15 -0500
Subject: [PATCH 5/7] made awips config reader be a subclass of the
 CSVConfigReader

---
 .../polar2grid/awips/awips_config.py          |  94 +++++------
 .../polar2grid/awips/awips_grids.conf         | 153 ++++++------------
 .../polar2grid/awips/awips_netcdf.py          |  50 +-----
 py/polar2grid/polar2grid/grids/grids.py       |  17 --
 py/polar2grid_core/polar2grid/core/roles.py   |  49 ++++--
 5 files changed, 132 insertions(+), 231 deletions(-)

diff --git a/py/polar2grid/polar2grid/awips/awips_config.py b/py/polar2grid/polar2grid/awips/awips_config.py
index c1cfed5e..fcd37e50 100644
--- a/py/polar2grid/polar2grid/awips/awips_config.py
+++ b/py/polar2grid/polar2grid/awips/awips_config.py
@@ -43,19 +43,12 @@
 from xml.etree import cElementTree
 from polar2grid.nc import ncml_tag
 from polar2grid.core.constants import NOT_APPLICABLE
+from polar2grid.core import roles
 
 import os
 import sys
 import logging
 
-try:
-    # try getting setuptools/distribute's version of resource retrieval first
-    import pkg_resources
-    get_resource_string = pkg_resources.resource_string
-except ImportError:
-    import pkgutil
-    get_resource_string = pkgutil.get_data
-
 log = logging.getLogger(__name__)
 
 script_dir = os.path.split(os.path.realpath(__file__))[0]
@@ -72,49 +65,6 @@
 CONFIG_DIR  = os.environ.get("AWIPS_CONFIG_DIR", DEFAULT_CONFIG_DIR)
 NCML_DIR    = os.environ.get("AWIPS_NCML_DIR", DEFAULT_NCML_DIR)
 
-# This isn't used anymore, but its a handy function to hold on to
-# This was replaced by the grids/grids.py API which requires grid size to be
-# in the grids.conf file
-def get_shape_from_ncml(fn, var_name):
-    """Returns (rows,cols) of the variable with name
-    `var_name` in the ncml file with filename `fn`.
-    """
-    xml_parser = cElementTree.parse(fn)
-    all_vars = xml_parser.findall(ncml_tag("variable"))
-    important_var = [ elem for elem in all_vars if elem.get("name") == var_name]
-    if len(important_var) != 1:
-        log.error("Couldn't find a variable with name %s in the ncml file %s" % (var_name,fn))
-        raise ValueError("Couldn't find a variable with name %s in the ncml file %s" % (var_name,fn))
-    important_var = important_var[0]
-
-    # Get the shape out
-    shape = important_var.get("shape")
-    if shape is None:
-        log.error("NCML variable %s does not have the required shape attribute" % (var_name,))
-        raise ValueError("NCML variable %s does not have the required shape attribute" % (var_name,))
-
-    # Get dimension names from shape attribute
-    shape_dims = shape.split(" ")
-    if len(shape_dims) != 2:
-        log.error("2 dimensions are required for variable %s" % (var_name,))
-        raise ValueError("2 dimensions are required for variable %s" % (var_name,))
-    rows_name,cols_name = shape_dims
-
-    # Get the dimensions by their names
-    all_dims = xml_parser.findall(ncml_tag("dimension"))
-    important_dims = [ elem for elem in all_dims if elem.get("name") == rows_name or elem.get("name") == cols_name ]
-    if len(important_dims) != 2:
-        log.error("Corrupt NCML file %s, can't find both of %s's dimensions" % (fn, var_name))
-        raise ValueError("Corrupt NCML file %s, can't find both of %s's dimensions" % (fn, var_name))
-
-    if important_dims[0].get("name") == rows_name:
-        rows = int(important_dims[0].get("length"))
-        cols = int(important_dims[1].get("length"))
-    else:
-        rows = int(important_dims[1].get("length"))
-        cols = int(important_dims[0].get("length"))
-    return rows,cols
-
 def _create_config_id(sat, instrument, nav_set_uid, kind, band, data_kind, grid_name=None):
     if grid_name is None:
         # This is used for searching the configs
@@ -243,5 +193,47 @@ def get_awips_info(config_dict, sat, instrument, nav_set_uid, kind, band, data_k
 
     return config_dict[config_id]
 
+class AWIPSConfigReader(roles.CSVConfigReader):
+    """Read an AWIPS Backend Configuration file
+
+    Example:
+    npp,viirs,i_nav,i,01,reflectance,211w,55779608,0.64 um,SSEC,NPP-VIIRS,grid211w.ncml,SSEC_AWIPS_VIIRS-WCONUS_1KM_SVI01_%Y%m%d_%H%M.55779608
+    """
+    NUM_ID_ELEMENTS = 6
+
+    def parse_entry_parts(self, entry_parts):
+        # Parse out the awips specific elements
+        grid_name = entry_parts[0]
+        product_id = entry_parts[1]
+        awips2_channel = entry_parts[2]
+        awips2_source = entry_parts[3]
+        awips2_satellitename = entry_parts[4]
+        ncml_template = _rel_to_abs(entry_parts[5], NCML_DIR)
+        nc_format = entry_parts[6]
+        config_entry = {
+                "grid_name" : grid_name,
+                "product_id" : product_id,
+                "awips2_channel" : awips2_channel,
+                "awips2_source" : awips2_source,
+                "awips2_satellitename" : awips2_satellitename,
+                "ncml_template" : ncml_template,
+                "nc_format" : nc_format
+                }
+        return config_entry
+
+    def get_config_entry(self, *args, **kwargs):
+        if len(args) == self.NUM_ID_ELEMENTS:
+            try:
+                return super(AWIPSConfigReader, self).get_config_entry(*args, **kwargs)
+            except ValueError:
+                log.error("'%s' could not be found in the loaded configuration" % (args,))
+                raise ValueError("'%s' could not be found in the loaded configuration" % (args,))
+        else:
+            for config_info in self.get_all_matching_entries(*args[:-1]):
+                if config_info["grid_name"] == args[-1]:
+                    return config_info
+            log.error("'%s' could not be found in the loaded configuration" % (args,))
+            raise ValueError("'%s' could not be found in the loaded configuration" % (args,))
+
 if __name__ == "__main__":
     sys.exit(0)
diff --git a/py/polar2grid/polar2grid/awips/awips_grids.conf b/py/polar2grid/polar2grid/awips/awips_grids.conf
index 068d7c70..e48e909f 100644
--- a/py/polar2grid/polar2grid/awips/awips_grids.conf
+++ b/py/polar2grid/polar2grid/awips/awips_grids.conf
@@ -12,40 +12,22 @@ npp,   viirs,      dnb_nav,      dnb,  none, radiance,    211e,55779604,DNB,SSEC
 npp,   viirs,      dnb_nav,      dnb,  new,  radiance,    211e,55779606,DNB,SSEC,NPP-VIIRS,grid211e.ncml,SSEC_AWIPS_VIIRS-ECONUS_1KM_SVDNB_TRANS_%Y%m%d_%H%M.55779606
 npp,   viirs,      i_nav,        i,    fog,  fog,         211e,55779605,Fog,SSEC,NPP-VIIRS,grid211e.ncml,SSEC_AWIPS_VIIRS-ECONUS_1KM_SVIFOG_%Y%m%d_%H%M.55779605
 # MODIS bands for grid 211e
-# Aqua MODIS
-aqua,  modis,      geo_250m_nav, visible,                  01,    reflectance,      211e,  7340,  0.64 um,  SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Visible_%Y%m%d_%H%M.7340
-aqua,  modis,      geo_250m_nav, visible,                  02,    reflectance,      211e,  7367,  0.83 um,  SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Vis086_%Y%m%d_%H%M.7367
-aqua,  modis,      geo_1000m_nav,      visible,                  01,    reflectance,      211e,  7340,  0.64 um,  SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Visible_%Y%m%d_%H%M.7340
-aqua,  modis,      geo_1000m_nav,      visible,                  02,    reflectance,      211e,  7367,  0.83 um,  SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Vis086_%Y%m%d_%H%M.7367
-aqua,  modis,      geo_1000m_nav,      visible,                  07,    reflectance,      211e,  7341,  2.1 um,   SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_SnowIce_%Y%m%d_%H%M.7341
-aqua,  modis,      geo_1000m_nav,      visible,                  26,    reflectance,      211e,  7342,  1.38 um,  SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Cirrus_%Y%m%d_%H%M.7342
-aqua,  modis,      geo_1000m_nav,      infrared,                 20,    btemp,            211e,  7343,  3.75 um,  SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_4um_%Y%m%d_%H%M.7343
-aqua,  modis,      geo_1000m_nav,      infrared,                 27,    btemp,            211e,  7344,  6.7 um,   SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_WV_%Y%m%d_%H%M.7344
-aqua,  modis,      geo_1000m_nav,      infrared,                 31,    btemp,            211e,  7345,  11 um,    SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_IR_%Y%m%d_%H%M.7345
-aqua,  modis,      geo_1000m_nav,      sea_surface_temp,         none,  btemp,            211e,  7370,  SST,      SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_SST_%Y%m%d_%H%M.7370
-aqua,  modis,      geo_1000m_nav,      land_surface_temp,        none,  btemp,            211e,  7347,  LST,      SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_LST_%Y%m%d_%H%M.7347
-aqua,  modis,      geo_1000m_nav,      summer_land_surface_temp, none,  btemp,            211e,  7349,  LSTSUM,   SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_LSTSUM_%Y%m%d_%H%M.7349
-aqua,  modis,      geo_1000m_nav,      infrared,                 fog,   fog,              211e,  7346,  Fog,      SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Fog_%Y%m%d_%H%M.7346
-aqua,  modis,      mod06_nav,    cloud_top_temperature,    none,  btemp,            211e,  7362,  CTT,      SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_4KM_CTT_%Y%m%d_%H%M.7362
-aqua,  modis,      geo_1000m_nav,      ndvi,                     none,  contiguous_index, 211e,  7348,  NDVI,     SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_NDVI_%Y%m%d_%H%M.7348
-aqua,  modis,      mod07_nav,    total_precipitable_water, none,  distance,         211e,  7360,  TPW,      SSEC,  MODIS,   grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_4KM_TPW_%Y%m%d_%H%M.7360
-# Terra MODIS
-terra, modis,      geo_250m_nav, visible,                  01,    reflectance,      211e,  7340,  0.64 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Visible_%Y%m%d_%H%M.7340
-terra, modis,      geo_250m_nav, visible,                  02,    reflectance,      211e,  7367,  0.83 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Vis086_%Y%m%d_%H%M.7367
-terra, modis,      geo_1000m_nav,      visible,                  01,    reflectance,      211e,  7340,  0.64 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Visible_%Y%m%d_%H%M.7340
-terra, modis,      geo_1000m_nav,      visible,                  02,    reflectance,      211e,  7367,  0.83 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Vis086_%Y%m%d_%H%M.7367
-terra, modis,      geo_1000m_nav,      visible,                  07,    reflectance,      211e,  7341,  2.1 um,   SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_SnowIce_%Y%m%d_%H%M.7341
-terra, modis,      geo_1000m_nav,      visible,                  26,    reflectance,      211e,  7342,  1.38 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Cirrus_%Y%m%d_%H%M.7342
-terra, modis,      geo_1000m_nav,      infrared,                 20,    btemp,            211e,  7343,  3.75 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_4um_%Y%m%d_%H%M.7343
-terra, modis,      geo_1000m_nav,      infrared,                 27,    btemp,            211e,  7344,  6.7 um,   SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_WV_%Y%m%d_%H%M.7344
-terra, modis,      geo_1000m_nav,      infrared,                 31,    btemp,            211e,  7345,  11 um,    SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_IR_%Y%m%d_%H%M.7345
-terra, modis,      geo_1000m_nav,      sea_surface_temp,         none,  btemp,            211e,  7370,  SST,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_SST_%Y%m%d_%H%M.7370
-terra, modis,      geo_1000m_nav,      land_surface_temp,        none,  btemp,            211e,  7347,  LST,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_LST_%Y%m%d_%H%M.7347
-terra, modis,      geo_1000m_nav,      summer_land_surface_temp, none,  btemp,            211e,  7349,  LSTSUM,   SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_LSTSUM_%Y%m%d_%H%M.7349
-terra, modis,      geo_1000m_nav,      infrared,                 fog,   fog,              211e,  7346,  Fog,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Fog_%Y%m%d_%H%M.7346
-terra, modis,      mod06_nav,    cloud_top_temperature,    none,  btemp,            211e,  7362,  CTT,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_4KM_CTT_%Y%m%d_%H%M.7362
-terra, modis,      geo_1000m_nav,      ndvi,                     none,  contiguous_index, 211e,  7348,  NDVI,     SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_NDVI_%Y%m%d_%H%M.7348
-terra, modis,      mod07_nav,    total_precipitable_water, none,  distance,         211e,  7360,  TPW,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_4KM_TPW_%Y%m%d_%H%M.7360
+*, modis,      geo_250m_nav, visible,                  01,    reflectance,      211e,  7340,  0.64 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Visible_%Y%m%d_%H%M.7340
+*, modis,      geo_250m_nav, visible,                  02,    reflectance,      211e,  7367,  0.83 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Vis086_%Y%m%d_%H%M.7367
+*, modis,      geo_1000m_nav,      visible,                  01,    reflectance,      211e,  7340,  0.64 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Visible_%Y%m%d_%H%M.7340
+*, modis,      geo_1000m_nav,      visible,                  02,    reflectance,      211e,  7367,  0.83 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Vis086_%Y%m%d_%H%M.7367
+*, modis,      geo_1000m_nav,      visible,                  07,    reflectance,      211e,  7341,  2.1 um,   SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_SnowIce_%Y%m%d_%H%M.7341
+*, modis,      geo_1000m_nav,      visible,                  26,    reflectance,      211e,  7342,  1.38 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Cirrus_%Y%m%d_%H%M.7342
+*, modis,      geo_1000m_nav,      infrared,                 20,    btemp,            211e,  7343,  3.75 um,  SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_4um_%Y%m%d_%H%M.7343
+*, modis,      geo_1000m_nav,      infrared,                 27,    btemp,            211e,  7344,  6.7 um,   SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_WV_%Y%m%d_%H%M.7344
+*, modis,      geo_1000m_nav,      infrared,                 31,    btemp,            211e,  7345,  11 um,    SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_IR_%Y%m%d_%H%M.7345
+*, modis,      geo_1000m_nav,      sea_surface_temp,         none,  btemp,            211e,  7370,  SST,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_SST_%Y%m%d_%H%M.7370
+*, modis,      geo_1000m_nav,      land_surface_temp,        none,  btemp,            211e,  7347,  LST,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_LST_%Y%m%d_%H%M.7347
+*, modis,      geo_1000m_nav,      summer_land_surface_temp, none,  btemp,            211e,  7349,  LSTSUM,   SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_LSTSUM_%Y%m%d_%H%M.7349
+*, modis,      geo_1000m_nav,      infrared,                 fog,   fog,              211e,  7346,  Fog,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_Fog_%Y%m%d_%H%M.7346
+*, modis,      mod06_nav,    cloud_top_temperature,    none,  btemp,            211e,  7362,  CTT,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_4KM_CTT_%Y%m%d_%H%M.7362
+*, modis,      geo_1000m_nav,      ndvi,                     none,  contiguous_index, 211e,  7348,  NDVI,     SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_1KM_NDVI_%Y%m%d_%H%M.7348
+*, modis,      mod07_nav,    total_precipitable_water, none,  distance,         211e,  7360,  TPW,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_4KM_TPW_%Y%m%d_%H%M.7360
 
 # TODO, this is missing a lot of information and I'm not planning to do the other versions of these lines until I've got that information
 #terra, modis,      ice_surface_temperature,  none,  btemp,            211e,  0000,  IST,      SSEC,  MODIS,  grid211e.ncml,  SSEC_AWIPS_MODIS_EAST_IST_%Y%m%d_%H%M.0000
@@ -63,41 +45,23 @@ npp,   viirs,      i_nav,        i,    05,   btemp,       211w,55779611,11.0 um,
 npp,   viirs,      dnb_nav,      dnb,  none, radiance,    211w,55779612,DNB,SSEC,NPP-VIIRS,grid211w.ncml,SSEC_AWIPS_VIIRS-WCONUS_1KM_SVDNB_%Y%m%d_%H%M.55779612
 npp,   viirs,      dnb_nav,      dnb,  new,  radiance,    211w,55779614,DNB,SSEC,NPP-VIIRS,grid211w.ncml,SSEC_AWIPS_VIIRS-WCONUS_1KM_SVDNB_TRANS_%Y%m%d_%H%M.55779614
 npp,   viirs,      i_nav,        i,    fog,  fog,         211w,55779613,Fog,SSEC,NPP-VIIRS,grid211w.ncml,SSEC_AWIPS_VIIRS-WCONUS_1KM_SVIFOG_%Y%m%d_%H%M.55779613
-# MODIS bands for this grid
-# Aqua MODIS
-aqua,  modis,      geo_250m_nav, visible,                  01,    reflectance,      211w,  7350,  0.64 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Visible_%Y%m%d_%H%M.7350
-aqua,  modis,      geo_250m_nav, visible,                  02,    reflectance,      211w,  7387,  0.83 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Vis086_%Y%m%d_%H%M.7387
-aqua,  modis,      geo_1000m_nav,      visible,                  01,    reflectance,      211w,  7350,  0.64 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Visible_%Y%m%d_%H%M.7350
-aqua,  modis,      geo_1000m_nav,      visible,                  02,    reflectance,      211w,  7387,  0.83 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Vis086_%Y%m%d_%H%M.7387
-aqua,  modis,      geo_1000m_nav,      visible,                  07,    reflectance,      211w,  7351,  2.1 um,   SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_SnowIce_%Y%m%d_%H%M.7351
-aqua,  modis,      geo_1000m_nav,      visible,                  26,    reflectance,      211w,  7352,  1.38 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Cirrus_%Y%m%d_%H%M.7352
-aqua,  modis,      geo_1000m_nav,      infrared,                 20,    btemp,            211w,  7353,  3.75 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_4um_%Y%m%d_%H%M.7353
-aqua,  modis,      geo_1000m_nav,      infrared,                 27,    btemp,            211w,  7354,  6.7 um,   SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_WV_%Y%m%d_%H%M.7354
-aqua,  modis,      geo_1000m_nav,      infrared,                 31,    btemp,            211w,  7355,  11 um,    SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_IR_%Y%m%d_%H%M.7355
-aqua,  modis,      geo_1000m_nav,      sea_surface_temp,         none,  btemp,            211w,  7371,  SST,      SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_SST_%Y%m%d_%H%M.7371
-aqua,  modis,      geo_1000m_nav,      land_surface_temp,        none,  btemp,            211w,  7357,  LST,      SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_LST_%Y%m%d_%H%M.7357
-aqua,  modis,      geo_1000m_nav,      summer_land_surface_temp, none,  btemp,            211w,  7359,  LSTSUM,   SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_LSTSUM_%Y%m%d_%H%M.7359
-aqua,  modis,      geo_1000m_nav,      infrared,                 fog,   fog,              211w,  7356,  Fog,      SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Fog_%Y%m%d_%H%M.7356
-aqua,  modis,      mod06_nav,    cloud_top_temperature,    none,  btemp,            211w,  7382,  CTT,      SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_4KM_CTT_%Y%m%d_%H%M.7382
-aqua,  modis,      geo_1000m_nav,      ndvi,                     none,  contiguous_index, 211w,  7358,  NDVI,     SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_NDVI_%Y%m%d_%H%M.7358
-aqua,  modis,      mod07_nav,    total_precipitable_water, none,  distance,         211w,  7380,  TPW,      SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_4KM_TPW_%Y%m%d_%H%M.7380
-# Terra MODIS
-terra, modis,      geo_250m_nav, visible,                  01,    reflectance,      211w,  7350,  0.64 um,  SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Visible_%Y%m%d_%H%M.7350
-terra, modis,      geo_250m_nav, visible,                  02,    reflectance,      211w,  7387,  0.83 um,  SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Vis086_%Y%m%d_%H%M.7387
-terra, modis,      geo_1000m_nav,      visible,                  01,    reflectance,      211w,  7350,  0.64 um,  SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Visible_%Y%m%d_%H%M.7350
-terra, modis,      geo_1000m_nav,      visible,                  02,    reflectance,      211w,  7387,  0.83 um,  SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Vis086_%Y%m%d_%H%M.7387
-terra, modis,      geo_1000m_nav,      visible,                  07,    reflectance,      211w,  7351,  2.1 um,   SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_SnowIce_%Y%m%d_%H%M.7351
-terra, modis,      geo_1000m_nav,      visible,                  26,    reflectance,      211w,  7352,  1.38 um,  SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Cirrus_%Y%m%d_%H%M.7352
-terra, modis,      geo_1000m_nav,      infrared,                 20,    btemp,            211w,  7353,  3.75 um,  SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_4um_%Y%m%d_%H%M.7353
-terra, modis,      geo_1000m_nav,      infrared,                 27,    btemp,            211w,  7354,  6.7 um,   SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_WV_%Y%m%d_%H%M.7354
-terra, modis,      geo_1000m_nav,      infrared,                 31,    btemp,            211w,  7355,  11 um,    SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_IR_%Y%m%d_%H%M.7355
-terra, modis,      geo_1000m_nav,      sea_surface_temp,         none,  btemp,            211w,  7371,  SST,      SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_SST_%Y%m%d_%H%M.7371
-terra, modis,      geo_1000m_nav,      land_surface_temp,        none,  btemp,            211w,  7357,  LST,      SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_LST_%Y%m%d_%H%M.7357
-terra, modis,      geo_1000m_nav,      summer_land_surface_temp, none,  btemp,            211w,  7359,  LSTSUM,   SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_LSTSUM_%Y%m%d_%H%M.7359
-terra, modis,      geo_1000m_nav,      infrared,                 fog,   fog,              211w,  7356,  Fog,      SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Fog_%Y%m%d_%H%M.7356
-terra, modis,      mod06_nav,    cloud_top_temperature,    none,  btemp,            211w,  7382,  CTT,      SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_4KM_CTT_%Y%m%d_%H%M.7382
-terra, modis,      geo_1000m_nav,      ndvi,                     none,  contiguous_index, 211w,  7358,  NDVI,     SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_NDVI_%Y%m%d_%H%M.7358
-terra, modis,      mod07_nav,    total_precipitable_water, none,  distance,         211w,  7380,  TPW,      SSEC,  MODIS,  grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_4KM_TPW_%Y%m%d_%H%M.7380
+# MODIS bands for grid 211w
+*,  modis,      geo_250m_nav, visible,                  01,    reflectance,      211w,  7350,  0.64 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Visible_%Y%m%d_%H%M.7350
+*,  modis,      geo_250m_nav, visible,                  02,    reflectance,      211w,  7387,  0.83 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Vis086_%Y%m%d_%H%M.7387
+*,  modis,      geo_1000m_nav,      visible,                  01,    reflectance,      211w,  7350,  0.64 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Visible_%Y%m%d_%H%M.7350
+*,  modis,      geo_1000m_nav,      visible,                  02,    reflectance,      211w,  7387,  0.83 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Vis086_%Y%m%d_%H%M.7387
+*,  modis,      geo_1000m_nav,      visible,                  07,    reflectance,      211w,  7351,  2.1 um,   SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_SnowIce_%Y%m%d_%H%M.7351
+*,  modis,      geo_1000m_nav,      visible,                  26,    reflectance,      211w,  7352,  1.38 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Cirrus_%Y%m%d_%H%M.7352
+*,  modis,      geo_1000m_nav,      infrared,                 20,    btemp,            211w,  7353,  3.75 um,  SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_4um_%Y%m%d_%H%M.7353
+*,  modis,      geo_1000m_nav,      infrared,                 27,    btemp,            211w,  7354,  6.7 um,   SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_WV_%Y%m%d_%H%M.7354
+*,  modis,      geo_1000m_nav,      infrared,                 31,    btemp,            211w,  7355,  11 um,    SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_IR_%Y%m%d_%H%M.7355
+*,  modis,      geo_1000m_nav,      sea_surface_temp,         none,  btemp,            211w,  7371,  SST,      SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_SST_%Y%m%d_%H%M.7371
+*,  modis,      geo_1000m_nav,      land_surface_temp,        none,  btemp,            211w,  7357,  LST,      SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_LST_%Y%m%d_%H%M.7357
+*,  modis,      geo_1000m_nav,      summer_land_surface_temp, none,  btemp,            211w,  7359,  LSTSUM,   SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_LSTSUM_%Y%m%d_%H%M.7359
+*,  modis,      geo_1000m_nav,      infrared,                 fog,   fog,              211w,  7356,  Fog,      SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_Fog_%Y%m%d_%H%M.7356
+*,  modis,      mod06_nav,    cloud_top_temperature,    none,  btemp,            211w,  7382,  CTT,      SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_4KM_CTT_%Y%m%d_%H%M.7382
+*,  modis,      geo_1000m_nav,      ndvi,                     none,  contiguous_index, 211w,  7358,  NDVI,     SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_1KM_NDVI_%Y%m%d_%H%M.7358
+*,  modis,      mod07_nav,    total_precipitable_water, none,  distance,         211w,  7380,  TPW,      SSEC,  MODIS,   grid211w.ncml,  SSEC_AWIPS_MODIS_WEST_4KM_TPW_%Y%m%d_%H%M.7380
 
 #
 ### Grid 203 - Alaska ###
@@ -120,38 +84,21 @@ npp,   viirs,      i_nav,        i,    05,   btemp,       204,55779627,11.0 um,S
 npp,   viirs,      dnb_nav,      dnb,  none, radiance,    204,55779628,DNB,SSEC,NPP-VIIRS,grid204.ncml,SSEC_AWIPS_VIIRS-HI_1KM_SVDNB_%Y%m%d_%H%M.55779628
 npp,   viirs,      dnb_nav,      dnb,  new,  radiance,    204,55779630,DNB,SSEC,NPP-VIIRS,grid204.ncml,SSEC_AWIPS_VIIRS-HI_1KM_SVDNB_TRANS_%Y%m%d_%H%M.55779630
 npp,   viirs,      i_nav,        i,    fog,  fog,         204,55779629,Fog,SSEC,NPP-VIIRS,grid204.ncml,SSEC_AWIPS_VIIRS-HI_1KM_SVIFOG_%Y%m%d_%H%M.55779629
-# Aqua MODIS
-aqua,  modis,      geo_250m_nav, visible,                  01,    reflectance,      204,  55777350,  0.64 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Visible_%Y%m%d_%H%M.55777350
-aqua,  modis,      geo_250m_nav, visible,                  02,    reflectance,      204,  55777387,  0.83 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Vis086_%Y%m%d_%H%M.55777387
-aqua,  modis,      geo_1000m_nav,      visible,                  01,    reflectance,      204,  55777350,  0.64 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Visible_%Y%m%d_%H%M.55777350
-aqua,  modis,      geo_1000m_nav,      visible,                  02,    reflectance,      204,  55777387,  0.83 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Vis086_%Y%m%d_%H%M.55777387
-aqua,  modis,      geo_1000m_nav,      visible,                  07,    reflectance,      204,  55777351,  2.1 um,   SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_SnowIce_%Y%m%d_%H%M.55777351
-aqua,  modis,      geo_1000m_nav,      visible,                  26,    reflectance,      204,  55777352,  1.38 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Cirrus_%Y%m%d_%H%M.55777352
-aqua,  modis,      geo_1000m_nav,      infrared,                 20,    btemp,            204,  55777353,  3.75 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_4um_%Y%m%d_%H%M.55777353
-aqua,  modis,      geo_1000m_nav,      infrared,                 27,    btemp,            204,  55777354,  6.7 um,   SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_WV_%Y%m%d_%H%M.55777354
-aqua,  modis,      geo_1000m_nav,      infrared,                 31,    btemp,            204,  55777355,  11 um,    SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_IR_%Y%m%d_%H%M.55777355
-aqua,  modis,      geo_1000m_nav,      sea_surface_temp,         none,  btemp,            204,  55777371,  SST,      SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_SST_%Y%m%d_%H%M.55777371
-aqua,  modis,      geo_1000m_nav,      land_surface_temp,        none,  btemp,            204,  55777357,  LST,      SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_LST_%Y%m%d_%H%M.55777357
-aqua,  modis,      geo_1000m_nav,      summer_land_surface_temp, none,  btemp,            204,  55777359,  LSTSUM,   SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_LSTSUM_%Y%m%d_%H%M.55777359
-aqua,  modis,      geo_1000m_nav,      infrared,                 fog,   fog,              204,  55777356,  Fog,      SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Fog_%Y%m%d_%H%M.55777356
-aqua,  modis,      mod06_nav,    cloud_top_temperature,    none,  btemp,            204,  55777382,  CTT,      SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_4KM_CTT_%Y%m%d_%H%M.55777382
-aqua,  modis,      geo_1000m_nav,      ndvi,                     none,  contiguous_index, 204,  55777358,  NDVI,     SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_NDVI_%Y%m%d_%H%M.55777358
-aqua,  modis,      mod07_nav,    total_precipitable_water, none,  distance,         204,  55777380,  TPW,      SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_4KM_TPW_%Y%m%d_%H%M.55777380
-# Terra MODIS
-terra, modis,      geo_250m_nav, visible,                  01,    reflectance,      204,  55777350,  0.64 um,  SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Visible_%Y%m%d_%H%M.55777350
-terra, modis,      geo_250m_nav, visible,                  02,    reflectance,      204,  55777387,  0.83 um,  SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Vis086_%Y%m%d_%H%M.55777387
-terra, modis,      geo_1000m_nav,      visible,                  01,    reflectance,      204,  55777350,  0.64 um,  SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Visible_%Y%m%d_%H%M.55777350
-terra, modis,      geo_1000m_nav,      visible,                  02,    reflectance,      204,  55777387,  0.83 um,  SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Vis086_%Y%m%d_%H%M.55777387
-terra, modis,      geo_1000m_nav,      visible,                  07,    reflectance,      204,  55777351,  2.1 um,   SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_SnowIce_%Y%m%d_%H%M.55777351
-terra, modis,      geo_1000m_nav,      visible,                  26,    reflectance,      204,  55777352,  1.38 um,  SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Cirrus_%Y%m%d_%H%M.55777352
-terra, modis,      geo_1000m_nav,      infrared,                 20,    btemp,            204,  55777353,  3.75 um,  SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_4um_%Y%m%d_%H%M.55777353
-terra, modis,      geo_1000m_nav,      infrared,                 27,    btemp,            204,  55777354,  6.7 um,   SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_WV_%Y%m%d_%H%M.55777354
-terra, modis,      geo_1000m_nav,      infrared,                 31,    btemp,            204,  55777355,  11 um,    SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_IR_%Y%m%d_%H%M.55777355
-terra, modis,      geo_1000m_nav,      sea_surface_temp,         none,  btemp,            204,  55777371,  SST,      SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_SST_%Y%m%d_%H%M.55777371
-terra, modis,      geo_1000m_nav,      land_surface_temp,        none,  btemp,            204,  55777357,  LST,      SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_LST_%Y%m%d_%H%M.55777357
-terra, modis,      geo_1000m_nav,      summer_land_surface_temp, none,  btemp,            204,  55777359,  LSTSUM,   SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_LSTSUM_%Y%m%d_%H%M.55777359
-terra, modis,      geo_1000m_nav,      infrared,                 fog,   fog,              204,  55777356,  Fog,      SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Fog_%Y%m%d_%H%M.55777356
-terra, modis,      mod06_nav,    cloud_top_temperature,    none,  btemp,            204,  55777382,  CTT,      SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_4KM_CTT_%Y%m%d_%H%M.55777382
-terra, modis,      geo_1000m_nav,      ndvi,                     none,  contiguous_index, 204,  55777358,  NDVI,     SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_NDVI_%Y%m%d_%H%M.55777358
-terra, modis,      mod07_nav,    total_precipitable_water, none,  distance,         204,  55777380,  TPW,      SSEC,  MODIS,  grid204.ncml,  SSEC_AWIPS_MODIS_HI_4KM_TPW_%Y%m%d_%H%M.55777380
+# MODIS bands for grid 204
+*,  modis,      geo_250m_nav, visible,                  01,    reflectance,      204,  55777350,  0.64 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Visible_%Y%m%d_%H%M.55777350
+*,  modis,      geo_250m_nav, visible,                  02,    reflectance,      204,  55777387,  0.83 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Vis086_%Y%m%d_%H%M.55777387
+*,  modis,      geo_1000m_nav,      visible,                  01,    reflectance,      204,  55777350,  0.64 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Visible_%Y%m%d_%H%M.55777350
+*,  modis,      geo_1000m_nav,      visible,                  02,    reflectance,      204,  55777387,  0.83 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Vis086_%Y%m%d_%H%M.55777387
+*,  modis,      geo_1000m_nav,      visible,                  07,    reflectance,      204,  55777351,  2.1 um,   SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_SnowIce_%Y%m%d_%H%M.55777351
+*,  modis,      geo_1000m_nav,      visible,                  26,    reflectance,      204,  55777352,  1.38 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Cirrus_%Y%m%d_%H%M.55777352
+*,  modis,      geo_1000m_nav,      infrared,                 20,    btemp,            204,  55777353,  3.75 um,  SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_4um_%Y%m%d_%H%M.55777353
+*,  modis,      geo_1000m_nav,      infrared,                 27,    btemp,            204,  55777354,  6.7 um,   SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_WV_%Y%m%d_%H%M.55777354
+*,  modis,      geo_1000m_nav,      infrared,                 31,    btemp,            204,  55777355,  11 um,    SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_IR_%Y%m%d_%H%M.55777355
+*,  modis,      geo_1000m_nav,      sea_surface_temp,         none,  btemp,            204,  55777371,  SST,      SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_SST_%Y%m%d_%H%M.55777371
+*,  modis,      geo_1000m_nav,      land_surface_temp,        none,  btemp,            204,  55777357,  LST,      SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_LST_%Y%m%d_%H%M.55777357
+*,  modis,      geo_1000m_nav,      summer_land_surface_temp, none,  btemp,            204,  55777359,  LSTSUM,   SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_LSTSUM_%Y%m%d_%H%M.55777359
+*,  modis,      geo_1000m_nav,      infrared,                 fog,   fog,              204,  55777356,  Fog,      SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_Fog_%Y%m%d_%H%M.55777356
+*,  modis,      mod06_nav,    cloud_top_temperature,    none,  btemp,            204,  55777382,  CTT,      SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_4KM_CTT_%Y%m%d_%H%M.55777382
+*,  modis,      geo_1000m_nav,      ndvi,                     none,  contiguous_index, 204,  55777358,  NDVI,     SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_1KM_NDVI_%Y%m%d_%H%M.55777358
+*,  modis,      mod07_nav,    total_precipitable_water, none,  distance,         204,  55777380,  TPW,      SSEC,  MODIS,   grid204.ncml,  SSEC_AWIPS_MODIS_HI_4KM_TPW_%Y%m%d_%H%M.55777380
 
diff --git a/py/polar2grid/polar2grid/awips/awips_netcdf.py b/py/polar2grid/polar2grid/awips/awips_netcdf.py
index 414d0dd3..e2d5c52c 100644
--- a/py/polar2grid/polar2grid/awips/awips_netcdf.py
+++ b/py/polar2grid/polar2grid/awips/awips_netcdf.py
@@ -42,53 +42,19 @@
 """
 __docformat__ = "restructuredtext en"
 
-NCDUMP = """# /data/fxa/modis/7380/20111127_0352
-netcdf \20111127_0352 {
-dimensions:
-	y = 1280 ;
-	x = 1100 ;
-variables:
-	double validTime ;
-		validTime:units = "seconds since 1970-1-1 00:00:00.00 0:00" ;
-	byte image(y, x) ;
-		image:long_name = "image" ;
-
-// global attributes:
-		:depictorName = "westConus" ;
-		:projName = "LAMBERT_CONFORMAL" ;
-		:projIndex = 3 ;
-		:lat00 = 54.53548f ;
-		:lon00 = -152.8565f ;
-		:latNxNy = 17.51429f ;
-		:lonNxNy = -92.71996f ;
-		:centralLat = 25.f ;
-		:centralLon = -95.f ;
-		:latDxDy = 39.25658f ;
-		:lonDxDy = -117.4858f ;
-		:dyKm = 3.931511f ;
-		:dxKm = 3.932111f ;
-		:rotation = 25.f ;
-		:xMin = -0.2556496f ;
-		:xMax = 0.01474848f ;
-		:yMin = -0.8768771f ;
-		:yMax = -0.5622397f ;
-}
-"""
-
 from polar2grid.core import Workspace
 from polar2grid.core import roles
 from polar2grid.core.constants import *
 from polar2grid.nc import create_nc_from_ncml
 from polar2grid.core.rescale import Rescaler
 from polar2grid.core.dtype import clip_to_data_type
-from .awips_config import get_awips_info,load_config as load_awips_config,can_handle_inputs as config_can_handle_inputs,CONFIG_FILE as DEFAULT_AWIPS_CONFIG
+from .awips_config import AWIPSConfigReader,CONFIG_FILE as DEFAULT_AWIPS_CONFIG
 
-import os, sys, logging, re
+import os, sys, logging
 import calendar
 from datetime import datetime
 
 log = logging.getLogger(__name__)
-AWIPS_ATTRS = set(re.findall(r'\W:(\w+)', NCDUMP))
 DEFAULT_8BIT_RCONFIG = "rescale_configs/rescale.8bit.conf"
 
 def create_netcdf(nc_name, image, template, start_dt,
@@ -143,24 +109,20 @@ class Backend(roles.BackendRole):
             "SSEC_AWIPS_*"
             ]
 
-    config = {}
     def __init__(self, backend_config=None, rescale_config=None, fill_value=DEFAULT_FILL_VALUE):
         # Load AWIPS backend configuration
         if backend_config is None:
             log.debug("Using default AWIPS configuration: '%s'" % DEFAULT_AWIPS_CONFIG)
             backend_config = DEFAULT_AWIPS_CONFIG
-
-        self.backend_config = backend_config
-        load_awips_config(self.config, self.backend_config)
+        self.awips_config_reader = AWIPSConfigReader(backend_config)
 
         # Load rescaling configuration
         if rescale_config is None:
             log.debug("Using default 8bit rescaling '%s'" % DEFAULT_8BIT_RCONFIG)
             rescale_config = DEFAULT_8BIT_RCONFIG
-        self.rescale_config = rescale_config
         self.fill_in = fill_value
         self.fill_out = DEFAULT_FILL_VALUE
-        self.rescaler = Rescaler(self.rescale_config, fill_in=self.fill_in, fill_out=self.fill_out)
+        self.rescaler = Rescaler(rescale_config, fill_in=self.fill_in, fill_out=self.fill_out)
 
     def can_handle_inputs(self, sat, instrument, nav_set_uid, kind, band, data_kind):
         """Function for backend-calling script to ask if the backend will be
@@ -171,7 +133,7 @@ def can_handle_inputs(self, sat, instrument, nav_set_uid, kind, band, data_kind)
         It is also assumed that rescaling will be able to handle the `data_kind`
         provided.
         """
-        return config_can_handle_inputs(self.config, sat, instrument, nav_set_uid, kind, band, data_kind)
+        return [ config_info["grid_name"] for config_info in self.awips_config_reader.get_all_matching_entries(sat, instrument, nav_set_uid, kind, band, data_kind) ]
 
     def create_product(self, sat, instrument, nav_set_uid, kind, band, data_kind, data,
             start_time=None, end_time=None, grid_name=None,
@@ -189,7 +151,7 @@ def create_product(self, sat, instrument, nav_set_uid, kind, band, data_kind, da
         data = self.rescaler(sat, instrument, nav_set_uid, kind, band, data_kind, data, fill_in=fill_in, fill_out=self.fill_out)
 
         # Get information from the configuration files
-        awips_info = get_awips_info(self.config, sat, instrument, nav_set_uid, kind, band, data_kind, grid_name)
+        awips_info = self.awips_config_reader.get_config_entry(sat, instrument, nav_set_uid, kind, band, data_kind, grid_name)
         # Get the proper output name if it wasn't forced to something else
         if output_filename is None:
             output_filename = start_time.strftime(awips_info["nc_format"])
diff --git a/py/polar2grid/polar2grid/grids/grids.py b/py/polar2grid/polar2grid/grids/grids.py
index 7b824946..9bca9bc5 100644
--- a/py/polar2grid/polar2grid/grids/grids.py
+++ b/py/polar2grid/polar2grid/grids/grids.py
@@ -660,23 +660,6 @@ def get_grid_info(self, grid_name, with_corners=False):
             log.error("Unknown grid '%s'" % (grid_name,))
             raise ValueError("Unknown grid '%s'" % (grid_name,))
 
-    def remove_grid(self, grid_name):
-        """Remove ``grid_name`` from the loaded grid information
-        for this object.
-
-        :raises ValueError: if ``grid_name`` does not exist
-        """
-        if grid_name in self.grid_information:
-            del self.grid_information[grid_name]
-        else:
-            log.error("Unknown grid '%s' can't be removed" % (grid_name,))
-            raise ValueError("Unknown grid '%s' can't be removed" % (grid_name,))
-
-    def remove_all(self):
-        """Remove any loaded grid information from this instance.
-        """
-        self.grid_information = {}
-
     def read_grids_config(self, config_filepath):
         """Read the "grids.conf" file and create dictionaries mapping the
         grid name to the necessary information. There are two dictionaries
diff --git a/py/polar2grid_core/polar2grid/core/roles.py b/py/polar2grid_core/polar2grid/core/roles.py
index 69a8198e..a8fde2d7 100644
--- a/py/polar2grid_core/polar2grid/core/roles.py
+++ b/py/polar2grid_core/polar2grid/core/roles.py
@@ -240,11 +240,18 @@ def prepare_config_entry(self, entry_info, id_info):
         case the structure of the entry is based on the specifics of the
         match. For example, if a wildcard is matched, the configuration
         information might take on a different meaning based on what matched.
+        It is best practice to copy any information that is being provided
+        so it can't be changed by the user.
 
         This is where a user could convert a tuple to a dictionary with
         specific key names.
         """
-        return entry_info
+        if hasattr(entry_info, "copy"):
+            return entry_info.copy()
+        elif isinstance(entry_info, list) or isinstance(entry_info, tuple):
+            return entry_info[:]
+        else:
+            return entry_info
 
     def get_config_entry(self, *args, **kwargs):
         """Retrieve configuration information.
@@ -261,10 +268,34 @@ def get_config_entry(self, *args, **kwargs):
             m = regex_pattern.match(search_id)
             if m is None: continue
 
-            return self.prepare_config_entry(entry_info, args)
+            entry_info = self.prepare_config_entry(entry_info, args)
+            return entry_info
 
         raise ValueError("No config entry found matching: '%s'" % (search_id,))
 
+    def get_all_matching_entries(self, *args, **kwargs):
+        """Retrieve configuration information.
+        Passed arguments will be matched against loaded configuration
+        entry identities, therefore there must be the same number of elements
+        as ``NUM_ID_ELEMENTS``.
+        """
+        if len(args) != self.NUM_ID_ELEMENTS:
+            log.error("Incorrect number of identifying elements when searching configuration")
+            raise ValueError("Incorrect number of identifying elements when searching configuration")
+
+        search_id = "_".join([ (x is not None and x) or "" for x in args ])
+        matching_entries = []
+        for regex_pattern,entry_info in self.config_storage:
+            m = regex_pattern.match(search_id)
+            if m is None: continue
+
+            matching_entries.append(self.prepare_config_entry(entry_info, args))
+
+        if len(matching_entries) != 0:
+            return matching_entries
+        else:
+            raise ValueError("No config entry found matching: '%s'" % (search_id,))
+
 class RescalerRole(CSVConfigReader):
     __metaclass__ = ABCMeta
 
@@ -611,20 +642,6 @@ def add_grid_config(self, grid_config_filename):
         """
         raise NotImplementedError("Child class must implement this method")
 
-    @abstractmethod
-    def remove_grid(self, grid_name):
-        """Remove ``grid_name`` from the internal grid information storage
-        of this object.
-        """
-        raise NotImplementedError("Child class must implement this method")
-
-    @abstractmethod
-    def remove_all(self):
-        """Remove all grids from the internal grid information storage of
-        this object.
-        """
-        raise NotImplementedError("Child class must implement this method")
-
 def main():
     """Run some tests on the interfaces/roles
     """

From 311b224654335f257992a1a5129727819b4c2cdf Mon Sep 17 00:00:00 2001
From: davidh-ssec <david.hoese@ssec.wisc.edu>
Date: Wed, 22 Jan 2014 11:34:22 -0600
Subject: [PATCH 6/7] Made custom P2GProj to make use easier. Needs testing and
 reorganization.

---
 py/misc/gridloc.py                            |  9 +--
 py/polar2grid/polar2grid/crefl2gtiff.py       |  8 +--
 .../polar2grid/grids/config_helper.py         |  4 +-
 py/polar2grid/polar2grid/grids/grids.conf     |  6 +-
 py/polar2grid/polar2grid/grids/grids.py       | 58 +++++++++++++++----
 py/polar2grid/polar2grid/ll2cr.py             | 35 +++--------
 py/polar2grid/polar2grid/p2g_proj.py          |  4 +-
 7 files changed, 67 insertions(+), 57 deletions(-)

diff --git a/py/misc/gridloc.py b/py/misc/gridloc.py
index 9b6ff7da..752b45a6 100644
--- a/py/misc/gridloc.py
+++ b/py/misc/gridloc.py
@@ -11,7 +11,7 @@
 """
 __docformat__ = "restructuredtext en"
 
-from pyproj import Proj
+from polar2grid.grids.grids import P2GProj
 from polar2grid.core import Workspace
 import numpy
 
@@ -42,12 +42,7 @@ def create_nav_binaries(lon_fn, lat_fn, proj4_str, width, height,
     """Create longitude and latitude binaries from the projection definition
     provided.
     """
-    # Handle EPSG codes properly
-    if "EPSG" in proj4_str:
-        p = Proj(init=proj4_str)
-    else:
-        p = Proj(proj4_str)
-
+    p = Proj(proj4_str)
 
     # Open the files and a memory mapped array
     lon_file = numpy.memmap(lon_fn, dtype=numpy.float32, mode="w+", shape=(abs(height),abs(width)))
diff --git a/py/polar2grid/polar2grid/crefl2gtiff.py b/py/polar2grid/polar2grid/crefl2gtiff.py
index 1cf0b254..d2c9a202 100644
--- a/py/polar2grid/polar2grid/crefl2gtiff.py
+++ b/py/polar2grid/polar2grid/crefl2gtiff.py
@@ -168,10 +168,10 @@ def process_data_sets(nav_set_dict,
                                 band_rep = nav_set_jobs[MBAND_NAV_UID]["remapped_jobs"][grid_name].values()[0]
                             else:
                                 band_rep = nav_set_jobs[GEO_NAV_UID]["remapped_jobs"][grid_name].values()[0]
-                            band_info["grid_info"]["pixel_size_x"] = band_rep["pixel_size_x_grid_units"]
-                            band_info["grid_info"]["pixel_size_y"] = band_rep["pixel_size_y_grid_units"]
-                            band_info["grid_info"]["grid_origin_x"] = band_rep["grid_origin_x_grid_units"]
-                            band_info["grid_info"]["grid_origin_y"] = band_rep["grid_origin_y_grid_units"]
+                            band_info["grid_info"]["pixel_size_x"] = band_rep["pixel_size_x"]
+                            band_info["grid_info"]["pixel_size_y"] = band_rep["pixel_size_y"]
+                            band_info["grid_info"]["grid_origin_x"] = band_rep["grid_origin_x"]
+                            band_info["grid_info"]["grid_origin_y"] = band_rep["grid_origin_y"]
                             band_info["grid_info"]["grid_width"] = band_rep["grid_width"]
                             band_info["grid_info"]["grid_height"] = band_rep["grid_height"]
 
diff --git a/py/polar2grid/polar2grid/grids/config_helper.py b/py/polar2grid/polar2grid/grids/config_helper.py
index 394380a0..a7d31858 100644
--- a/py/polar2grid/polar2grid/grids/config_helper.py
+++ b/py/polar2grid/polar2grid/grids/config_helper.py
@@ -43,7 +43,7 @@
 """
 __docformat__ = "restructuredtext en"
 
-from pyproj import Proj
+from .grids import P2GProj
 
 import os
 import sys
@@ -112,7 +112,7 @@ def main():
 
     proj_str = args.proj_str or determine_projection(clon, clat)
     proj_str = proj_str % {"center_lon":clon, "center_lat":clat}
-    p = Proj(proj_str)
+    p = P2GProj(proj_str)
     origin_x = grid_width / 2.0 * pixel_size_x * -1
     origin_y = grid_height / 2.0 * pixel_size_y * -1
 
diff --git a/py/polar2grid/polar2grid/grids/grids.conf b/py/polar2grid/polar2grid/grids/grids.conf
index a6e00d57..71073d6a 100644
--- a/py/polar2grid/polar2grid/grids/grids.conf
+++ b/py/polar2grid/polar2grid/grids/grids.conf
@@ -13,11 +13,11 @@ australia2,             gpd,   australia2.gpd,       105.000,  5.000,  175.000,
 # proj4 grids may have None for sizes, origins, or pixel sizes to be considered 'dynamic'
 # pixel size or grid size must be specified
 # grid_name,            proj4, proj4_str,                                                                                width,  height, pixel_size_x, pixel_size_y,           origin_x,          origin_y
-p4_211e,                proj4, +proj=lcc +datum=NAD83 +ellps=GRS80 +lat_0=25 +lat_1=25 +lon_0=-95 +units=m +no_defs,      5120,    5120,       1015.9,      -1015.9, -1956254.806724622, 4364276.201489102
-p4_211e_hi,             proj4, +proj=lcc +datum=NAD83 +ellps=GRS80 +lat_0=25 +lat_1=25 +lon_0=-95 +units=m +no_defs,     10000,   10000,          500,         -500, -1956254.806724622, 4364276.201489102
+p4_211e,                proj4, +proj=lcc +datum=NAD83 +ellps=GRS80 +lat_0=25 +lat_1=25 +lon_0=-95 +units=m +no_defs,      5120,    5120,       1015.9,      -1015.9,         -122.94deg,          59.86deg
+p4_211e_hi,             proj4, +proj=lcc +datum=NAD83 +ellps=GRS80 +lat_0=25 +lat_1=25 +lon_0=-95 +units=m +no_defs,     10000,   10000,          500,         -500,         -122.94deg,          59.86deg
 lcc_fit,                proj4, +proj=lcc +datum=WGS84 +ellps=WGS84 +lat_0=25 +lat_1=25 +lon_0=-95 +units=m +no_defs,      None,    None,         1000,        -1000,               None,              None
 lcc_fit_hr,             proj4, +proj=lcc +datum=WGS84 +ellps=WGS84 +lat_0=25 +lat_1=25 +lon_0=-95 +units=m +no_defs,      None,    None,          400,         -400,               None,              None
-wgs84_fit,              proj4, +proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs,                                          None,    None,       0.0001,      -0.0001,               None,              None
+wgs84_fit,              proj4, +proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs,                                          None,    None,        0.006,       -0.006,               None,              None
 eqc_fit,                proj4, +proj=eqc +datum=WGS84 +ellps=WGS84 +lat_ts=0 +lon_0=-100 +units=m +no_defs,               None,    None,          250,         -250,               None,              None
 polar_canada,           proj4, +proj=stere +datum=WGS84 +ellps=WGS84 +lat_0=90 +lat_ts=45.0 +lon_0=-150 +units=m,         None,    None,         1000,        -1000,               None,              None
 polar_north_pacific,    proj4, +proj=stere +datum=WGS84 +ellps=WGS84 +lat_0=90 +lat_ts=45.0 +lon_0=-170 +units=m,         None,    None,          400,         -400,               None,              None
diff --git a/py/polar2grid/polar2grid/grids/grids.py b/py/polar2grid/polar2grid/grids/grids.py
index 9bca9bc5..58433122 100644
--- a/py/polar2grid/polar2grid/grids/grids.py
+++ b/py/polar2grid/polar2grid/grids/grids.py
@@ -70,6 +70,14 @@
 GRIDS_CONFIG_FILEPATH   = os.environ.get("POLAR2GRID_GRIDS_CONFIG", "grids.conf")
 GRID_COVERAGE_THRESHOLD = float(os.environ.get("POLAR2GRID_GRID_COVERAGE", "0.1")) # 10%
 
+
+class P2GProj(pyproj.Proj):
+    def __call__(self, data1, data2, **kwargs):
+        if self.is_latlong():
+            return data1, data2
+
+        return super(P2GProj, self).__call__(data1, data2, **kwargs)
+
 ### GPD Reading Functions ###
 def clean_string(s):
     s = s.replace("-", "")
@@ -161,14 +169,6 @@ def parse_gpd_file(gpd_filepath):
 
 ### Configuration file functions ###
 
-def _load_proj_string(proj_str):
-    """Wrapper to accept epsg strings or proj4 strings
-    """
-    if proj_str[:4].lower() == "epsg":
-        return pyproj.Proj(init=proj_str)
-    else:
-        return pyproj.Proj(proj_str)
-
 def parse_gpd_config_line(grid_name, parts):
     """Return a dictionary of information for a specific GPD grid from
     a grid configuration line. ``parts`` should be every comma-separated
@@ -206,6 +206,23 @@ def parse_gpd_config_line(grid_name, parts):
 
     return info
 
+def _parse_meter_degree_param(param):
+    """Parse a configuration parameter that could be meters or degrees.
+
+    Degrees are denoted with a suffix of 'deg'. Meters are denoted with a suffix of either 'm' or no suffix at all.
+
+    :returns: (float param, True if degrees/False if meters)
+    """
+    convert_to_meters = False
+    if param.endswith("deg"):
+        # Parameter is in degrees
+        convert_to_meters = True
+        param = param[:-3]
+    elif param.endswith("m"):
+        # Parameter is in meters
+        param = param[:-1]
+    return float(param), convert_to_meters
+
 def parse_proj4_config_line(grid_name, parts):
     """Return a dictionary of information for a specific PROJ.4 grid from
     a grid configuration line. ``parts`` should be every comma-separated
@@ -216,7 +233,7 @@ def parse_proj4_config_line(grid_name, parts):
     proj4_str = parts[2]
     # Test to make sure the proj4_str is valid in pyproj's eyes
     try:
-        p = _load_proj_string(proj4_str)
+        p = P2GProj(proj4_str)
         del p
     except StandardError:
         log.error("Invalid proj4 string in '%s' : '%s'" % (grid_name,proj4_str))
@@ -249,17 +266,19 @@ def parse_proj4_config_line(grid_name, parts):
         else:
             pixel_size_y  = float(parts[6])
 
+        convert_xorigin_to_meters = False
         if parts[7] == "None" or parts[7] == '':
             static        = False
             grid_origin_x = None
         else:
-            grid_origin_x = float(parts[7])
+            grid_origin_x, convert_xorigin_to_meters = _parse_meter_degree_param(parts[7])
 
+        convert_yorigin_to_meters = False
         if parts[8] == "None" or parts[8] == '':
             static        = False
             grid_origin_y = None
         else:
-            grid_origin_y = float(parts[8])
+            grid_origin_y, convert_yorigin_to_meters = _parse_meter_degree_param(parts[8])
     except StandardError:
         log.error("Could not parse proj4 grid configuration: '%s'" % (grid_name,))
         raise
@@ -279,6 +298,21 @@ def parse_proj4_config_line(grid_name, parts):
     if grid_width is None and pixel_size_x is None:
         log.error("Either grid size or pixel size must be specified for '%s'" % grid_name)
         raise ValueError("Either grid size or pixel size must be specified for '%s'" % grid_name)
+    if convert_xorigin_to_meters != convert_yorigin_to_meters:
+        log.error("Grid origin parameters must be in the same units (meters vs degrees)")
+        raise ValueError("Grid origin parameters must be in the same units (meters vs degrees)")
+
+    # Convert any parameters from degrees to meters (we already made sure both need to be converted above)
+    p = P2GProj(proj4_str)
+    if convert_xorigin_to_meters and not p.is_latlong():
+        meters_x, meters_y = p(grid_origin_x, grid_origin_y)
+        log.info("Converted grid '%s' origin from (lon: %f, lat: %f) to (x: %f, y: %f)",
+                 grid_name, grid_origin_x, grid_origin_y, meters_x, meters_y)
+        grid_origin_x, grid_origin_y = meters_x, meters_y
+    elif not convert_xorigin_to_meters and (grid_origin_x is not None and p.is_latlong()):
+        log.error("Lat/Lon grid '%s' must have its origin in degrees", grid_name)
+        raise ValueError("Lat/Lon grid '%s' must have its origin in degrees" % (grid_name,))
+
 
     info["grid_kind"]         = GRID_KIND_PROJ4
     info["static"]            = static
@@ -628,7 +662,7 @@ def calculate_grid_corners(self, grid_name):
         if not grid_info["static"]:
             log.debug("Won't calculate corners for a dynamic grid: '%s'" % (grid_name,))
 
-        p = pyproj.Proj(grid_info["proj4_str"])
+        p = P2GProj(grid_info["proj4_str"])
         right_x  = grid_info["grid_origin_x"] + grid_info["pixel_size_x"] * grid_info["grid_width"]
         bottom_y = grid_info["grid_origin_y"] + grid_info["pixel_size_y"] * grid_info["grid_height"]
         grid_info["ul_corner"] = p(grid_info["grid_origin_x"], grid_info["grid_origin_y"], inverse=True)
diff --git a/py/polar2grid/polar2grid/ll2cr.py b/py/polar2grid/polar2grid/ll2cr.py
index 24458958..86672704 100644
--- a/py/polar2grid/polar2grid/ll2cr.py
+++ b/py/polar2grid/polar2grid/ll2cr.py
@@ -43,7 +43,7 @@
 
 from polar2grid.core import Workspace
 from polar2grid.core.constants import DEFAULT_FILL_VALUE
-import pyproj
+from .grids.grids import P2GProj
 import numpy
 
 import os
@@ -130,10 +130,7 @@ def ll2cr(lon_arr, lat_arr, proj4_str,
         raise ValueError("Either pixel size or grid width/height must be specified")
 
     # Handle EPSG codes
-    if proj4_str[:4].lower() == "epsg":
-        tformer = pyproj.Proj(init=proj4_str)
-    else:
-        tformer = pyproj.Proj(proj4_str)
+    tformer = P2GProj(proj4_str)
 
     if dtype is None:
         dtype = lat_arr.dtype
@@ -257,30 +254,12 @@ def ll2cr(lon_arr, lat_arr, proj4_str,
 
     ### Handle special cases for certain projections ###
     ll2cr_info = {}
-    if "latlong" in proj4_str:
-        # Everyone else uses degrees, not radians
-        ll2cr_info["grid_origin_x"],ll2cr_info["grid_origin_y"] = tformer(grid_origin_x,grid_origin_y, inverse=True)
-        ll2cr_info["grid_origin_x_grid_units"] = grid_origin_x
-        ll2cr_info["grid_origin_y_grid_units"] = grid_origin_y
-    else:
-        ll2cr_info["grid_origin_x"] = grid_origin_x
-        ll2cr_info["grid_origin_y"] = grid_origin_y
-        ll2cr_info["grid_origin_x_grid_units"] = grid_origin_x
-        ll2cr_info["grid_origin_y_grid_units"] = grid_origin_y
+    ll2cr_info["grid_origin_x"] = grid_origin_x
+    ll2cr_info["grid_origin_y"] = grid_origin_y
     ll2cr_info["grid_width"] = grid_width
     ll2cr_info["grid_height"] = grid_height
-    if "latlong" in proj4_str:
-        # Everyone else uses degrees, not radians
-        x,y = tformer(pixel_size_x, pixel_size_y, inverse=True)
-        ll2cr_info["pixel_size_x"] = x
-        ll2cr_info["pixel_size_y"] = y
-        ll2cr_info["pixel_size_x_grid_units"] = pixel_size_x
-        ll2cr_info["pixel_size_y_grid_units"] = pixel_size_y
-    else:
-        ll2cr_info["pixel_size_x"] = pixel_size_x
-        ll2cr_info["pixel_size_y"] = pixel_size_y
-        ll2cr_info["pixel_size_x_grid_units"] = pixel_size_y
-        ll2cr_info["pixel_size_y_grid_units"] = pixel_size_y
+    ll2cr_info["pixel_size_x"] = pixel_size_x
+    ll2cr_info["pixel_size_y"] = pixel_size_y
     ll2cr_info["rows_filename"] = rows_fn
     ll2cr_info["cols_filename"] = cols_fn
 
@@ -288,6 +267,8 @@ def ll2cr(lon_arr, lat_arr, proj4_str,
     # Go per row to save on memory, disk load
     for idx in range(lon_arr.shape[0]):
         x_tmp,y_tmp = _transform_array(tformer, lon_arr[idx], lat_arr[idx], proj_circum, stradles_anti=stradles_anti)
+        x_tmp = x_tmp.copy()
+        y_tmp = y_tmp.copy()
         numpy.subtract(x_tmp, grid_origin_x, x_tmp)
         numpy.divide(x_tmp, pixel_size_x, x_tmp)
         numpy.subtract(y_tmp, grid_origin_y, y_tmp)
diff --git a/py/polar2grid/polar2grid/p2g_proj.py b/py/polar2grid/polar2grid/p2g_proj.py
index 89e08015..7e726c9a 100644
--- a/py/polar2grid/polar2grid/p2g_proj.py
+++ b/py/polar2grid/polar2grid/p2g_proj.py
@@ -43,7 +43,7 @@
 """
 __docformat__ = "restructuredtext en"
 
-from pyproj import Proj
+from .grids.grids import P2GProj
 
 import os
 import sys
@@ -61,7 +61,7 @@ def main():
             help="Latitude of the point to be converted (single value only)")
     args = parser.parse_args()
 
-    p = Proj(args.proj4_str)
+    p = P2GProj(args.proj4_str)
     x,y = p(args.lon_point, args.lat_point, inverse=args.inv)
     print x,y
 

From ebf2930a7857086fa799c530d5073377ed096e8d Mon Sep 17 00:00:00 2001
From: davidh-ssec <david.hoese@ssec.wisc.edu>
Date: Wed, 12 Mar 2014 13:25:13 -0500
Subject: [PATCH 7/7] Moved new Proj class to polar2grid.proj (required
 rename). Changed wgs84_fit grid to be closer to old pixel size.

---
 py/misc/gridloc.py                            |  3 ++-
 .../polar2grid/grids/config_helper.py         |  6 +++---
 py/polar2grid/polar2grid/grids/grids.conf     |  2 +-
 py/polar2grid/polar2grid/grids/grids.py       | 16 +++++-----------
 py/polar2grid/polar2grid/ll2cr.py             |  5 +++--
 .../polar2grid/{p2g_proj.py => proj.py}       | 19 +++++++++++++------
 swbundle/p2g_proj.sh                          |  2 +-
 7 files changed, 28 insertions(+), 25 deletions(-)
 rename py/polar2grid/polar2grid/{p2g_proj.py => proj.py} (88%)

diff --git a/py/misc/gridloc.py b/py/misc/gridloc.py
index 752b45a6..dda975e5 100644
--- a/py/misc/gridloc.py
+++ b/py/misc/gridloc.py
@@ -9,9 +9,10 @@
 :date:         Dec 2012
 :license:      GNU GPLv3
 """
+from polar2grid.proj import Proj
+
 __docformat__ = "restructuredtext en"
 
-from polar2grid.grids.grids import P2GProj
 from polar2grid.core import Workspace
 import numpy
 
diff --git a/py/polar2grid/polar2grid/grids/config_helper.py b/py/polar2grid/polar2grid/grids/config_helper.py
index a7d31858..cb9f6231 100644
--- a/py/polar2grid/polar2grid/grids/config_helper.py
+++ b/py/polar2grid/polar2grid/grids/config_helper.py
@@ -41,9 +41,9 @@
     david.hoese@ssec.wisc.edu
 
 """
-__docformat__ = "restructuredtext en"
+from polar2grid.proj import Proj
 
-from .grids import P2GProj
+__docformat__ = "restructuredtext en"
 
 import os
 import sys
@@ -112,7 +112,7 @@ def main():
 
     proj_str = args.proj_str or determine_projection(clon, clat)
     proj_str = proj_str % {"center_lon":clon, "center_lat":clat}
-    p = P2GProj(proj_str)
+    p = Proj(proj_str)
     origin_x = grid_width / 2.0 * pixel_size_x * -1
     origin_y = grid_height / 2.0 * pixel_size_y * -1
 
diff --git a/py/polar2grid/polar2grid/grids/grids.conf b/py/polar2grid/polar2grid/grids/grids.conf
index 71073d6a..80ff4b02 100644
--- a/py/polar2grid/polar2grid/grids/grids.conf
+++ b/py/polar2grid/polar2grid/grids/grids.conf
@@ -17,7 +17,7 @@ p4_211e,                proj4, +proj=lcc +datum=NAD83 +ellps=GRS80 +lat_0=25 +la
 p4_211e_hi,             proj4, +proj=lcc +datum=NAD83 +ellps=GRS80 +lat_0=25 +lat_1=25 +lon_0=-95 +units=m +no_defs,     10000,   10000,          500,         -500,         -122.94deg,          59.86deg
 lcc_fit,                proj4, +proj=lcc +datum=WGS84 +ellps=WGS84 +lat_0=25 +lat_1=25 +lon_0=-95 +units=m +no_defs,      None,    None,         1000,        -1000,               None,              None
 lcc_fit_hr,             proj4, +proj=lcc +datum=WGS84 +ellps=WGS84 +lat_0=25 +lat_1=25 +lon_0=-95 +units=m +no_defs,      None,    None,          400,         -400,               None,              None
-wgs84_fit,              proj4, +proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs,                                          None,    None,        0.006,       -0.006,               None,              None
+wgs84_fit,              proj4, +proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs,                                          None,    None,       0.0057,      -0.0057,               None,              None
 eqc_fit,                proj4, +proj=eqc +datum=WGS84 +ellps=WGS84 +lat_ts=0 +lon_0=-100 +units=m +no_defs,               None,    None,          250,         -250,               None,              None
 polar_canada,           proj4, +proj=stere +datum=WGS84 +ellps=WGS84 +lat_0=90 +lat_ts=45.0 +lon_0=-150 +units=m,         None,    None,         1000,        -1000,               None,              None
 polar_north_pacific,    proj4, +proj=stere +datum=WGS84 +ellps=WGS84 +lat_0=90 +lat_ts=45.0 +lon_0=-170 +units=m,         None,    None,          400,         -400,               None,              None
diff --git a/py/polar2grid/polar2grid/grids/grids.py b/py/polar2grid/polar2grid/grids/grids.py
index 58433122..ebf8ee30 100644
--- a/py/polar2grid/polar2grid/grids/grids.py
+++ b/py/polar2grid/polar2grid/grids/grids.py
@@ -43,12 +43,13 @@
     david.hoese@ssec.wisc.edu
 
 """
+from polar2grid.proj import Proj
+
 __docformat__ = "restructuredtext en"
 
 from polar2grid.core.constants import *
 from polar2grid.core import Workspace,roles
 from shapely import geometry
-import pyproj
 import numpy
 
 import os
@@ -71,13 +72,6 @@
 GRID_COVERAGE_THRESHOLD = float(os.environ.get("POLAR2GRID_GRID_COVERAGE", "0.1")) # 10%
 
 
-class P2GProj(pyproj.Proj):
-    def __call__(self, data1, data2, **kwargs):
-        if self.is_latlong():
-            return data1, data2
-
-        return super(P2GProj, self).__call__(data1, data2, **kwargs)
-
 ### GPD Reading Functions ###
 def clean_string(s):
     s = s.replace("-", "")
@@ -233,7 +227,7 @@ def parse_proj4_config_line(grid_name, parts):
     proj4_str = parts[2]
     # Test to make sure the proj4_str is valid in pyproj's eyes
     try:
-        p = P2GProj(proj4_str)
+        p = Proj(proj4_str)
         del p
     except StandardError:
         log.error("Invalid proj4 string in '%s' : '%s'" % (grid_name,proj4_str))
@@ -303,7 +297,7 @@ def parse_proj4_config_line(grid_name, parts):
         raise ValueError("Grid origin parameters must be in the same units (meters vs degrees)")
 
     # Convert any parameters from degrees to meters (we already made sure both need to be converted above)
-    p = P2GProj(proj4_str)
+    p = Proj(proj4_str)
     if convert_xorigin_to_meters and not p.is_latlong():
         meters_x, meters_y = p(grid_origin_x, grid_origin_y)
         log.info("Converted grid '%s' origin from (lon: %f, lat: %f) to (x: %f, y: %f)",
@@ -662,7 +656,7 @@ def calculate_grid_corners(self, grid_name):
         if not grid_info["static"]:
             log.debug("Won't calculate corners for a dynamic grid: '%s'" % (grid_name,))
 
-        p = P2GProj(grid_info["proj4_str"])
+        p = Proj(grid_info["proj4_str"])
         right_x  = grid_info["grid_origin_x"] + grid_info["pixel_size_x"] * grid_info["grid_width"]
         bottom_y = grid_info["grid_origin_y"] + grid_info["pixel_size_y"] * grid_info["grid_height"]
         grid_info["ul_corner"] = p(grid_info["grid_origin_x"], grid_info["grid_origin_y"], inverse=True)
diff --git a/py/polar2grid/polar2grid/ll2cr.py b/py/polar2grid/polar2grid/ll2cr.py
index 86672704..25084b22 100644
--- a/py/polar2grid/polar2grid/ll2cr.py
+++ b/py/polar2grid/polar2grid/ll2cr.py
@@ -39,11 +39,12 @@
     david.hoese@ssec.wisc.edu
 
 """
+from polar2grid.proj import Proj
+
 __docformat__ = "restructuredtext en"
 
 from polar2grid.core import Workspace
 from polar2grid.core.constants import DEFAULT_FILL_VALUE
-from .grids.grids import P2GProj
 import numpy
 
 import os
@@ -130,7 +131,7 @@ def ll2cr(lon_arr, lat_arr, proj4_str,
         raise ValueError("Either pixel size or grid width/height must be specified")
 
     # Handle EPSG codes
-    tformer = P2GProj(proj4_str)
+    tformer = Proj(proj4_str)
 
     if dtype is None:
         dtype = lat_arr.dtype
diff --git a/py/polar2grid/polar2grid/p2g_proj.py b/py/polar2grid/polar2grid/proj.py
similarity index 88%
rename from py/polar2grid/polar2grid/p2g_proj.py
rename to py/polar2grid/polar2grid/proj.py
index 7e726c9a..c7bb8839 100644
--- a/py/polar2grid/polar2grid/p2g_proj.py
+++ b/py/polar2grid/polar2grid/proj.py
@@ -43,11 +43,18 @@
 """
 __docformat__ = "restructuredtext en"
 
-from .grids.grids import P2GProj
-
-import os
+import pyproj
 import sys
 
+
+class Proj(pyproj.Proj):
+    def __call__(self, data1, data2, **kwargs):
+        if self.is_latlong():
+            return data1, data2
+
+        return super(Proj, self).__call__(data1, data2, **kwargs)
+
+
 def main():
     from argparse import ArgumentParser
     parser = ArgumentParser(description="Convert lon/lat points to X/Y values")
@@ -61,9 +68,9 @@ def main():
             help="Latitude of the point to be converted (single value only)")
     args = parser.parse_args()
 
-    p = P2GProj(args.proj4_str)
-    x,y = p(args.lon_point, args.lat_point, inverse=args.inv)
-    print x,y
+    p = Proj(args.proj4_str)
+    x, y = p(args.lon_point, args.lat_point, inverse=args.inv)
+    print(x, y)
 
 if __name__ == "__main__":
     sys.exit(main())
diff --git a/swbundle/p2g_proj.sh b/swbundle/p2g_proj.sh
index 2e5738f7..ce73a072 100755
--- a/swbundle/p2g_proj.sh
+++ b/swbundle/p2g_proj.sh
@@ -37,5 +37,5 @@ fi
 source $POLAR2GRID_HOME/bin/polar2grid_env.sh
 
 # Call the script
-$POLAR2GRID_HOME/ShellB3/bin/python -m polar2grid.p2g_proj "$@"
+$POLAR2GRID_HOME/ShellB3/bin/python -m polar2grid.proj "$@"