Merge branch 'CLIMATE-962'

README.rst

@@ -0,0 +1,105 @@
+Apache Open Climate Workbench
+-----------------------------
+
+|BuildStatus|_
+|ImageLink|_
+|requires|_
+|landscape|_
+|pypi|_
+|pythonbadge|_
+|anacondainstaller|_
+|anacondadownload|_
+|anacondaversion|_
+
+.. image:: ./docs/source/ocw-logo-variant-sm-01-01-new.png
+   :width: 20px
+   :height: 100px
+   :scale: 50%
+   :alt: alternate text
+   :align: right
+
+
+`Apache Open Climate Workbench`_ is an effort to develop software that
+performs climate model evaluations using model outputs from a variety of
+different sources (the Earth System Grid Federation, the Coordinated
+Regional Downscaling Experiment, the U.S. National Climate Assessment
+and the North American Regional Climate Change Assessment Program) and
+temporal/spatial scales with remote sensing data from NASA, NOAA and
+other agencies. The toolkit includes capabilities for rebinning, metrics
+computation and visualization. For additional project information,
+please check the `project website`_.
+
+Getting Started
+---------------
+
+The `project’s wiki`_ is the best location for help and project
+information. New users should check out the `Getting Started`_ and `Easy
+OCW`_ pages for help getting the necessary dependencies installed. If
+you would prefer to have an isolated environment set up in a virtual
+machine you should read the `OCW VM`_ documentation. It will help you
+get up and running quickly with a fresh VM image for OCW work.
+
+There are a number of examples in the *examples* directory to help users
+get started with the toolkit API. If you have questions, the best way to
+get help is to email the project mailing lists which can be found on the
+`project's community page`_
+
+
+Development
+---------------
+
+OCW always welcomes pull request. Please check the `Developer Area`_ on the wiki for additional information on how to contribute. The `project's JIRA`_ is a great place to start looking for issues to solve. Make sure to stop by the mailing lists and introduce yourself as well!
+
+Documentation
+---------------
+
+The project host the documentation built from the last release artifact on `the project website`_ 
+
+If you would like to build the documentation for the project run the following command from the root of the repository:
+::
+         cd docs && make html
+
+
+You will need to have installed the project dependencies first. Checkout the `Getting Started`_ and `Easy OCW`_ pages for help getting the necessary dependencies installed.
+
+
+.. |ImageLink| image:: https://coveralls.io/repos/github/apache/climate/badge.svg?branch=master
+.. _ImageLink: https://coveralls.io/github/apache/climate?branch=master
+
+.. |BuildStatus| image:: https://api.travis-ci.org/apache/climate.svg?branch=master
+.. _BuildStatus:  https://travis-ci.org/apache/climate
+
+.. |requires| image:: https://requires.io/github/apache/climate/requirements.svg?branch=master
+.. _requires:  https://requires.io/github/apache/climate/requirements/?branch=master
+
+.. |landscape| image:: https://landscape.io/github/apache/climate/master/landscape.svg?style=flat-square
+.. _landscape: https://landscape.io/github/apache/climate/master
+
+.. |pypi| image:: https://img.shields.io/pypi/v/ocw.svg?maxAge=2592000?style=plastic
+.. _pypi:  https://pypi.python.org/pypi/ocw
+
+.. |pythonbadge| image:: https://img.shields.io/badge/python-3-blue.svg
+.. _pythonbadge: https://www.python.org/downloads/
+
+.. |anacondainstaller| image:: https://anaconda.org/conda-forge/ocw/badges/installer/conda.svg
+.. _anacondainstaller: https://anaconda.org/conda-forge/ocw
+
+.. |anacondadownload| image:: https://anaconda.org/conda-forge/ocw/badges/downloads.svg
+.. _anacondadownload: https://anaconda.org/conda-forge/ocw
+
+.. |anacondaversion| image:: https://anaconda.org/conda-forge/ocw/badges/version.svg
+.. _anacondaversion: https://anaconda.org/conda-forge/ocw
+
+
+.. _Apache Open Climate Workbench: http://climate.apache.org
+.. _project website: http://climate.apache.org/
+.. _project’s wiki: https://cwiki.apache.org/confluence/display/CLIMATE/Home
+.. _Getting Started: https://cwiki.apache.org/confluence/display/CLIMATE/Getting+Started
+.. _Easy OCW: https://cwiki.apache.org/confluence/display/CLIMATE/Easy-OCW+-+A+Guide+to+Simplifying+OCW+Installation
+.. _OCW VM: https://cwiki.apache.org/confluence/display/CLIMATE/OCW+VM+-+A+Self+Contained+OCW+Environment
+.. _project's community page: http://climate.apache.org/community/mailing-li
+.. _Developer Area: https://cwiki.apache.org/confluence/display/CLIMATE/Developer+Area
+.. _project's JIRA: https://issues.apache.org/jira/browse/CLIMATE
+.. _the project website: https://climate.apache.org/api/current/index.html
+.. _Getting Started: https://cwiki.apache.org/confluence/display/CLIMATE/Getting+Started
+.. _Easy OCW: https://cwiki.apache.org/confluence/display/CLIMATE/Easy-OCW+-+A+Guide+to+Simplifying+OCW+Installation

deps_py2.txt

@@ -1,15 +1,18 @@
+cython
 numpy
 scipy
 matplotlib
 basemap
 netcdf4
 h5py
 bottle
-pydap
+Pydap
+lxml
 python-dateutil
 mock
 webtest
 myproxyclient
 esgf-pyclient
 podaacpy
-networkx
+networkx
+netcdftime

deps_py3.txt

@@ -1,3 +1,4 @@
+cython
 numpy
 scipy
 matplotlib
@@ -9,4 +10,6 @@ python-dateutil
 mock
 webtest
 podaacpy
-networkx
+networkx
+netcdftime
+lxml

ocw/data_source/local.py

@@ -260,6 +260,7 @@ def load_file(file_path,
     times = utils.decode_time_values(netcdf, time_name)
     times = numpy.array(times)
     values = ma.array(netcdf.variables[variable_name][:])
+    values = ma.array(values, mask=numpy.isnan(values))
     if not variable_unit:
         if hasattr(netcdf.variables[variable_name], 'units'):
             variable_unit = netcdf.variables[variable_name].units
@@ -536,7 +537,7 @@ def load_dataset_from_multiple_netcdf_files(variable_name, variable_unit=None,
         if ifile == 0:
             data_values = values0
         else:
-            data_values = numpy.concatenate((data_values, values0))
+            data_values = ma.concatenate((data_values, values0))
     times = numpy.array(times)
     variable_unit = dataset0.units if not variable_unit else variable_unit
     return Dataset(lats, lons, times, data_values,

ocw/metrics.py

@@ -89,6 +89,28 @@ def run(self, ref_dataset, target_dataset):
         :rtype: :class:`numpy.ndarray`
         '''
         return calc_bias(target_dataset.values, ref_dataset.values)
+
+
+class AbsoluteBias(BinaryMetric):
+    '''Calculate the absolute bias between a reference and target dataset.'''
+
+    def run(self, ref_dataset, target_dataset):
+        '''Calculate the absolute bias between a reference and target dataset.
+
+        .. note::
+           Overrides BinaryMetric.run()
+
+        :param ref_dataset: The reference dataset to use in this metric run.
+        :type ref_dataset: :class:`dataset.Dataset`
+
+        :param target_dataset: The target dataset to evaluate against the
+            reference dataset in this metric run.
+        :type target_dataset: :class:`dataset.Dataset`
+
+        :returns: The absolute difference between the reference and target datasets.
+        :rtype: :class:`numpy.ndarray`
+        '''
+        return calc_absbias(target_dataset.values, ref_dataset.values)
 
 
 class SpatialPatternTaylorDiagram(BinaryMetric):
@@ -281,6 +303,29 @@ def calc_bias(target_array, reference_array, average_over_time=False):
         return ma.average(bias, axis=0)
     else:
         return bias
+
+
+def calc_absbias(target_array, reference_array, average_over_time=False):
+    ''' Calculate absolute difference between two arrays
+
+    :param target_array: an array to be evaluated, as model output
+    :type target_array: :class:'numpy.ma.core.MaskedArray'
+
+    :param reference_array: an array of reference dataset
+    :type reference_array: :class:'numpy.ma.core.MaskedArray'
+
+    :param average_over_time: if True, calculated bias is averaged for the axis=0
+    :type average_over_time: 'bool'
+
+    :returns: Absolute Biases array of the target dataset
+    :rtype: :class:'numpy.ma.core.MaskedArray'
+    '''
+
+    bias = abs(target_array - reference_array)
+    if average_over_time:
+        return ma.average(bias, axis=0)
+    else:
+        return bias
 
 
 def calc_stddev(array, axis=None):

ocw/tests/test_metrics.py

@@ -66,6 +66,46 @@ def test_function_run(self):
         expected_result.fill(-300)
         np.testing.assert_array_equal(self.bias.run(
             self.target_dataset, self.reference_dataset), expected_result)
+
+
+class TestAbsoluteBias(unittest.TestCase):
+    '''Test the metrics.Bias metric.'''
+
+    def setUp(self):
+        self.bias = metrics.AbsoluteBias()
+        # Initialize reference dataset
+        self.reference_lat = np.array([10, 12, 14, 16, 18])
+        self.reference_lon = np.array([100, 102, 104, 106, 108])
+        self.reference_time = np.array(
+            [dt.datetime(2000, x, 1) for x in range(1, 13)])
+        flat_array = np.array(range(300))
+        self.reference_value = flat_array.reshape(12, 5, 5)
+        self.reference_variable = 'prec'
+        self.reference_dataset = Dataset(self.reference_lat,
+                                         self.reference_lon,
+                                         self.reference_time,
+                                         self.reference_value,
+                                         self.reference_variable)
+        # Initialize target dataset
+        self.target_lat = np.array([1, 2, 4, 6, 8])
+        self.target_lon = np.array([10, 12, 14, 16, 18])
+        self.target_time = np.array(
+            [dt.datetime(2001, x, 1) for x in range(1, 13)])
+        flat_array = np.array(range(300, 600))
+        self.target_value = flat_array.reshape(12, 5, 5)
+        self.target_variable = 'tasmax'
+        self.target_dataset = Dataset(self.target_lat,
+                                      self.target_lon,
+                                      self.target_time,
+                                      self.target_value,
+                                      self.target_variable)
+
+    def test_function_run(self):
+        '''Test bias function between reference dataset and target dataset.'''
+        expected_result = np.zeros((12, 5, 5), dtype=np.int)
+        expected_result.fill(300)
+        np.testing.assert_array_equal(self.bias.run(
+            self.target_dataset, self.reference_dataset), expected_result)
 
 
 class TestSpatialPatternTaylorDiagram(unittest.TestCase):

setup.py

@@ -61,7 +61,7 @@
         to='rst',
         outputfile='README.rst')
 except(IOError, ImportError):
-    _long_description = open('README.md').read()
+    _long_description = open('README.rst').read()
 
 open('doc.txt', 'w').write(_long_description)