Merge branch 'master' into CLIMATE-943

docs/source/conf.py

@@ -14,6 +14,11 @@
 import sys
 import os
 
+# esgf is not currently available for Python 3 and will throw an
+# error when building the documents.
+if sys.version_info[0] >= 3:
+    autodoc_mock_imports = ["esgf"]
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -33,11 +38,15 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+
+# Note that 'sphinxcontrib.autohttp.bottle' is currently broken in Sphinx > 1.56
+# Remove from the extension list if documentation fails on Sphinx hard failure.
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.intersphinx',
     'sphinx.ext.ifconfig',
     'sphinxcontrib.httpdomain',
+    'sphinxcontrib.autohttp.bottle'
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/source/index.rst

@@ -18,6 +18,7 @@ Contents:
    ocw/evaluation
    ocw/metrics
    ocw/plotter
+   ocw/statistical_downscaling
    ocw/utils
    data_source/data_sources
    ui-backend/backend

docs/source/ocw/statistical_downscaling.rst

@@ -0,0 +1,8 @@
+Downscaling Module
+******************
+
+Downscaling
+===========
+.. autoclass:: statistical_downscaling.Downscaling
+    :members:
+

examples/GPM_WRF24_JPDF_comparison.py

@@ -15,6 +15,32 @@
 # specific language governing permissions and limitations
 # under the License.
 
+"""
+    GPM_WRF24_JPDF_comparison.py
+
+    This is an example of calculating the joint probability distribution
+    function of rainfall intensity and duration for the Northern Great
+    Plains using GPM IMERG data for June/01/2015
+
+    In this example:
+
+    1. Load the GPM and WRF24 datasets with spatial filter.
+    2. Load the spatial filter (Bukovsky region mask).
+    3. Spatially subset the WRF data.
+    4. Analyze the wet spells.
+    5. Calculate the joint PDF(JPDF) of spell_duration and peak_rainfall.
+    6. Visualize the JPDF.
+
+    OCW modules demonstrated:
+
+    1. datasource/local
+    2. dataset
+    3. dataset_processor
+    4. metrics
+    5. plotter
+
+"""
+
 from ocw.dataset import Bounds
 import ocw.data_source.local as local
 import ocw.dataset_processor as dsp

examples/draw_climatology_map_MISR_AOD.py

@@ -15,6 +15,32 @@
 # specific language governing permissions and limitations
 # under the License.
 
+"""
+    draw_climatology_map_MISR_AOD.py
+
+    Use OCW to download an MISR dataset, subset the data, calculate the 16 and 5 year
+    mean and draw a countour map of the means and the current values.
+
+    In this example:
+
+    1. Download a dataset from https://dx.doi.org/10.6084/m9.figshare.3753321.v1.
+    ***  Note *** The dataset for this example is not downloaded as part of the example
+    and must be downloaded to examples directory before running the example.
+    *** Note *** Depending on the OS on which the example is being run, the download
+    may remove the - in the filename.  Rename the file appropriately.
+    2. Subset the data set (lat / lon / start date / end date).
+    3. Calculate the 16, 5 and 1 year mean.
+    4. Draw a three contour maps using the calculated means and current values.
+
+    OCW modules demonstrated:
+
+    1. datasource/local
+    2. dataset
+    3. dataset_processor
+    4. plotter
+
+"""
+
 import ocw.dataset as ds
 import ocw.data_source.local as local
 import ocw.dataset_processor as dsp

examples/esgf_integration_example.py

@@ -15,6 +15,21 @@
 # specific language governing permissions and limitations
 # under the License.
 
+"""
+    esgf_integration_example.py
+
+    Use OCW to download an ESGF dataset into the common format of an OCW dataset object.
+
+    In this example:
+
+    1. Download an ESGF (https://esgf.llnl.gov/) dataset and load it into a OCW dataset object.
+
+    OCW modules demonstrated:
+
+    1. datasource/esgf
+
+"""
+
 import ocw.data_source.esgf as esgf
 from getpass import getpass
 import ssl

examples/knmi_to_cru31_full_bias.py

@@ -15,6 +15,38 @@
 # specific language governing permissions and limitations
 # under the License.
 
+"""
+    knmi_to_cru31_full_bias.py
+
+    Use OCW to download, evaluate and plot (contour map) a dataset
+    against a reference dataset and OCW standard metrics (bias).
+
+    In this example:
+
+    1. Download a netCDF files from a local site.
+        AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc
+    2. Load the local files into OCW dataset objects.
+    3. Interface with the Regional Climate Model Evalutaion Database (https://rcmes.jpl.nasa.gov/)
+       to load the CRU3.1 Daily-Max Temp dataset (https://rcmes.jpl.nasa.gov/content/cru31).
+    4. Process each dataset to the same same shape.
+    5. Temporally rebin the datasets to a single timestep.
+    6. Spatially regrid the dataset objects to a 1/2 degree grid.
+    7. Build a bias metric to use for evaluation use the standard OCW metric set.
+    8. Create an evaluation object using the datasets and metric.
+    9. Plot the results of the evaluation (contour map).
+
+    OCW modules demonstrated:
+
+    1. datasource/local
+    2. datasource/rcmed
+    3. dataset
+    4. dataset_processor
+    5. evaluation
+    6. metrics
+    7. plotter
+
+"""
+
 import datetime
 import urllib
 from os import path

examples/model_ensemble_to_rcmed.py

@@ -15,6 +15,38 @@
 # specific language governing permissions and limitations
 # under the License.
 
+"""
+    model_ensemble_to_rcmed.py
+
+    Use OCW to download, evaluate and plot (contour map) two datasets
+    against a reference dataset and OCW standard metrics (bias).
+
+    In this example:
+
+    1. Download two netCDF files from a local site.
+        AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc
+        AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax.nc
+    2. Load the local files into OCW dataset objects.
+    3. Interface with the Regional Climate Model Evaluation Database (https://rcmes.jpl.nasa.gov/)
+       to load the CRU3.1 Daily-Max Temp dataset (https://rcmes.jpl.nasa.gov/content/cru31).
+    4. Temporally rebin the datasets to annual.
+    5. Spatially regrid the dataset objects to a 1/2 degree grid.
+    6. Build a bias metric to use for evaluation use the standard OCW metric set.
+    7. Create an evaluation object using the datasets and metric.
+    8. Plot the results of the evaluation (contour map).
+
+    OCW modules demonstrated:
+
+    1. datasource/local
+    2. datasource/rcmed
+    3. dataset
+    4. dataset_processor
+    5. metrics
+    6. evaluation
+    7. plotter
+
+"""
+
 import datetime
 import math
 import urllib

examples/multi_model_evaluation.py

@@ -15,6 +15,42 @@
 # specific language governing permissions and limitations
 # under the License.
 
+"""
+    multi_model_evaluation.py
+
+    Use OCW to download, evaluate and plot (contour map) two datasets
+    against a reference dataset and OCW standard metrics.
+
+    In this example:
+
+    1. Download two netCDF files from a local site.
+        AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_pr.nc
+        AFRICA_UCT-PRECIS_CTL_ERAINT_MM_50km_1989-2008_pr.nc
+    2. Load the local files into OCW dataset objects.
+    3. Interface with the Regional Climate Model Evaluation Database (https://rcmes.jpl.nasa.gov/)
+       to load the CRU3.1 Daily Precipitation dataset (https://rcmes.jpl.nasa.gov/content/cru31).
+    4. Process each dataset to the same same shape.
+        a.) Restrict the datasets re: geographic and time boundaries.
+        b.) Convert the dataset water flux to common units.
+        c.) Normalize the dataset date / times to monthly.
+        d.) Spatially regrid each dataset.
+    5.  Calculate the mean annual value for each dataset.
+    6.  Evaluate the datasets against the reference data set and OCW standard metric and plot
+        a contour map.
+
+    OCW modules demonstrated:
+
+    1. datasource/local
+    2. datasource/rcmed
+    3. dataset
+    4. dataset_processor
+    5. metrics
+    6. evaluation
+    7. plotter
+    8. utils
+
+"""
+
 import datetime
 import urllib
 import numpy as np

examples/multi_model_taylor_diagram.py

@@ -1,3 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+    multi_model_taylor_diagram.py
+
+    Use OCW to download, normalize and evaluate three datasets
+    against a reference dataset and OCW standard metrics
+    drawing a Taylor diagram of the results of the evaluation.
+
+    In this example:
+
+    1. Download three netCDF files from a local site.
+        AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_pr.nc
+        AFRICA_ICTP-REGCM3_CTL_ERAINT_MM_50km-rg_1989-2008_pr.nc
+        AFRICA_UCT-PRECIS_CTL_ERAINT_MM_50km_1989-2008_pr.nc
+    2. Load the local files into OCW dataset objects.
+    3. Process each dataset to the same same shape.
+        a.) Restrict the datasets re: geographic and time boundaries.
+        b.) Temporally rebin the data (monthly).
+        c.) Spatially regrid each dataset.
+    4.  Extract the metrics used for the evaluation and evaluate
+        against a reference dataset and standard OCW metrics.
+    5.  Draw evaluation results Taylor diagram.
+
+    OCW modules demonstrated:
+
+    1. datasource/local
+    2. dataset
+    3. dataset_processor
+    4. evaluation
+    5. metrics
+    6. plotter
+    7. utils
+
+"""
+
 # Apache OCW lib immports
 from ocw.dataset import Dataset, Bounds
 import ocw.data_source.local as local

examples/podaac_integration_example.py

@@ -15,6 +15,27 @@
 # specific language governing permissions and limitations
 # under the License.
 
+"""
+    podaac_integration_example.py
+
+    Use OCW to download a PODACC dataset, evaluate and plot (contour map).
+
+    In this example:
+
+    1. Download a remote PO.DAAC (https://podaac.jpl.nasa.gov/) dataset
+       and read it into an OCW dataset object.
+    2. Create a temporal STD metric using one of the OCW standard metrics.
+    3. Evaluate the dataset against the metric and plot a contour map.
+
+    OCW modules demonstrated:
+
+    1. datasource/podaac_datasource
+    2. metrics
+    3. evaluation
+    4. plotter
+
+"""
+
 import ocw.data_source.podaac_datasource as podaac
 import ocw.evaluation as evaluation
 import ocw.metrics as metrics

examples/simple_model_to_model_bias.py

@@ -15,6 +15,35 @@
 # specific language governing permissions and limitations
 # under the License.
 
+"""
+    simple_model_to_model_bias.py
+
+    Use OCW to download, normalize and evaluate two datasets
+    against an OCW metric (bias) and plot the results of the
+    evaluation (contour map).
+
+    In this example:
+
+    1. Download two netCDF files from a local site.
+        AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc
+        AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax.nc
+    2. Load the local files into OCW dataset objects.
+    3. Temporally rebin the data anually.
+    4. Spatially regrid the dataset objects to a 1 degree grid.
+    5. Build a bias metric to use for evaluation use the standard OCW metric set.
+    6. Create an evaluation object using the datasets and metric.
+    7. Plot the results of the evaluation (contour map).
+
+    OCW modules demonstrated:
+
+    1. datasource/local
+    2. dataset_processor
+    3. evaluation
+    4. metrics
+    5. plotter
+
+"""
+
 import datetime
 from os import path
 import sys