CLIMATE-883 - Ensure python 3 builds pass unit tests

RCMES/run_RCMES.py

@@ -113,7 +113,7 @@ def load_datasets_from_config(extra_opts, *loader_opts):
                                                           temporal_resolution)
 
     if multiplying_factor[i] != 1:
-        obs_dataset.values *= multiplying_factor[i]
+        obs_datasets[i].values *= multiplying_factor[i]
 
 """ Step 2: Load model NetCDF Files into OCW Dataset Objects """
 model_datasets = load_datasets_from_config(extra_opts, *model_data_info)
@@ -148,7 +148,7 @@ def load_datasets_from_config(extra_opts, *loader_opts):
                     start=start_time,
                     end=end_time)
 else:
-    bounds = Bounds(boundary_type=space_info['boundary_type'], 
+    bounds = Bounds(boundary_type=space_info['boundary_type'],
                     start=start_time,
                     end=end_time)
 

ocw/data_source/rcmed.py

@@ -21,9 +21,14 @@
     More information about the RCMED Query Specification can be found below:
     https://rcmes.jpl.nasa.gov/query-api/query.php?
 '''
+# Needed Python 2/3 urllib compatability
+try:
+    from urllib.parse import urlencode
+    from urllib.request import urlopen
+except ImportError:
+    from urllib import urlencode
+    from urllib2 import urlopen
 
-import urllib
-import urllib2
 import re
 import json
 import numpy as np
@@ -45,8 +50,8 @@ def get_parameters_metadata():
 
     param_info_list = []
     url = URL + "&param_info=yes"
-    string = urllib2.urlopen(url)
-    data_string = string.read()
+    string = urlopen(url)
+    data_string = string.read().decode('utf-8')
     json_format_data = json.loads(data_string)
     fields_name = json_format_data['fields_name']
     data = json_format_data['data']
@@ -74,7 +79,8 @@ def _make_mask_array(values, parameter_id, parameters_metadata):
     '''
 
     for each in parameters_metadata:
-        if each['parameter_id'].encode() == str(parameter_id):
+        if str(each['parameter_id']) == str(parameter_id):
+            print('ok')
             missing_values = each['missingdataflag'].encode()
             break
     missing_values = float(missing_values)
@@ -89,7 +95,7 @@ def _reshape_values(values, unique_values):
     :param values: Raw values data
     :type values: numpy array
     :param unique_values: Tuple of unique latitudes, longitudes and times data.
-    :type unique_values: Tuple 
+    :type unique_values: Tuple
 
     :returns: Reshaped values data
     :rtype: Numpy array
@@ -118,7 +124,8 @@ def _calculate_time(unique_times, time_step):
 
     time_format = "%Y-%m-%d %H:%M:%S"
     unique_times = np.array(
-        [datetime.strptime(time, time_format) for time in unique_times])
+        [datetime.strptime(time.decode('utf-8'), time_format)
+                           for time in unique_times])
     # There is no need to sort time.
     # This function may required still in RCMES
     # unique_times.sort()
@@ -158,11 +165,11 @@ def _get_data(url):
     :rtype: (Numpy array, Numpy array, Numpy array, Numpy array)
     '''
 
-    string = urllib2.urlopen(url)
+    string = urlopen(url)
     data_string = string.read()
-    index_of_data = re.search('data: \r\n', data_string)
+    index_of_data = re.search(b'data: \r\n', data_string)
     data = data_string[index_of_data.end():len(data_string)]
-    data = data.split('\r\n')
+    data = data.split(b'\r\n')
 
     lats = []
     lons = []
@@ -172,7 +179,7 @@ def _get_data(url):
 
     # Because the last row is empty, "len(data)-1" is used.
     for i in range(len(data) - 1):
-        row = data[i].split(',')
+        row = data[i].split(b',')
         lats.append(np.float32(row[0]))
         lons.append(np.float32(row[1]))
         # Level is not currently supported in Dataset class.
@@ -248,7 +255,7 @@ def _generate_query_url(dataset_id, parameter_id, min_lat, max_lat, min_lon, max
     :type max_lon: Float
     :param start_time: Start time
     :type start_time: Datetime
-    :param end_time: End time 
+    :param end_time: End time
     :type end_time: Datetime
     :param time_step: Time step
     :type time_step: String
@@ -265,7 +272,7 @@ def _generate_query_url(dataset_id, parameter_id, min_lat, max_lat, min_lon, max
     query = [('datasetId', dataset_id), ('parameterId', parameter_id), ('latMin', min_lat), ('latMax', max_lat),
              ('lonMin', min_lon), ('lonMax', max_lon), ('timeStart', start_time), ('timeEnd', end_time)]
 
-    query_url = urllib.urlencode(query)
+    query_url = urlencode(query)
     url_request = URL + query_url
 
     return url_request
@@ -320,7 +327,7 @@ def parameter_dataset(dataset_id, parameter_id, min_lat, max_lat, min_lon, max_l
     :param start_time: Start time
     :type start_time: :class:`datetime.datetime`
 
-    :param end_time: End time 
+    :param end_time: End time
     :type end_time: :class:`datetime.datetime`
 
     :param name: (Optional) A name for the loaded dataset.

ocw/dataset_processor.py

@@ -54,8 +54,8 @@ def temporal_subset(target_dataset, month_start, month_end,
     """
 
     if month_start > month_end:
-        month_index = range(month_start, 13)
-        month_index.extend(range(1, month_end + 1))
+        month_index = list(range(month_start, 13))
+        month_index.extend(list(range(1, month_end + 1)))
     else:
         month_index = range(month_start, month_end + 1)
 
@@ -150,7 +150,7 @@ def temporal_rebin_with_time_index(target_dataset, nt_average):
                'be a multiple of nt_average')
         print(msg)
     # nt2 is the length of time dimension in the rebinned dataset
-    nt2 = nt / nt_average
+    nt2 = nt // nt_average
     binned_dates = target_dataset.times[np.arange(nt2) * nt_average]
     binned_values = ma.zeros(
         np.insert(target_dataset.values.shape[1:], 0, nt2))
@@ -1069,7 +1069,7 @@ def _rcmes_calc_average_on_new_time_unit(data, dates, unit):
     nt, ny, nx = data.shape
     if unit == 'full':
         new_data = ma.mean(data, axis=0)
-        new_date = [dates[dates.size / 2]]
+        new_date = [dates[dates.size // 2]]
     if unit == 'annual':
         years = [d.year for d in dates]
         years_sorted = np.unique(years)
@@ -1108,8 +1108,8 @@ def _rcmes_calc_average_on_new_time_unit(data, dates, unit):
         for day in days_sorted:
             index = np.where(days == day)[0]
             new_data[it, :] = ma.mean(data[index, :], axis=0)
-            y = int(day / 10000)
-            m = int(day % 10000) / 100
+            y = int(day // 10000)
+            m = int(day % 10000) // 100
             d = int(day % 100)
             new_date.append(datetime.datetime(year=y, month=m, day=d))
             it = it + 1

ocw/evaluation.py

@@ -16,13 +16,13 @@
 # under the License.
 
 '''
-Classes: 
+Classes:
     Evaluation - Container for running an evaluation
 '''
 
 import logging
-from metrics import Metric, UnaryMetric, BinaryMetric
-from dataset import Dataset, Bounds
+from ocw.metrics import Metric, UnaryMetric, BinaryMetric
+from ocw.dataset import Dataset, Bounds
 import ocw.dataset_processor as DSP
 
 import numpy.ma as ma
@@ -33,14 +33,14 @@
 class Evaluation(object):
     '''Container for running an evaluation
 
-    An *Evaluation* is the running of one or more metrics on one or more 
+    An *Evaluation* is the running of one or more metrics on one or more
     target datasets and a (possibly optional) reference dataset. Evaluation
     can handle two types of metrics, ``unary`` and ``binary``. The validity
     of an Evaluation is dependent upon the number and type of metrics as well
     as the number of datasets.
 
     A ``unary`` metric is a metric that runs over a single dataset. If you add
-    a ``unary`` metric to the Evaluation you are only required to add a 
+    a ``unary`` metric to the Evaluation you are only required to add a
     reference dataset or a target dataset. If there are multiple datasets
     in the evaluation then the ``unary`` metric is run over all of them.
 
@@ -50,7 +50,7 @@ class Evaluation(object):
     are run over every (reference dataset, target dataset) pair in the
     Evaluation.
 
-    An Evaluation must have at least one metric to be valid. 
+    An Evaluation must have at least one metric to be valid.
     '''
 
     def __init__(self, reference, targets, metrics, subregions=None):
@@ -59,19 +59,19 @@ def __init__(self, reference, targets, metrics, subregions=None):
         :param reference: The reference Dataset for the evaluation.
         :type reference: :class:`dataset.Dataset`
 
-        :param targets: A list of one or more target datasets for the 
+        :param targets: A list of one or more target datasets for the
                 evaluation.
         :type targets: :class:`list` of :class:`dataset.Dataset`
 
-        :param metrics: A list of one or more Metric instances to run 
+        :param metrics: A list of one or more Metric instances to run
                 in the evaluation.
         :type metrics: :class:`list` of :mod:`metrics`
 
         :param subregions: (Optional) Subregion information to use in the
                 evaluation. A subregion is specified with a Bounds object.
         :type subregions: :class:`list` of :class:`dataset.Bounds`
 
-        :raises: ValueError 
+        :raises: ValueError
         '''
         #: The reference dataset.
         self._ref_dataset = reference
@@ -140,7 +140,7 @@ def subregions(self, value):
     def add_dataset(self, target_dataset):
         '''Add a Dataset to the Evaluation.
 
-        A target Dataset is compared against the reference dataset when the 
+        A target Dataset is compared against the reference dataset when the
         Evaluation is run with one or more metrics.
 
         :param target_dataset: The target Dataset to add to the Evaluation.
@@ -161,7 +161,7 @@ def add_dataset(self, target_dataset):
     def add_datasets(self, target_datasets):
         '''Add multiple Datasets to the Evaluation.
 
-        :param target_datasets: The list of datasets that should be added to 
+        :param target_datasets: The list of datasets that should be added to
             the Evaluation.
         :type target_datasets: :class:`list` of :class:`dataset.Dataset`
 
@@ -217,7 +217,7 @@ def run(self):
         target dataset.
 
         If there is subregion information provided then each dataset is subset
-        before being run through the binary metrics. 
+        before being run through the binary metrics.
 
         ..note:: Only the binary metrics are subset with subregion information.
 

ocw/plotter.py

@@ -146,10 +146,10 @@ def _fig_size(gridshape, aspect=None):
     nrows, ncols = gridshape
     if nrows >= ncols:
         # If more rows keep width constant
-        width, height = (aspect * 5.5), 5.5 * (nrows / ncols)
+        width, height = (aspect * 5.5), 5.5 * (nrows // ncols)
     else:
         # If more columns keep height constant
-        width, height = (aspect * 5.5) * (ncols / nrows), 5.5
+        width, height = (aspect * 5.5) * (ncols // nrows), 5.5
 
     return width, height
 
@@ -509,7 +509,7 @@ def draw_barchart(results, yvalues, fname, ptitle='', fmt='png',
     :type results: :class:`numpy.ndarray`
 
     :param yvalues: List of y-axis labels
-    :type times: :class:`list` 
+    :type times: :class:`list`
 
     :param fname: Filename of the plot.
     :type fname: :mod:`string`

ocw/tests/test_dataset.py

@@ -43,13 +43,13 @@ def setUp(self):
                                     origin=self.origin)
 
     def test_lats(self):
-        self.assertItemsEqual(self.test_dataset.lats, self.lat)
+        np.testing.assert_array_equal(self.test_dataset.lats, self.lat)
 
     def test_lons(self):
-        self.assertItemsEqual(self.test_dataset.lons, self.lon)
+        np.testing.assert_array_equal(self.test_dataset.lons, self.lon)
 
     def test_times(self):
-        self.assertItemsEqual(self.test_dataset.times, self.time)
+        np.testing.assert_array_equal(self.test_dataset.times, self.time)
 
     def test_values(self):
         self.assertEqual(self.test_dataset.values.all(), self.value.all())

ocw/tests/test_dataset_processor.py

@@ -99,16 +99,16 @@ def setUp(self):
 
     def test_time_dimension_multiple_of_orig_time_dimension(self):
         # ten_year_dataset.times.size is 120
-        nt_avg = self.ten_year_dataset.times.size / 2
+        nt_avg = self.ten_year_dataset.times.size // 2
         # Temporal Rebin to exactly 2 (time) values
         dataset = dp.temporal_rebin_with_time_index(
             self.ten_year_dataset, nt_avg)
         start_time = self.ten_year_dataset.times[0]
         # First month of the middle year
-        middle_element = self.ten_year_dataset.times.size / 2
+        middle_element = self.ten_year_dataset.times.size // 2
         end_time = self.ten_year_dataset.times[middle_element]
         self.assertEqual(dataset.times.size,
-                         self.ten_year_dataset.times.size / nt_avg)
+                         self.ten_year_dataset.times.size // nt_avg)
         np.testing.assert_array_equal(dataset.times, [start_time, end_time])
 
     def test_time_dimension_not_multiple_of_orig_time_dimension(self):
@@ -119,7 +119,7 @@ def test_time_dimension_not_multiple_of_orig_time_dimension(self):
             self.ten_year_dataset, nt_avg)
         new_times = self.ten_year_dataset.times[::11][:-1]
         self.assertEqual(dataset.times.size,
-                         self.ten_year_dataset.times.size / nt_avg)
+                         self.ten_year_dataset.times.size // nt_avg)
         np.testing.assert_array_equal(dataset.times, new_times)
 
     def test_returned_dataset_attributes(self):