Merge 951aac6 into 68ef726

.travis/environment_py27.yaml

@@ -10,20 +10,24 @@ dependencies:
 - numexpr
 - rasterio
 - singledispatch
-- netcdf4
+- netcdf4 = 1.2.7 # Travis fails with 1.2.8
 - psycopg2
 - gdal
 - dask
 - xarray
 - redis-py # redis client lib, used by celery
 - redis # redis server
-- pylint # testing
+- pylint = 1.6.5 # testing
 - pep8 # testing
 - fiona # movie generator app
 - mock # testing
 - hypothesis # testing
 - matplotlib # pixel drill app
 - pathlib
+- compliance-checker
+- boto3
+- pathos
+- zstandard
 - compliance-checker = 3.0.3
 - cython # used by netcdf4 at compile only, but needed for compliance checker to load plugins
 - pygeoif = 0.6 # compliance-checker 3.0.3 fails with 0.7
@@ -36,3 +40,4 @@ dependencies:
   - pytest-cov # testing
   - pytest-logging
   - pytest-faulthandler
+  - SharedArray

.travis/environment_py35.yaml

@@ -11,20 +11,24 @@ dependencies:
 - numexpr
 - rasterio
 - singledispatch
-- netcdf4
+- netcdf4 = 1.2.7 # Travis fails with 1.2.8
 - psycopg2
 - gdal
 - dask
 - xarray
 - redis-py # redis client lib, used by celery
 - redis # redis server
-- pylint # testing
+- pylint = 1.6.5 # testing
 - pep8 # testing
 - fiona # movie generator app
 - mock # testing
 - hypothesis # testing
 - matplotlib # pixel drill app
 - pathlib
+- compliance-checker
+- boto3
+- pathos
+- zstandard
 - compliance-checker = 3.0.3
 - cython # used by netcdf4 at compile only, but needed for compliance checker to load plugins
 - pygeoif = 0.6 # compliance-checker 3.0.3 fails with 0.7
@@ -37,3 +41,4 @@ dependencies:
   - pytest-cov # testing
   - pytest-logging
   - pytest-faulthandler
+  - SharedArray

datacube/analytics/analytics_engine.py

@@ -65,12 +65,12 @@ class AnalyticsEngine(object):
             'median': 'median(array1)'
         }
 
-    def __init__(self, api=None, index=None):
+    def __init__(self, api=None, index=None, driver_manager=None):
         LOG.info('Initialise Analytics Module.')
         self.plan = []
         self.plan_dict = {}
 
-        self.api = api or API(index=index)
+        self.api = api or API(index=index, driver_manager=driver_manager)
         self.api_descriptors = {}
         self.api_products = []
 

datacube/analytics/utils/analytics_utils.py

@@ -28,20 +28,16 @@
 from __future__ import absolute_import
 from __future__ import print_function
 import math
-import csv
 import numpy as np
 from scipy import ndimage
-from scipy.io import netcdf
-from osgeo import gdal, osr
-import xarray
 
 
 def plot(array_result):
-    '''
+    """
     Wrapper to Plot a 1D, 2D and 3D array
     Parameters:
         array_result: computed array as a result of execution
-    '''
+    """
 
     dims = len(array_result['array_result'].values()[0].shape)
 
@@ -54,11 +50,11 @@ def plot(array_result):
 
 
 def plot_1d(array_result):
-    '''
+    """
     Plot a 1D array
     Parameters:
         array_result: computed array as a result of execution
-    '''
+    """
     print('plot1D')
     img = array_result['array_result'].values()[0]
 
@@ -74,11 +70,11 @@ def plot_1d(array_result):
 
 
 def plot_2d(array_result):
-    '''
+    """
     Plot a 2D array
     Parameters:
         array_result: computed array as a result of execution
-    '''
+    """
     print('plot2D')
     import matplotlib.pyplot as plt
     img = array_result['array_result'].values()[0]
@@ -96,11 +92,11 @@ def plot_2d(array_result):
 
 
 def plot_3d(array_result):
-    '''
+    """
     Plot a 3D array
     Parameters:
         array_result: computed array as a result of execution
-    '''
+    """
     print('plot3D')
     import matplotlib.pyplot as plt
     img = array_result['array_result'].values()[0]
@@ -125,14 +121,14 @@ def plot_3d(array_result):
 
 
 def get_pqa_mask(pqa_ndarray):
-    '''
+    """
     create pqa_mask from a ndarray
 
     Parameters:
         pqa_ndarray: input pqa array
         good_pixel_masks: known good pixel values
         dilation: amount of dilation to apply
-    '''
+    """
 
     good_pixel_masks = [32767, 16383, 2457]
     dilation = 3

datacube/api/_api.py

@@ -35,27 +35,37 @@ class API(object):
     """
     Interface for use by the ``AnalyticsEngine`` and ``ExecutionEngine`` modules.
     """
-    def __init__(self, index=None, app=None, datacube=None):
+    def __init__(self, index=None, app=None, datacube=None, driver_manager=None):
         """
         Creates the interface for query and storage access.
 
         If no datacube or index is given, the default configuration is used for database connection, etc.
 
-        :param index: The database index to use, from :py:class:`datacube.index.index_connect`
-        :type index: :py:class:`datacube.index._api.Index` or None
+        :param Index index: The database index to use. This feature
+          will become deprecated, so `driver_manager` should be used
+          instead, unless a specific index DB needs to be set in the
+          driver manager for testing purposes.
         :param app: A short, alphanumeric name to identify this application.
             The application name is used to track down problems with database queries, so it is strongly
             advised that be used.  If an index is supplied, application name is ignored.
         :type app: string, required if no index is given
         :param datacube:
         :type datacube: :class:`datacube.Datacube`
+        :param DriverManager driver_manager: The driver manager to
+          use. If not specified, an new manager will be created using
+          the index if specified, or the default configuration
+          otherwise.
         """
         warnings.warn("Descriptor interface is deprecated.", DeprecationWarning)
 
+        self.driver_manager = driver_manager
+
         if datacube is not None:
             self.datacube = datacube
         elif index is not None:
-            self.datacube = Datacube(index)
+            self.datacube = Datacube(index, driver_manager=driver_manager)
+        elif driver_manager is not None:
+            self.datacube = Datacube(driver_manager=driver_manager)
         else:
             app = app or 'Datacube-API'
             self.datacube = Datacube(app=app)
@@ -405,8 +415,10 @@ def _get_data_for_measurement(self, dataset_type, sources, measurements, geobox,
         }
         for measurement_name, measurement in dataset_type.measurements.items():
             if measurements is None or measurement_name in measurements:
-                dt_data['arrays'][measurement_name] = self.datacube.measurement_data(sources, geobox, measurement,
-                                                                                     dask_chunks=dask_chunks)
+                dt_data['arrays'][measurement_name] = self.datacube.measurement_data(
+                    sources, geobox, measurement,
+                    dask_chunks=dask_chunks,
+                    driver_manager=self.driver_manager)
         return dt_data
 
     def list_products(self):