Merge branch 'develop' of https://github.com/GeoscienceAustralia/tcrm …

…into develop

.travis.yml

@@ -31,39 +31,31 @@ addons:
     - gdal-bin
     - libgdal-dev
     - python-gdal
-before-install:
-- travis_wait . ./travis_gdal_install.sh  
-- export PATH=$GDALINST/gdal-$GDALVERSION/bin:$PATH
-- export LD_LIBRARY_PATH=$GDALINST/gdal-$GDALVERSION/lib:$LD_LIBRARY_PATH
+
 install:
 - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh
   -O miniconda.sh; else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
   -O miniconda.sh; fi
 - bash miniconda.sh -b -p $HOME/miniconda
 - export PATH="$HOME/miniconda/bin:$PATH"
-- export LD_LIBRARY_PATH=$GDALINST/gdal-$GDALVERSION
-- export CPLUS_INCLUDE_PATH=$GDALINST/gdal-$GDALVERSION
-- export C_INCLUDE_PATH=$GDALINST/gdal-$GDALVERSION
+- export CPLUS_INCLUDE_PATH="/usr/include/gdal"
+- export C_INCLUDE_PATH="/usr/include/gdal"
 - hash -r
 - conda config --set always_yes yes --set changeps1 no
 - conda update -q conda
-- conda install -c anaconda sqlite
-- conda install -c conda-forge gdal=1.11.2
-- deps='pip numpy scipy matplotlib basemap shapely nose coverage pycurl'
-- conda create -p $HOME/py --yes $deps "python=$TRAVIS_PYTHON_VERSION"
-- export PATH=$HOME/py/bin:$PATH
-- pip install pycurl
-- pip -v install -r requirements.txt
-#- "GDAL_CONFIG=$GDALINST/gdal-$GDALVERSION/bin/gdal-config python -m pip install --upgrade --force-reinstall"# -e .[test,plot]"
-- pip install coveralls
-- pip install gdal==1.11.2 --global-option=build_ext --global-option="-I$HOME/miniconda/include" #"-I$GDALINST/gdal-$GDALVERSION/include" #$($GDALINST/gdal-$GDALVERSION/bin/gdal-config --version) --global-option=build_ext --global-option="-I$GDALINST/gdal-$GDALVERSION/include" 
+- conda config --append channels conda-forge
+
+- deps='pip numpy scipy matplotlib basemap shapely nose netcdf4 coverage coveralls pycurl pyproj seaborn simplejson sqlite statsmodels libgdal gdal'
+- conda create -q -n tcrm-env --yes $deps "python=$TRAVIS_PYTHON_VERSION"
+- source activate tcrm-env
+
 branches:
   except:
     - config
     - notebooks
 script:
   - python installer/setup.py build_ext -i
-  - nosetests -v --with-coverage
+  - nosetests -v --with-coverage --cover-package=.
 after_success: coveralls
 notifications:
   slack:

Evaluate/genesisDensity.py

@@ -31,8 +31,8 @@
 
 from PlotInterface.maps import FilledContourMapFigure, saveFigure, levels
 
-LOG = logging.getLogger(__name__)
-LOG.addHandler(logging.NullHandler())
+log = logging.getLogger(__name__)
+log.addHandler(logging.NullHandler())
 
 def loadTracks(trackfile):
     """
@@ -73,7 +73,7 @@ def loadTracksFromPath(path):
     files = os.listdir(path)
     trackfiles = [pjoin(path, f) for f in files if f.startswith('tracks')]
     msg = 'Processing %d track files in %s' % (len(trackfiles), path)
-    LOG.info(msg)
+    log.info(msg)
     return loadTracksFromFiles(sorted(trackfiles))
 
 class GenesisDensity(object):
@@ -166,7 +166,7 @@ def _calculate(self, tracks):
 
         :param tracks: Collection of :class:`Track` objects.
         """
-        LOG.debug("Calculating PDF for set of {0:d} tracks".format(len(tracks)))
+        log.debug("Calculating PDF for set of {0:d} tracks".format(len(tracks)))
 
         hist = ma.zeros((len(self.lon_range) - 1,
                          len(self.lat_range) - 1))
@@ -216,7 +216,7 @@ def calculateMeans(self):
     @disableOnWorkers
     def historic(self):
         """Load historic data and calculate histogram"""
-        LOG.info("Processing historic track records")
+        log.info("Processing historic track records")
         config = ConfigParser()
         config.read(self.configFile)
         inputFile = config.get('DataProcess', 'InputFile')
@@ -229,7 +229,7 @@ def historic(self):
             tracks = loadTrackFile(self.configFile, inputFile, source)
 
         except (TypeError, IOError, ValueError):
-            LOG.critical("Cannot load historical track file: {0}".\
+            log.critical("Cannot load historical track file: {0}".\
                          format(inputFile))
             raise
         else:
@@ -239,7 +239,7 @@ def historic(self):
                 startYr = min(startYr, min(t.Year))
                 endYr = max(endYr, max(t.Year))
             numYears = endYr - startYr
-            LOG.info("Range of years: %d - %d" % (startYr, endYr))
+            log.info("Range of years: %d - %d" % (startYr, endYr))
             try:
                 self.hist = self._calculate(tracks)
             #self.hist = self._calculate(tracks) / numYears
@@ -265,7 +265,7 @@ def synthetic(self):
             n = 0
             for d in range(1, pp.size()):
                 pp.send(trackfiles[w], destination=d, tag=work_tag)
-                LOG.debug("Processing track file {0:d} of {1:d}".\
+                log.debug("Processing track file {0:d} of {1:d}".\
                           format(w, len(trackfiles)))
                 w += 1
 
@@ -279,7 +279,7 @@ def synthetic(self):
                 d = status.source
                 if w < len(trackfiles):
                     pp.send(trackfiles[w], destination=d, tag=work_tag)
-                    LOG.debug("Processing track file {0:d} of {1:d}".\
+                    log.debug("Processing track file {0:d} of {1:d}".\
                               format(w, len(trackfiles)))
                     w += 1
                 else:
@@ -294,14 +294,14 @@ def synthetic(self):
                 if trackfile is None:
                     break
 
-                LOG.debug("Processing %s", trackfile)
+                log.debug("Processing %s", trackfile)
                 tracks = loadTracks(trackfile)
                 results = self._calculate(tracks) #/ self.synNumYears
                 pp.send(results, destination=0, tag=result_tag)
 
         elif (pp.size() == 1) and (pp.rank() == 0):
             for n, trackfile in enumerate(trackfiles):
-                LOG.debug("Processing track file {0:d} of {1:d}".\
+                log.debug("Processing track file {0:d} of {1:d}".\
                           format(n + 1, len(trackfiles)))
                 tracks = loadTracks(trackfile)
                 self.synHist[n, :, :] = self._calculate(tracks) #/ self.synNumYears
@@ -314,12 +314,12 @@ def save(self):
 
         # Simple sanity check (should also include the synthetic data):
         if not hasattr(self, 'hist'):
-            LOG.critical("No historical data available!")
-            LOG.critical(("Check that data has been processed "
+            log.critical("No historical data available!")
+            log.critical(("Check that data has been processed "
                           "before trying to save data"))
             return
 
-        LOG.info('Saving genesis density data to {0}'.format(dataFile))
+        log.info('Saving genesis density data to {0}'.format(dataFile))
         dimensions = {
             0: {
                 'name': 'lat',

PlotInterface/maps.py

@@ -102,9 +102,14 @@ def selectColormap(data_range, percent=0.1):
         palette = sns.color_palette("RdBu", 7)
         cmap = sns.blend_palette(palette, as_cmap=True)
     else:
-        palette = [(1, 1, 1), (0.000, 0.627, 0.235), (0.412, 0.627, 0.235), (0.663, 0.780, 0.282),
-        (0.957, 0.812, 0.000), (0.925, 0.643, 0.016), (0.835, 0.314, 0.118),
-        (0.780, 0.086, 0.118)]
+        palette = [(1, 1, 1),
+                   (0.000, 0.627, 0.235),
+                   (0.412, 0.627, 0.235), 
+                   (0.663, 0.780, 0.282),
+                   (0.957, 0.812, 0.000),
+                   (0.925, 0.643, 0.016),
+                   (0.835, 0.314, 0.118),
+                   (0.780, 0.086, 0.118)]
         cmap = sns.blend_palette(palette, as_cmap=True)
 
     return cmap
@@ -193,9 +198,9 @@ def addGraticule(self, axes, mapobj):
         parallels = np.arange(dl*np.floor(ymin / dl),
                               dl*np.ceil(ymax / dl) + dl, dl)
 
-        mapobj.drawparallels(parallels, linewidth=0.25,
+        mapobj.drawparallels(parallels, linewidth=0.25, fontsize='x-small',
                              labels=[1, 0, 0, 1], style="italic")
-        mapobj.drawmeridians(meridians, linewidth=0.25,
+        mapobj.drawmeridians(meridians, linewidth=0.25, fontsize='x-small',
                              labels=[1, 0, 0, 1], style='italic')
         axes.tick_params(axis='both', direction='out', length=4,
                          width=1, color='k')
@@ -439,7 +444,7 @@ def subplot(self, axes, subfigure):
         vmax = datarange[1]
         CS = mapobj.pcolormesh(mx, my, data, vmin=vmin,
                                vmax=vmax, cmap=cmap)
-        CB = mapobj.colorbar(CS, location='right', pad='5%',
+        CB = mapobj.colorbar(CS, location='bottom', pad='10%',
                              fig=self, ax=axes)
         CB.set_label(cbarlab)
         axes.set_title(title)

ProcessMultipliers/processMultipliers.py

@@ -501,11 +501,11 @@ def processMult(wspd, uu, vv, lon, lat, working_dir, m4_max_file = 'm4_ne.tif'):
 
     reprojectDataset(wind_raster, m4_max_file, wind_prj_file)
     reprojectDataset(bear_raster, m4_max_file, bear_prj_file,
-                     resampling_method = GRA_NearestNeighbour)
+                     resampling_method = gdalconst.GRA_NearestNeighbour)
     reprojectDataset(uu_raster, m4_max_file, uu_prj_file,
-                     resampling_method = GRA_NearestNeighbour)
+                     resampling_method = gdalconst.GRA_NearestNeighbour)
     reprojectDataset(vv_raster, m4_max_file, vv_prj_file,
-                     resampling_method = GRA_NearestNeighbour)
+                     resampling_method = gdalconst.GRA_NearestNeighbour)
 
     wind_prj_ds = gdal.Open(wind_prj_file, gdal.GA_ReadOnly)
     wind_prj = wind_prj_ds.GetRasterBand(1)
@@ -666,7 +666,6 @@ def main(self):
         gM.combineDirections(self.dirns, self.working_dir)
 
         # Load the wind data:
-        from netCDF4 import Dataset
         log.info("Loading regional wind data from {0}".format(self.gust_file))
         ncobj = Dataset(self.gust_file, 'r')
 

Utilities/get_multipliers.py

@@ -22,6 +22,8 @@
 from osgeo import gdal
 from osgeo.gdalnumeric import *
 from osgeo.gdalconst import *
+from osgeo.gdal_array import BandReadAsArray, CopyDatasetInfo, BandWriteArray
+
 import numpy as np
 import numpy.ma as ma
 import logging as log
@@ -228,7 +230,7 @@ def main(configFile):
     type_mapping = {'shielding': 'Ms', 'terrain': 'Mz', 'topographic': 'Mt'}
     dirns = ['e', 'n', 'ne', 'nw', 's', 'se', 'sw', 'w']
 
-    num_tiles = copyTranslateMultipliers(configFile, type_mapping, output_path)
+    copyTranslateMultipliers(configFile, type_mapping, output_path)
     mergeWindMultipliers(type_mapping, dirns, output_path)
     combineDirections(dirns, output_path)
 

Utilities/nctools.py

@@ -194,7 +194,7 @@ def ncGetTimes(ncobj, name='time'):
     else:
         calendar = 'standard'
 
-    dates = num2date(times[:], units, calendar)
+    dates = num2date(times[:].data, units, calendar)
 
     return np.array(dates, dtype=datetime)
 

database/__init__.py

@@ -363,7 +363,7 @@ def insertEvents(self, eventparams):
         try:
             self.execute(INSEVENTS, eventparams)
 
-        except IntegrityError as err:
+        except sqlite3.IntegrityError as err:
             log.exception("Problem inserting events into tblEvents: ")
             log.exception("Pre-existing event with the same eventNumber attribute")
             log.exception("Check that you are not overwriting an existing database.")

matplotlibrc

@@ -1,23 +1,53 @@
 # TCRM Plotting Configuration
 backend: TkAgg
-lines.linewidth: 0.5
-lines.markersize: 1
-font.size: 12.0
-text.color: black
-text.antialiased: True
-axes.titlesize: large
-axes.labelsize: small
-axes.labelcolor: black
-axes.color_cycle: k, b, g, r, c, m, y
+
+#### LINES
+lines.linewidth   : 0.5
+lines.linestyle   : -
+lines.markersize  : 1
+lines.color       : C0
+
+#### FONTS
+font.family       : sans-serif
+font.size         : 10.0
+font.style        : normal
+
+
+#### TEXT
+text.color        : black
+text.antialiased  : True
+
+#### AXES
+axes.titlesize    : large
+axes.labelsize    : small
+axes.labelcolor   : black
+axes.prop_cycle   : cycler('color', ['1f77b4', 'ff7f0e', '2ca02c', 'd62728', '9467bd', '8c564b', 'e377c2', '7f7f7f', 'bcbd22', '17becf'])
+
+#### LEGEND
 legend.fancybox: True
 legend.frameon: True
-xtick.direction: out
-ytick.direction: out
-xtick.labelsize: x-small
-ytick.labelsize: x-small
+
+#### TICKS
+xtick.direction   : out
+ytick.direction   : out
+xtick.labelsize   : x-small
+ytick.labelsize   : x-small
+
+#### CONTOURS
 contour.negative_linestyle: dashed
-figure.dpi: 300
-figure.figsize: 16, 12
+
+#### GRIDS
+grid.color        : 808080
+grid.linestyle    : --
+grid.linewidth    : 0.5
+grid.alpha        : 0.75
+
+#### FIGURES
+figure.dpi        : 300
+figure.figsize    : 12, 8
 figure.subplot.left: 0.0675
 figure.subplot.right: 0.9125
-image.cmap: Blues
+
+#### IMAGES
+image.aspect      : equal
+image.cmap        : viridis

requirements.txt

@@ -1,10 +1,4 @@
-six>=1.7.3
-python-dateutil
-tornado
-pyparsing
 netCDF4>=1.0.8
-pytz>=2014.4
 seaborn>=0.8.1
 simplejson==3.6.0
-Shapely>=1.2.15
 statsmodels>=0.5.0

tests/test_maps.py

@@ -61,7 +61,7 @@ def assertColorMapEqual(self, actual, expected):
 
     def setUp(self):
         import seaborn as sns
-        palette = [(0.000, 0.627, 0.235), (0.412, 0.627, 0.235), (0.663, 0.780, 0.282),
+        palette = [(1, 1, 1), (0.000, 0.627, 0.235), (0.412, 0.627, 0.235), (0.663, 0.780, 0.282),
         (0.957, 0.812, 0.000), (0.925, 0.643, 0.016), (0.835, 0.314, 0.118),
         (0.780, 0.086, 0.118)]
         div_pal = sns.color_palette("RdBu", 7)

travis_gdal_install.sh

@@ -43,7 +43,7 @@ GDALOPTS="  --with-ogr \
             --without-perl \
             --without-php \
             --without-ruby \
-            --without-python \
+            --with-python \
             --with-static-proj4=/usr/lib"
 
 # Create build dir if not exists
@@ -89,4 +89,4 @@ if [ "$GDALVERSION" != "trunk" -a ! -d "$GDALINST/gdal-$GDALVERSION" ]; then
 fi
 
 # change back to travis build dir
-cd $TRAVIS_BUILD_DIR
+cd $TRAVIS_BUILD_DIR