Build matrix / env vars for travis

.travis.yml

@@ -1,24 +1,49 @@
-
 language: python
 
-python:
-  - "3.5"
+dist: trusty
 
 env:
-  - PYTHON_TEST_VERSION=2.7
-  - PYTHON_TEST_VERSION=3.5
-  - PYTHON_TEST_VERSION=3.6
+  global:
+    - EARTHIO_TEST_ENV=earth-test-env
+    - ELM_EXAMPLE_DATA_PATH=/tmp/elm-data
+    - EARTHIO_CHANNEL_STR=" -c ioam -c conda-forge -c scitools/label/dev "
+
+  matrix:
+    - PYTHON=3.6 NUMPY=1.12
+    - PYTHON=3.5 NUMPY=1.11
+    - PYTHON=2.7 NUMPY=1.9
+
+matrix:
+  # Support for these versions of Python is still a work-in-progress
+  allow_failures:
+    - env: PYTHON=2.7 NUMPY=1.9
+
+  # The build result is determined as soon as the required builds pass/fail
+  fast_finish: true
+
+before_install:
+  - mkdir -p $ELM_EXAMPLE_DATA_PATH
+  - rm -rf $ELM_EXAMPLE_DATA_PATH/*
 
 install:
-  # Name of the environment for testing
-  - export EARTHIO_TEST_ENV=earth-env-test
-  # Path to example data downloaded.  Dont put it in earthio dir
-  - export ELM_EXAMPLE_DATA_PATH=/tmp/elm-data
-  - mkdir -p $ELM_EXAMPLE_DATA_PATH; rm -rf $ELM_EXAMPLE_DATA_PATH/*
-  # Build the env
   - MAKE_MINICONDA=1 . build_earthio_env.sh
 
 script:
   - cd $EARTHIO_BUILD_DIR
-  - py.test -m 'not slow and not requires_elm' -v earthio/tests
-  - rm -rf $ELM_EXAMPLE_DATA_PATH/*
+  - . activate $EARTHIO_TEST_ENV
+  - py.test -m 'not slow and not requires_elm' --ignore=earthio/tests/test_netcdf.py -v earthio/tests
+  - rm -rf $ELM_EXAMPLE_DATA_PATH/*
+
+notifications:
+  on_success: change
+  on_failure: always
+  flowdock: $FD_TOKEN
+
+deploy:
+  - provider: script
+    script:
+      - conda install --name root anaconda-client python=3.5 conda-build==2.0.4 && conda build $EARTHIO_CHANNEL_STR --output --python $PYTHON --numpy $NUMPY conda.recipe | xargs conda convert -p all -o _pkgs && find _pkgs -type f -name "*.tar.bz2" -exec anaconda -t $ANACONDA_UPLOAD_TOKEN upload --user $ANACONDA_UPLOAD_USER --label dev {} \+
+    on:
+      tags: false
+      all_branches: true
+    skip_cleanup: true

build_earthio_env.sh

@@ -1,47 +1,45 @@
 source deactivate;
+
 export EARTHIO_BUILD_DIR=`pwd -P`
-if [ "$EARTHIO_CHANNEL_STR" = "" ];then
-    export EARTHIO_CHANNEL_STR=" -c ioam -c conda-forge -c scitools/label/dev ";
-fi
-if [ "$EARTHIO_TEST_ENV" = "" ];then
-    export EARTHIO_TEST_ENV=earth-env-test;
-fi
-build_earthio_env(){
-    if [ "$PYTHON_TEST_VERSION" = "" ];then
-        echo Env variable PYTHON_TEST_VERSION is not defined. Set it to 2.7, 3.5 or 3.6 - FAIL ;
-        return 1;
-    fi
-    if [ "$MAKE_MINICONDA" = "" ];then
-        echo
-    else
-        wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh || return 1;
-        bash miniconda.sh -b -p $HOME/miniconda || return 1;
-        export PATH="$HOME/miniconda/bin:$PATH";
+export EARTHIO_CHANNEL_STR="${EARTHIO_CHANNEL_STR:- -c ioam -c conda-forge -c scitools/label/dev }"
+export EARTHIO_TEST_ENV="${EARTHIO_TEST_ENV:-earth-env-test}"
+export EARTHIO_INSTALL_METHOD="${EARTHIO_INSTALL_METHOD:-conda}"
+export ELM_EXAMPLE_DATA_PATH="${ELM_EXAMPLE_DATA_PATH:-${EARTHIO_BUILD_DIR}/../elm-data}";
+export PYTHON=${PYTHON:-3.5}
+export NUMPY=${NUMPY:-1.11}
+
+build_earthio_env() {
+    set -e
+
+    if [ -n "$MAKE_MINICONDA" ];then
+        wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+        bash miniconda.sh -b -p $HOME/miniconda
+        export PATH="$HOME/miniconda/bin:$PATH"
     fi
-    conda config --set always_yes true;
 
+    conda config --set always_yes true;
     conda install --name root conda conda-build;
     conda env remove --name ${EARTHIO_TEST_ENV} &> /dev/null;
-    if [ "$EARTHIO_INSTALL_METHOD" = "" ];then
-        export EARTHIO_INSTALL_METHOD="conda";
-    fi
-    if [ "$EARTHIO_INSTALL_METHOD" = "git" ];then
-        conda env create -n ${EARTHIO_TEST_ENV} -f environment.yml  || return 1;
-        source activate ${EARTHIO_TEST_ENV} || return 1;
-        python setup.py develop || return 1;
+
+    if [ "x$EARTHIO_INSTALL_METHOD" = "xgit" ];then
+        conda env create -n ${EARTHIO_TEST_ENV} -f environment.yml
+        source activate ${EARTHIO_TEST_ENV}
+        python setup.py develop
     else
-        conda build $EARTHIO_CHANNEL_STR --python $PYTHON_TEST_VERSION conda.recipe || return 1;
-        conda create --use-local --name $EARTHIO_TEST_ENV $EARTHIO_CHANNEL_STR python=$PYTHON_TEST_VERSION earthio || return 1;
-        source activate ${EARTHIO_TEST_ENV}  || return 1;
+        conda build $EARTHIO_CHANNEL_STR --python $PYTHON --numpy $NUMPY conda.recipe
+        conda create --use-local --name $EARTHIO_TEST_ENV $EARTHIO_CHANNEL_STR python=$PYTHON numpy=$NUMPY earthio
+        source activate ${EARTHIO_TEST_ENV}
     fi
-    if [ "$ELM_EXAMPLE_DATA_PATH" = "" ];then
-        export ELM_EXAMPLE_DATA_PATH="${EARTHIO_BUILD_DIR}/../elm-data";
-    fi
-    conda install -c defaults -c conda-forge requests pbzip2 python-magic six  || return 1; # for download_test_data.py
-    if [ "$IGNORE_ELM_DATA_DOWNLOAD" = "" ];then
-        mkdir -p $ELM_EXAMPLE_DATA_PATH && cd $ELM_EXAMPLE_DATA_PATH && python "${EARTHIO_BUILD_DIR}/scripts/download_test_data.py" || return 1;
+
+    if [ "x$IGNORE_ELM_DATA_DOWNLOAD" = "x" ];then
+        conda install -c defaults -c conda-forge requests pbzip2 python-magic six
+        mkdir -p $ELM_EXAMPLE_DATA_PATH
+        df -h
+        pushd $ELM_EXAMPLE_DATA_PATH && python "${EARTHIO_BUILD_DIR}/scripts/download_test_data.py" --files hdf4.tar.bz2 hdf5.tar.bz2 tif.tar.bz2 && popd
+        df -h
     fi
-    cd $EARTHIO_BUILD_DIR || return 1;
+
+    set +e
 }
-build_earthio_env && source activate ${EARTHIO_TEST_ENV} && echo OK
 
+build_earthio_env && source activate ${EARTHIO_TEST_ENV} && echo OK

earthio/tests/test_elm_store.py

@@ -19,7 +19,13 @@
               'tif': tif_band_specs,
               'netcdf': ['HQobservationTime']}
 
-FILES = {'hdf': HDF4_FILES, 'netcdf': NETCDF_FILES, 'tif': TIF_FILES}
+FILES = {}
+if HDF4_FILES:
+    FILES['hdf'] = HDF4_FILES
+if NETCDF_FILES:
+    FILES['netcdf'] = NETCDF_FILES
+if TIF_FILES:
+    FILES['tif'] = TIF_FILES
 
 
 def _setup(ftype, fnames_list):

earthio/tests/test_hdf4.py

@@ -16,7 +16,6 @@
                                     assertions_on_metadata,
                                     assertions_on_band_metadata)
 
-HDF4_DIR = os.path.dirname(HDF4_FILES[0])
 kwargs = {}
 band_specs = [
     BandSpec('long_name', 'Band 1 ', 'band_1', **kwargs),

earthio/tests/test_hdf5.py

@@ -18,7 +18,6 @@
 
 from earthio.util import BandSpec
 
-HDF5_DIR = os.path.dirname(HDF5_FILES[0])
 
 def get_band_specs(filename):
     if os.path.basename(filename).startswith('3B-MO'):

earthio/tests/test_load_array.py

@@ -7,10 +7,16 @@
                                     HDF4_FILES, NETCDF_FILES,
                                     ELM_HAS_EXAMPLES)
 from earthio.tests.test_tif import band_specs as tif_band_specs
-TRIALS = {'tif': os.path.dirname(TIF_FILES[0]),
-          'hdf5': HDF5_FILES[0],
-          'hdf4': HDF4_FILES[0],
-          'netcdf': NETCDF_FILES[0]}
+
+TRIALS = {}
+if TIF_FILES:
+    TRIALS['tif'] = os.path.dirname(TIF_FILES[0])
+if HDF5_FILES:
+    TRIALS['hdf5'] = HDF5_FILES[0]
+if HDF4_FILES:
+    TRIALS['hdf4'] = HDF4_FILES[0]
+if NETCDF_FILES:
+    TRIALS['netcdf'] = NETCDF_FILES[0]
 
 @pytest.mark.skipif(not ELM_HAS_EXAMPLES,
                reason='elm-data repo has not been cloned')

earthio/tests/test_netcdf.py

@@ -9,7 +9,6 @@
                                     NETCDF_FILES,
                                     assertions_on_metadata)
 from earthio.util import BandSpec
-NETCDF_DIR = os.path.dirname(NETCDF_FILES[0])
 
 variables_dict = dict(HQobservationTime='HQobservationTime')
 variables_list = ['HQobservationTime']

earthio/tests/test_readers_general.py

@@ -8,7 +8,7 @@
 from earthio import *
 from earthio.tests.test_hdf4 import HDF4_FILES, band_specs as hdf4_band_specs
 from earthio.tests.test_hdf5 import HDF5_FILES, get_band_specs
-from earthio.tests.test_tif import TIF_DIR, band_specs as tif_band_specs
+from earthio.tests.test_tif import TIF_FILES, band_specs as tif_band_specs
 
 
 def random_elm_store_no_meta(width=100, height=200):
@@ -72,6 +72,7 @@ def test_reader_kwargs_window(ftype):
         full_es = load_hdf4_array(HDF4_FILES[0], meta, band_specs)
     elif ftype == 'tif':
         band_specs = tif_band_specs[:2]
+        TIF_DIR = os.path.dirname(TIF_FILES[0])
         meta = load_dir_of_tifs_meta(TIF_DIR, band_specs=band_specs)
         full_es = load_dir_of_tifs_array(TIF_DIR, meta, band_specs)
     band_specs_window = []

earthio/tests/test_tif.py

@@ -18,7 +18,6 @@
 
 from earthio.util import BandSpec
 
-TIF_DIR = os.path.dirname(TIF_FILES[0])
 band_specs = [
     BandSpec('name', '_B1.TIF', 'band_1'),
     BandSpec('name', '_B2.TIF', 'band_2'),
@@ -34,6 +33,7 @@
 @pytest.mark.skipif(not ELM_HAS_EXAMPLES,
                reason='elm-data repo has not been cloned')
 def test_read_meta():
+    TIF_DIR = os.path.dirname(TIF_FILES[0])
     for tif in TIF_FILES:
         raster, meta = load_tif_meta(tif)
         assert hasattr(raster, 'read')
@@ -52,6 +52,7 @@ def test_read_meta():
 @pytest.mark.skipif(not ELM_HAS_EXAMPLES,
                reason='elm-data repo has not been cloned')
 def test_read_array():
+    TIF_DIR = os.path.dirname(TIF_FILES[0])
     meta = load_dir_of_tifs_meta(TIF_DIR, band_specs)
     es = load_dir_of_tifs_array(TIF_DIR, meta, band_specs)
     for var in es.data_vars:
@@ -72,6 +73,7 @@ def test_read_array():
 @pytest.mark.skipif(not ELM_HAS_EXAMPLES,
                reason='elm-data repo has not been cloned')
 def test_reader_kwargs():
+    TIF_DIR = os.path.dirname(TIF_FILES[0])
     band_specs_kwargs = []
     for b in band_specs:
         b = attr.asdict(b)