fixed conda env (#23)

* fixed conda env * skip test if data is not available
bird-house · Oct 30, 2018 · 124594b · 124594b
1 parent 2ab0196
commit 124594b
Show file tree

Hide file tree

Showing 7 changed files with 175 additions and 11 deletions.
diff --git a/environment.yml b/environment.yml
@@ -4,12 +4,16 @@ channels:
 - conda-forge
 - defaults
 dependencies:
-- pywps>=4.1.6
+- six
+# pywps
+- pywps=4.1
 - jinja2
 - click
 - psutil
 # cdo
+- cdo=1.9.4
 - python-cdo
 - xarray
+- netcdf4=1.4.1
 # compliance checker ioos
-- compliance-checker
+- compliance-checker=4.1
diff --git a/hummingbird/default.cfg b/hummingbird/default.cfg
@@ -10,7 +10,7 @@ provider_url=http://hummingbird.readthedocs.org/en/latest/
 url = http://localhost:5000/wps
 outputurl = http://localhost:5000/outputs
 allowedinputpaths = /
-maxsingleinputsize = 200mb
+maxsingleinputsize = 500mb
 maxprocesses = 10
 parallelprocesses = 2
 

diff --git a/hummingbird/processing.py b/hummingbird/processing.py
@@ -1,7 +1,9 @@
+import six
 import os
 import glob
 import subprocess
 from subprocess import check_output, CalledProcessError
+
 from cdo import Cdo
 cdo_version = Cdo().version()
 
@@ -24,17 +26,17 @@ def ncdump(dataset):
 
     try:
         output = check_output(['ncdump', '-h', dataset])
-        if not isinstance(output, str):
+        if not isinstance(output, six.string_types):
             output = output.decode('utf-8')
         lines = output.split('\n')
         # replace the filename for safety
         dataset_id = os.path.basename(dataset)  # 'uploaded-file'
-        lines[0] = 'netcdf %s {' % dataset_id
+        lines[0] = 'netcdf {} {{'.format(dataset_id)
         # decode to ascii
-        filtered_lines = [str(line) + '\n' for line in lines]
-    except CalledProcessError as err:
-        LOGGER.exception("could not generate ncdump")
-        return "Error: generating ncdump failed. Output: {0.output}".format(err)
+        filtered_lines = ['{}\n'.format(line) for line in lines]
+    except Exception as err:
+        LOGGER.error("Could not generate ncdump: {}".format(err))
+        return "Error: generating ncdump failed"
     return filtered_lines
 
 

diff --git a/spec-file.txt b/spec-file.txt
@@ -0,0 +1,139 @@
+# This file may be used to create an environment using:
+# $ conda create --name <env> --file <this file>
+# platform: linux-64
+@EXPLICIT
+https://repo.anaconda.com/pkgs/main/linux-64/blas-1.0-mkl.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2018.10.15-ha4d7672_0.tar.bz2
+https://repo.anaconda.com/pkgs/main/linux-64/intel-openmp-2019.0-118.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-7.2.0-hdf63c60_3.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libgfortran-3.0.0-1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-7.2.0-hdf63c60_3.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-7.2.0-hdf63c60_3.tar.bz2
+https://repo.anaconda.com/pkgs/free/linux-64/util-linux-2.21-0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.6-h470a237_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/expat-2.2.5-hfc679d8_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.8-h470a237_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/icu-58.2-hfc679d8_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/jpeg-9c-h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libffi-3.2.1-hfc679d8_5.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.15-h470a237_3.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libmagic-5.33-h470a237_0.tar.bz2
+https://repo.anaconda.com/pkgs/main/linux-64/mkl-2019.0-118.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.1-hfc679d8_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/openssl-1.0.2p-h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/proj4-4.9.3-h470a237_8.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.4-h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/yaml-0.1.7-h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.11-h470a237_3.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.13-h951d187_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.10.2-hc401514_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/jasper-1.900.1-hff1ad4c_5.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20170329-haf1bffa_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.35-ha92aebf_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.8.0-h5b517e9_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.0.9-he6b73bb_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.9.8-h422b904_5.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/readline-7.0-haf1bffa_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.8-ha92aebf_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.27.6-h3a4f0e9_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/freetype-2.9.1-h6debe1e_4.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/krb5-1.16.1-hbb41f41_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.32-h88dbc4e_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.25.2-hb1c47c0_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.61.1-hbdb9355_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/python-3.6.6-h5001a0f_3.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.12-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/asn1crypto-0.24.0-py36_1003.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/atomicwrites-1.2.1-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/attrs-18.2.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/bumpversion-0.5.3-py36_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/certifi-2018.10.15-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/chardet-3.0.4-py36_1003.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/click-7.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/cloudpickle-0.6.1-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/cryptography-vectors-2.3.1-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/curl-7.61.1-h74213dd_2.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/dask-core-0.20.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/docutils-0.14-py36_1001.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/heapdict-1.0.0-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/idna-2.7-py36_1002.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/imagesize-1.1.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/jsonschema-2.6.0-py36_1002.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/locket-0.2.0-py_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/lxml-4.2.5-py36hc9114bc_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/markupsafe-1.0-py36h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/mccabe-0.6.1-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-0.5.6-py36h2d50403_3.tar.bz2
+https://repo.anaconda.com/pkgs/main/linux-64/numpy-base-1.15.0-py36h3dfced4_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/olefile-0.46-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pluggy-0.8.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/psutil-5.4.7-py36h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/py-1.7.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pycodestyle-2.4.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pycparser-2.19-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pyflakes-2.0.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pygeoif-0.7-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pyparsing-2.2.2-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pysocks-1.6.8-py36_1002.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/python-magic-0.4.15-py36_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pytz-2018.7-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pytzdata-2018.7-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pyyaml-3.13-py36h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/regex-2017.07.28-py36_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/six-1.11.0-py36_1001.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-1.2.1-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/sortedcontainers-2.0.5-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-websupport-1.1.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/sqlalchemy-1.2.12-py36h470a237_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/tabulate-0.8.2-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/tblib-1.3.2-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/toolz-0.9.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/tornado-5.1.1-py36h470a237_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/werkzeug-0.14.1-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/babel-2.6.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/cffi-1.11.5-py36h5e8e0c9_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.9.0.1-py36h470a237_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/isodate-0.6.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.6.1-h628ed10_200.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/more-itertools-4.3.0-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/packaging-18.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/partd-0.3.9-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pillow-5.3.0-py36hc736899_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.7.5-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/setuptools-40.5.0-py36_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/tzlocal-1.5.1-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/zict-0.1.3-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/cryptography-2.3.1-py36hdffb7b8_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/distributed-1.24.0-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/eccodes-2.8.2-ha8b302a_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/flake8-3.6.0-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/jinja2-2.10-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pendulum-2.0.3-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pygments-2.2.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pytest-3.9.3-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/wheel-0.32.2-py36_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/cdo-1.9.4-h1d09544_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pip-18.1-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pyopenssl-18.0.0-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/python-cdo-1.4.0-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/urllib3-1.23-py36_1001.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/requests-2.20.0-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/sphinx-1.8.1-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/bokeh-1.0.0-py36_1000.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/bottleneck-1.2.1-py36h7eb728f_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/cftime-1.0.2.1-py36h7eb728f_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/cf_units-2.0.1-py36h7eb728f_2.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/h5netcdf-0.6.2-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/h5py-2.8.0-py36h7eb728f_3.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/mkl_fft-1.0.6-py36_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/mkl_random-1.0.1-py36_0.tar.bz2
+https://repo.anaconda.com/pkgs/main/linux-64/numpy-1.15.0-py36h1b885b7_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.4.1-py36ha292673_200.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pandas-0.23.4-py36hf8a1672_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pyproj-1.9.5.1-py36h508ed2a_5.tar.bz2
+https://repo.anaconda.com/pkgs/main/linux-64/scipy-1.1.0-py36hc49cb51_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/dask-0.20.0-py_0.tar.bz2
+https://conda.anaconda.org/birdhouse/noarch/owslib-0.16.0-py_57.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/compliance-checker-4.1.1-py_0.tar.bz2
+https://conda.anaconda.org/birdhouse/linux-64/pywps-4.1.21-py36_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/xarray-0.10.9-py36_0.tar.bz2
diff --git a/tests/common.py b/tests/common.py
@@ -1,4 +1,5 @@
 import os
+import requests
 from pywps.tests import WpsClient, WpsTestResponse
 
 TESTS_HOME = os.path.abspath(os.path.dirname(__file__))
@@ -22,6 +23,20 @@ def resource_file(filepath):
 }
 
 
+def service_ok(url, timeout=5):
+    try:
+        ok = requests.get(url, timeout=timeout).ok
+    except requests.exceptions.ReadTimeout:
+        print('Read Timeout')
+        ok = False
+    except requests.exceptions.ConnectTimeout:
+        print('Connect Timeout')
+        ok = False
+    except Exception:
+        ok = False
+    return ok
+
+
 class WpsTestClient(WpsClient):
 
     def get(self, *args, **kwargs):

diff --git a/tests/test_wps_cdo_copy.py b/tests/test_wps_cdo_copy.py
@@ -2,7 +2,7 @@
 from pywps import Service
 from pywps.tests import assert_response_success
 
-from .common import TESTDATA, client_for
+from .common import TESTDATA, client_for, service_ok
 from hummingbird.processes.wps_cdo_copy import CDOCopy
 
 
@@ -17,6 +17,8 @@ def test_wps_cdo_copy_file():
 
 
 @pytest.mark.online
+@pytest.mark.skipif(not service_ok(TESTDATA['noaa_dap_1']),
+                    reason="Test data not available.")
 def test_wps_cdo_copy_opendap():
     client = client_for(Service(processes=[CDOCopy()]))
     datainputs = "dataset_opendap=@xlink:href={0};".format(TESTDATA['noaa_dap_1'])

diff --git a/tests/test_wps_cdo_ensembles.py b/tests/test_wps_cdo_ensembles.py
@@ -2,7 +2,7 @@
 from pywps import Service
 from pywps.tests import assert_response_success
 
-from .common import TESTDATA, client_for
+from .common import TESTDATA, client_for, service_ok
 from hummingbird.processes.wps_cdo_ensembles import CDOEnsembles
 
 
@@ -17,6 +17,8 @@ def test_wps_ensembles_file():
 
 
 @pytest.mark.online
+@pytest.mark.skipif(not service_ok(TESTDATA['noaa_dap_1']),
+                    reason="Test data not available.")
 def test_wps_ensembles_opendap():
     client = client_for(Service(processes=[CDOEnsembles()]))
     datainputs = "dataset_opendap=@xlink:href={0};operator=ensmean".format(TESTDATA['noaa_dap_1'])