Merge pull request #4 from bird-house/update-esgf-cwt-api-20190704

update to esgf cwt api 2019-07-04

Makefile

@@ -23,6 +23,7 @@ help:
 	@echo "  install     to install $(APP_NAME) by running 'python setup.py develop'."
 	@echo "  start       to start $(APP_NAME) service as daemon (background process)."
 	@echo "  stop        to stop $(APP_NAME) service."
+	@echo "  restart     to restart $(APP_NAME) service."
 	@echo "  status      to show status of $(APP_NAME) service."
 	@echo "  clean       to remove *all* files that are not controlled by 'git'. WARNING: use it *only* if you know what you do!"
 	@echo "\nTesting targets:"
@@ -31,8 +32,10 @@ help:
 	@echo "  pep8        to run pep8 code style checks."
 	@echo "\nSphinx targets:"
 	@echo "  docs        to generate HTML documentation with Sphinx."
+	@echo "\nDeployment targets:"
+	@echo "  spec        to generate Conda spec file."
 
-## Anaconda targets
+## Conda targets
 
 .PHONY: check_conda
 check_conda:
@@ -50,6 +53,11 @@ envclean: check_conda
 	@echo "Removing conda env $(CONDA_ENV)"
 	@-"$(CONDA)" remove -n $(CONDA_ENV) --yes --all
 
+.PHONY: spec
+spec: check_conda
+	@echo "Updating conda environment specification file ..."
+	@-"$(CONDA)" list -n $(CONDA_ENV) --explicit > spec-file.txt
+
 ## Build targets
 
 .PHONY: bootstrap
@@ -59,7 +67,7 @@ bootstrap: check_conda conda_env bootstrap_dev
 .PHONY: bootstrap_dev
 bootstrap_dev:
 	@echo "Installing development requirements for tests and docs ..."
-	@-bash -c "$(CONDA) install -y -n $(CONDA_ENV) -c conda-forge pytest flake8 sphinx bumpversion"
+	@-bash -c "$(CONDA) install -y -n $(CONDA_ENV) pytest flake8 sphinx gunicorn psycopg2"
 	@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && pip install -r requirements_dev.txt"
 
 .PHONY: install
@@ -78,6 +86,10 @@ stop: check_conda
 	@echo "Stopping application ..."
 	@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && $(APP_NAME) stop"
 
+.PHONY: restart
+restart: stop start
+	@echo "Restarting application ..."
+
 .PHONY: status
 status: check_conda
 	@echo "Show status ..."

docs/source/dev_guide.rst

@@ -14,7 +14,7 @@ Building the docs
 
 First install dependencies for the documentation:
 
-.. code-block:: sh
+.. code-block:: console
 
   $ make bootstrap_dev
   $ make docs
@@ -28,26 +28,26 @@ Run tests using `pytest`_.
 
 First activate the ``pelican`` Conda environment and install ``pytest``.
 
-.. code-block:: sh
+.. code-block:: console
 
    $ source activate pelican
    $ conda install pytest flake8  # if not already installed
 
 Run quick tests (skip slow and online):
 
-.. code-block:: sh
+.. code-block:: console
 
     $ pytest -m 'not slow and not online'"
 
 Run all tests:
 
-.. code-block:: sh
+.. code-block:: console
 
     $ pytest
 
 Check pep8:
 
-.. code-block:: sh
+.. code-block:: console
 
     $ flake8
 
@@ -56,12 +56,28 @@ Run tests the lazy way
 
 Do the same as above using the ``Makefile``.
 
-.. code-block:: sh
+.. code-block:: console
 
     $ make test
     $ make testall
     $ make pep8
 
+Prepare a release
+-----------------
+
+Update the Conda specification file to build identical environments_ on a specific OS.
+
+.. note:: You should run this on your target OS, in our case Linux.
+
+.. code-block:: console
+
+  $ make clean
+  $ make install
+  $ make spec
+
+.. _`environments`: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#building-identical-conda-environments
+
+
 Bump a new version
 ------------------
 

environment.yml

@@ -1,21 +1,22 @@
 name: pelican
 channels:
-- conda-forge
-- birdhouse
+- defaults
 dependencies:
-- pywps=4.2
+- pip
 - jinja2
 - click
 - psutil
-# owslib esgfwps
-- owslib>=0.17
-- owslib-esgfwps=0.1.0
+# OWSLib
+- lxml
 # opendap support
 - netcdf4
 # dask
 - xarray
 # plot
 - cartopy
 - matplotlib
-#- pip:
-#  - -e git+https://github.com/bird-house/OWSLib-esgfwps.git@master#egg=owslib-esgfwps
+# tests
+- pytest
+- pip:
+  - -e git+https://github.com/geopython/pywps@master#egg=pywps
+  - -e git+https://github.com/bird-house/OWSLib-esgfwps.git@master#egg=owslib-esgfwps

pelican/cli.py

@@ -12,7 +12,7 @@
 from pywps import configuration
 
 from . import wsgi
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 PID_FILE = os.path.abspath(os.path.join(os.path.curdir, "pywps.pid"))
 

pelican/processes/wps_esgf_subset.py

@@ -143,20 +143,20 @@ def _handler(self, request, response):
         # subsetting
         import xarray as xr
         response.update_status('PyWPS Process started.', 0)
-
-        variable = Variable.from_json(json.loads(request.inputs['variable'][0].data))
-        domain = Domain.from_json(json.loads(request.inputs['domain'][0].data))
+        # get variable and domain from json input
+        variable = Variable.from_json(json.loads(request.inputs['variable'][0].data)[0])
+        domain = Domain.from_json(json.loads(request.inputs['domain'][0].data)[0])
 
         # TODO: Use chunks for parallel processing with dask.distributed
         output_file = self.workdir + '/out.nc'
         with xr.open_dataset(variable.uri) as ds:
             da = ds[variable.var_name]
             sl = {}
             for dim in domain.dimensions:
-                sl = {dim['name']: slice(dim['start'], dim['end'], dim['step'])}
-                if dim['crs'] == 'values':
+                sl = {dim.name: slice(dim.start, dim.end, dim.step)}
+                if dim.crs == 'values':
                     da = da.sel(**sl)
-                elif dim['crs'] == 'indices':
+                elif dim.crs == 'indices':
                     da = da.isel(**sl)
             if 0 in da.shape:
                 raise ValueError("Subsetting operation yields no values for `{}` dimension.".

requirements.txt

@@ -2,3 +2,8 @@ pywps>=4.2.0
 jinja2
 click
 psutil
+OWSLib
+netcdf4
+xarray
+cartopy
+matplotlib

tests/common.py

@@ -19,7 +19,7 @@ class WpsTestClient(WpsClient):
     def get(self, *args, **kwargs):
         query = "?"
         for key, value in kwargs.items():
-            query += u"{0}={1}&".format(key, value)
+            query += "{0}={1}&".format(key, value)
         return super(WpsTestClient, self).get(query)
 
 

tests/test_wps_esgf_subset.py

@@ -1,5 +1,3 @@
-import pytest
-
 from pywps import Service
 from pywps.tests import assert_response_success
 
@@ -16,7 +14,6 @@
 ])
 
 
-@pytest.mark.online
 def test_wps_esgf_subset():
     client = client_for(Service(processes=[PelicanSubset()]))
     datainputs = "variable={variable};" \