Added director to return files or URLs (#77)

* inventory classes

* Added new `rook/director` sub-package

* Filling in missing functions

* start alignment tests

* Fixed path to inventory directory

* added alignment tests

* add fix for unrecognised formats

* Working on: test_inv_cache.py

* Fixed test and download

* Created test_inventory.py

* director tests

* wrapped get inventory in try and except

* added cmip6 test

* formatting and some fixes for original files

* fix import and flake8

* fix start and end matches

* moved inventory_url and download_url and implemented config

* updated dependencies

* update to use file_uris

* update history

* handle urls

* formatting

Co-authored-by: Eleanor Smith <esmith88@sci5.jasmin.ac.uk>
Co-authored-by: agstephens <ag.stephens@stfc.ac.uk>

CHANGES.rst

@@ -3,7 +3,9 @@ Changes
 
 Unreleased
 ==========
-* ``apply_fixes`` option added for WPS processes and the ``Operator`` class.
+* ``apply_fixes`` and ``original_files`` option added for WPS processes and the ``Operator`` class.
+* ``director`` module added. This makes decisions on what is returned - NetCDF files or original file URLs.
+* ``python-dateutil>=2.8.1`` added as a new dependency.
 
 0.2.0 (2020-11-19)
 ==================

docs/source/conf.py

@@ -51,22 +51,55 @@
 # To avoid having to install these and burst memory limit on ReadTheDocs.
 # List of all tested working mock imports from all birds so new birds can
 # inherit without having to test which work which do not.
-autodoc_mock_imports = ["numpy", "xarray", "fiona", "rasterio", "shapely",
-                        "osgeo", "geopandas", "pandas", "statsmodels",
-                        "affine", "rasterstats", "spotpy", "matplotlib",
-                        "scipy", "unidecode", "gdal", "sentry_sdk", "dask",
-                        "numba", "parse", "siphon", "sklearn", "cftime",
-                        "netCDF4", "bottleneck", "ocgis", "geotiff", "geos",
-                        "hdf4", "hdf5", "zlib", "pyproj", "proj", "cartopy",
-                        "scikit-learn", "cairo", "networkx", "roocs_utils", "daops"]
+autodoc_mock_imports = [
+    "numpy",
+    "xarray",
+    "fiona",
+    "rasterio",
+    "shapely",
+    "osgeo",
+    "geopandas",
+    "pandas",
+    "statsmodels",
+    "affine",
+    "rasterstats",
+    "spotpy",
+    "matplotlib",
+    "scipy",
+    "unidecode",
+    "gdal",
+    "sentry_sdk",
+    "dask",
+    "numba",
+    "parse",
+    "siphon",
+    "sklearn",
+    "cftime",
+    "netCDF4",
+    "bottleneck",
+    "ocgis",
+    "geotiff",
+    "geos",
+    "hdf4",
+    "hdf5",
+    "zlib",
+    "pyproj",
+    "proj",
+    "cartopy",
+    "scikit-learn",
+    "cairo",
+    "networkx",
+    "roocs_utils",
+    "daops",
+]
 
 # Monkeypatch constant because the following are mock imports.
 # Only works if numpy is actually installed and at the same time being mocked.
-#import numpy
-#numpy.pi = 3.1416
+# import numpy
+# numpy.pi = 3.1416
 
 # We are using mock imports in readthedocs, so probably safer to not run the notebooks
-nbsphinx_execute = 'never'
+nbsphinx_execute = "never"
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
@@ -113,7 +146,7 @@
 todo_include_todos = False
 
 # Suppress "WARNING: unknown mimetype for ..." when building EPUB.
-suppress_warnings = ['epub.unknown_project_files']
+suppress_warnings = ["epub.unknown_project_files"]
 
 # Avoid "configuration.rst:4:duplicate label configuration, other instance in configuration.rst"
 autosectionlabel_prefix_document = True

environment.yml

@@ -27,3 +27,4 @@ dependencies:
 - graphviz
 # tests
 - pytest
+- python-dateutil>=2.8.1

requirements.txt

@@ -6,6 +6,7 @@ networkx
 xarray>=0.15
 dask[complete]
 netcdf4
+python-dateutil>=2.8.1
 # daops>=0.3.0
 # clisops>=0.4.0
 # roocs-utils>=0.1.5

rook/__init__.py

@@ -4,4 +4,9 @@
 
 from .__version__ import __author__, __email__, __version__  # noqa: F401
 
+from roocs_utils.config import get_config
+import rook
+
+CONFIG = get_config(rook)
+
 from .wsgi import application  # noqa: F401

rook/__version__.py

@@ -5,5 +5,5 @@
 # See: https://packaging.python.org/guides/single-sourcing-package-version
 
 __author__ = """Carsten Ehbrecht"""
-__email__ = 'ehbrecht@dkrz.de'
-__version__ = '0.2.0'
+__email__ = "ehbrecht@dkrz.de"
+__version__ = "0.2.0"

rook/cli.py

@@ -16,32 +16,29 @@
 
 PID_FILE = os.path.abspath(os.path.join(os.path.curdir, "pywps.pid"))
 
-CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 
-template_env = Environment(
-    loader=PackageLoader('rook', 'templates'),
-    autoescape=True
-)
+template_env = Environment(loader=PackageLoader("rook", "templates"), autoescape=True)
 
 
 def write_user_config(**kwargs):
-    config_templ = template_env.get_template('pywps.cfg')
+    config_templ = template_env.get_template("pywps.cfg")
     rendered_config = config_templ.render(**kwargs)
     config_file = os.path.abspath(os.path.join(os.path.curdir, ".custom.cfg"))
-    with open(config_file, 'w') as fp:
+    with open(config_file, "w") as fp:
         fp.write(rendered_config)
     return config_file
 
 
 def get_host():
-    url = configuration.get_config_value('server', 'url')
-    url = url or 'http://localhost:5000/wps'
+    url = configuration.get_config_value("server", "url")
+    url = url or "http://localhost:5000/wps"
 
     click.echo("starting WPS service on {}".format(url))
 
     parsed_url = urlparse(url)
-    if ':' in parsed_url.netloc:
-        host, port = parsed_url.netloc.split(':')
+    if ":" in parsed_url.netloc:
+        host, port = parsed_url.netloc.split(":")
         port = int(port)
     else:
         host = parsed_url.netloc
@@ -52,19 +49,21 @@ def get_host():
 def run_process_action(action=None):
     """Run an action with psutil on current process
     and return a status message."""
-    action = action or 'status'
+    action = action or "status"
     try:
-        with open(PID_FILE, 'r') as fp:
+        with open(PID_FILE, "r") as fp:
             pid = int(fp.read())
             p = psutil.Process(pid)
-            if action == 'stop':
+            if action == "stop":
                 p.terminate()
                 msg = "pid={}, status=terminated".format(p.pid)
             else:
                 from psutil import _pprint_secs
+
                 msg = "pid={}, status={}, created={}".format(
-                    p.pid, p.status(), _pprint_secs(p.create_time()))
-        if action == 'stop':
+                    p.pid, p.status(), _pprint_secs(p.create_time())
+                )
+        if action == "stop":
             os.remove(PID_FILE)
     except IOError:
         msg = 'No PID file found. Service not running? Try "netstat -nlp | grep :5000".'
@@ -75,13 +74,12 @@ def run_process_action(action=None):
 
 def _run(application, bind_host=None, daemon=False):
     from werkzeug.serving import run_simple
+
     # call this *after* app is initialized ... needs pywps config.
     host, port = get_host()
     bind_host = bind_host or host
     # need to serve the wps outputs
-    static_files = {
-        '/outputs': configuration.get_config_value('server', 'outputpath')
-    }
+    static_files = {"/outputs": configuration.get_config_value("server", "outputpath")}
     run_simple(
         hostname=bind_host,
         port=port,
@@ -91,7 +89,8 @@ def _run(application, bind_host=None, daemon=False):
         threaded=True,
         # processes=2,
         use_evalex=not daemon,
-        static_files=static_files)
+        static_files=static_files,
+    )
 
 
 @click.group(context_settings=CONTEXT_SETTINGS)
@@ -109,48 +108,102 @@ def cli():
 @cli.command()
 def status():
     """Show status of PyWPS service"""
-    run_process_action(action='status')
+    run_process_action(action="status")
 
 
 @cli.command()
 def stop():
     """Stop PyWPS service"""
-    run_process_action(action='stop')
+    run_process_action(action="stop")
 
 
 @cli.command()
-@click.option('--config', '-c', metavar='PATH', help='path to pywps configuration file.')
-@click.option('--bind-host', '-b', metavar='IP-ADDRESS', default='127.0.0.1',
-              help='IP address used to bind service.')
-@click.option('--daemon', '-d', is_flag=True, help='run in daemon mode.')
-@click.option('--hostname', metavar='HOSTNAME', default='localhost', help='hostname in PyWPS configuration.')
-@click.option('--port', metavar='PORT', default='5000', help='port in PyWPS configuration.')
-@click.option('--maxsingleinputsize', default='200mb', help='maxsingleinputsize in PyWPS configuration.')
-@click.option('--maxprocesses', metavar='INT', default='10', help='maxprocesses in PyWPS configuration.')
-@click.option('--parallelprocesses', metavar='INT', default='2', help='parallelprocesses in PyWPS configuration.')
-@click.option('--log-level', metavar='LEVEL', default='INFO', help='log level in PyWPS configuration.')
-@click.option('--log-file', metavar='PATH', default='pywps.log', help='log file in PyWPS configuration.')
-@click.option('--database', default='sqlite:///pywps-logs.sqlite', help='database in PyWPS configuration')
-def start(config, bind_host, daemon, hostname, port,
-          maxsingleinputsize, maxprocesses, parallelprocesses,
-          log_level, log_file, database):
+@click.option(
+    "--config", "-c", metavar="PATH", help="path to pywps configuration file."
+)
+@click.option(
+    "--bind-host",
+    "-b",
+    metavar="IP-ADDRESS",
+    default="127.0.0.1",
+    help="IP address used to bind service.",
+)
+@click.option("--daemon", "-d", is_flag=True, help="run in daemon mode.")
+@click.option(
+    "--hostname",
+    metavar="HOSTNAME",
+    default="localhost",
+    help="hostname in PyWPS configuration.",
+)
+@click.option(
+    "--port", metavar="PORT", default="5000", help="port in PyWPS configuration."
+)
+@click.option(
+    "--maxsingleinputsize",
+    default="200mb",
+    help="maxsingleinputsize in PyWPS configuration.",
+)
+@click.option(
+    "--maxprocesses",
+    metavar="INT",
+    default="10",
+    help="maxprocesses in PyWPS configuration.",
+)
+@click.option(
+    "--parallelprocesses",
+    metavar="INT",
+    default="2",
+    help="parallelprocesses in PyWPS configuration.",
+)
+@click.option(
+    "--log-level",
+    metavar="LEVEL",
+    default="INFO",
+    help="log level in PyWPS configuration.",
+)
+@click.option(
+    "--log-file",
+    metavar="PATH",
+    default="pywps.log",
+    help="log file in PyWPS configuration.",
+)
+@click.option(
+    "--database",
+    default="sqlite:///pywps-logs.sqlite",
+    help="database in PyWPS configuration",
+)
+def start(
+    config,
+    bind_host,
+    daemon,
+    hostname,
+    port,
+    maxsingleinputsize,
+    maxprocesses,
+    parallelprocesses,
+    log_level,
+    log_file,
+    database,
+):
     """Start PyWPS service.
     This service is by default available at http://localhost:5000/wps
     """
     if os.path.exists(PID_FILE):
         click.echo('PID file exists: "{}". Service still running?'.format(PID_FILE))
         os._exit(0)
     cfgfiles = []
-    cfgfiles.append(write_user_config(
-        wps_hostname=hostname,
-        wps_port=port,
-        wps_maxsingleinputsize=maxsingleinputsize,
-        wps_maxprocesses=maxprocesses,
-        wps_parallelprocesses=parallelprocesses,
-        wps_log_level=log_level,
-        wps_log_file=log_file,
-        wps_database=database,
-    ))
+    cfgfiles.append(
+        write_user_config(
+            wps_hostname=hostname,
+            wps_port=port,
+            wps_maxsingleinputsize=maxsingleinputsize,
+            wps_maxprocesses=maxprocesses,
+            wps_parallelprocesses=parallelprocesses,
+            wps_log_level=log_level,
+            wps_log_file=log_file,
+            wps_database=database,
+        )
+    )
     if config:
         cfgfiles.append(config)
     app = wsgi.create_app(cfgfiles)
@@ -164,8 +217,8 @@ def start(config, bind_host, daemon, hostname, port,
         try:
             pid = os.fork()
             if pid:
-                click.echo('forked process id: {}'.format(pid))
-                with open(PID_FILE, 'w') as fp:
+                click.echo("forked process id: {}".format(pid))
+                with open(PID_FILE, "w") as fp:
                     fp.write("{}".format(pid))
         except OSError as e:
             raise Exception("%s [%d]" % (e.strerror, e.errno))

rook/director/__init__.py

@@ -0,0 +1 @@
+from .director import Director