Update pywps (#203)

* update pywps=4.5 and other requirements * linted * update requirements pandas, dask, xarray
roocs · Feb 3, 2022 · 56c8b1a · 56c8b1a
1 parent fa2abfb
commit 56c8b1a
Show file tree

Hide file tree

Showing 13 changed files with 76 additions and 59 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -3,7 +3,7 @@ default_language_version:
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.3.0
+    rev: v3.4.0
     hooks:
     -   id: trailing-whitespace
         language_version: python3
@@ -16,13 +16,13 @@ repos:
     -   id: debug-statements
         language_version: python3
 -   repo: https://github.com/ambv/black
-    rev: 20.8b1
+    rev: 22.1.0
     hooks:
     -   id: black
         language_version: python3
         args: ["--target-version", "py36"]
 -   repo: https://github.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.1
     hooks:
     -   id: flake8
         language_version: python3
@@ -44,7 +44,7 @@ repos:
 #    -   id: pydocstyle
 #        args: ["--conventions=numpy"]
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.7.3
+    rev: v2.14.0
     hooks:
     -   id: pyupgrade
         language_version: python3

diff --git a/CHANGES.rst b/CHANGES.rst
@@ -6,7 +6,7 @@ Changes
 
 * Added "subset-by-point" (#190).
 * Updated to clisops 0.7.0.
-* Updated to daops 0.7.0. 
+* Updated to daops 0.7.0.
 * Updated dashboard (#195).
 * Updated provenance namespace (#188).
 

diff --git a/environment.yml b/environment.yml
@@ -5,19 +5,19 @@ channels:
 dependencies:
 - pip
 - python=3.7 # >=3.6,<3.8
-- pywps>=4.4.5,<4.5
+- pywps>=4.5.1,<4.6
 - jinja2
 - click
 - psutil
 - requests
 # daops
 - cftime>=1.2.1
-- xarray>=0.16
-- dask>=2.26
+- xarray>=0.20
+- dask>=2021.12
 - netcdf4>=1.4
 - bottleneck>=1.3.1,<1.4
 #- daops>=0.7.0,<0.8
-- clisops>=0.7.0,<0.9
+- clisops>=0.8.0,<0.9
 - roocs-utils>=0.5.0,<0.6
 # workflow
 - networkx
@@ -27,9 +27,7 @@ dependencies:
 - graphviz
 # catalog
 - intake
-- fsspec<0.9
-- pandas<1.3.0
-- dask<2021.7.0
+- pandas>=1.3.5
 - aiohttp
 # dashboard
 - bokeh=2.4.1

diff --git a/requirements.txt b/requirements.txt
@@ -1,13 +1,13 @@
-pywps>=4.4.5,<4.5
+pywps>=4.5.1,<4.6
 jinja2
 click
 psutil
 # daops
 # daops>=0.7.0,<0.8
 daops @ git+https://github.com/roocs/daops@master#egg=daops
-clisops>=0.7.0,<0.9
+clisops>=0.8.0,<0.9
 roocs-utils>=0.5.0,<0.6
-xarray>=0.16
+xarray>=0.20
 dask[complete]
 netcdf4
 # workflow
@@ -17,9 +17,7 @@ prov>=2.0.0
 pydot
 # catalog
 intake
-fsspec<0.9
-pandas<1.3.0
-dask<2021.7.0
+pandas>=1.3.5
 aiohttp
 # dashboard
 bokeh>=2.4.1

diff --git a/rook/dashboard/plots/transfer.py b/rook/dashboard/plots/transfer.py
@@ -12,7 +12,7 @@ def data(self):
         pdf = pd.DataFrame()
         pdf["time"] = self.df.datetime
         pdf["size"] = self.df["size"]
-        pdf["size"] = pdf["size"].apply(lambda x: x / 1024 ** 3)
+        pdf["size"] = pdf["size"].apply(lambda x: x / 1024**3)
         pdf = pdf.groupby(pd.Grouper(key="time", freq="1D")).sum()
         pdf = pdf.sort_values(by=["time"], ascending=False)
         return pdf

diff --git a/rook/dashboard/tables/overview.py b/rook/dashboard/tables/overview.py
@@ -30,7 +30,7 @@ def data(self):
         ).request_type.count()
         # data transfer
         tdf = self.df_downloads.groupby(pd.Grouper(key="datetime", freq="1D")).sum()
-        tdf["size"] = tdf["size"].apply(lambda x: x / 1024 ** 3)
+        tdf["size"] = tdf["size"].apply(lambda x: x / 1024**3)
         transfer = tdf["size"]
         data_ = dict(
             property=[

diff --git a/rook/director/alignment.py b/rook/director/alignment.py
@@ -17,7 +17,11 @@ def __init__(self, input_files, inputs):
 
     def _deduce_alignment(self, inputs):
         # At present, we reject alignment if any "time_components", "area" or "level" subset is requested
-        if inputs.get("time_components", None) or inputs.get("area", None) or inputs.get("level", None):
+        if (
+            inputs.get("time_components", None)
+            or inputs.get("area", None)
+            or inputs.get("level", None)
+        ):
             return
 
         time = inputs.get("time", None)

diff --git a/rook/director/director.py b/rook/director/director.py
@@ -48,7 +48,11 @@ def __init__(self, coll, inputs):
         self._check_apply_fixes()
 
     def _check_apply_fixes(self):
-        if self.inputs.get("apply_fixes") and not self.use_original_files and self.requires_fixes():
+        if (
+            self.inputs.get("apply_fixes")
+            and not self.use_original_files
+            and self.requires_fixes()
+        ):
             self.inputs["apply_fixes"] = True
         else:
             self.inputs["apply_fixes"] = False
@@ -76,7 +80,7 @@ def _resolve(self):
         self.search_result = self.catalog.search(
             collection=self.coll,
             time=self.inputs.get("time"),
-            time_components=self.inputs.get("time_components")
+            time_components=self.inputs.get("time_components"),
         )
         # Raise exception if any of the dataset ids is not in the inventory
         if len(self.search_result) != len(self.coll):

diff --git a/rook/processes/wps_subset.py b/rook/processes/wps_subset.py
@@ -143,17 +143,17 @@ def _handler(self, request, response):
         inputs = {
             "collection": collection,
             "output_dir": self.workdir,
-            "apply_fixes": parse_wps_input(
-                request.inputs, "apply_fixes", default=True
-            ),
+            "apply_fixes": parse_wps_input(request.inputs, "apply_fixes", default=True),
             "pre_checked": parse_wps_input(
                 request.inputs, "pre_checked", default=False
             ),
             "original_files": parse_wps_input(
                 request.inputs, "original_files", default=False
             ),
             "time": parse_wps_input(request.inputs, "time", default=None),
-            "time_components": parse_wps_input(request.inputs, "time_components", default=None),
+            "time_components": parse_wps_input(
+                request.inputs, "time_components", default=None
+            ),
             "level": parse_wps_input(request.inputs, "level", default=None),
             "area": parse_wps_input(request.inputs, "area", default=None),
         }

diff --git a/rook/processes/wps_usage.py b/rook/processes/wps_usage.py
@@ -13,7 +13,7 @@
 EMPTY_CSV = """\
 remote_host_ip,ip_number,datetime,timezone,request_type,request,protocol,status_code,size,referer,user_agent
 127.0.0.1,1000000000,2021-01-01 12:00:00,+0200,GET,dummy.nc,HTTP/1.1,200,1000000,-,python
-""" # noqa
+"""  # noqa
 
 
 class Usage(Process):

diff --git a/rook/provenance.py b/rook/provenance.py
@@ -104,7 +104,14 @@ def stop(self):
 
     def add_operator(self, operator, parameters, collection, output):
         attributes = {}
-        for param in ["time", "time_components", "area", "level", "dims", "apply_fixes"]:
+        for param in [
+            "time",
+            "time_components",
+            "area",
+            "level",
+            "dims",
+            "apply_fixes",
+        ]:
             if param in parameters:
                 attributes[ROOCS[param]] = parameters[param]
         op = self._execution_activity(

diff --git a/tests/test_director/test_alignment.py b/tests/test_director/test_alignment.py
@@ -9,7 +9,7 @@
 
 
 class TestYearMonth:
-    """ Tests with year and month only, not day"""
+    """Tests with year and month only, not day"""
 
     test_path = (
         f"{MINI_ESGF_MASTER_DIR}/test_data/badc/cmip5/data/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon"
@@ -59,7 +59,7 @@ def test_time_subset_match(self, get_files):
 
 
 class TestYearMonthDay1200:
-    """ Tests with year month and day for dataset with a 12:00:00 time step """
+    """Tests with year month and day for dataset with a 12:00:00 time step"""
 
     test_path = (
         f"{MINI_ESGF_MASTER_DIR}/test_data/gws/nopw/j04/cp4cds1_vol1/data/c3s-cmip5/output1/ICHEC/"
@@ -174,7 +174,7 @@ def test_time_subset_no_match_including_hour(self, get_files):
 
 
 class TestYearMonthDay0000:
-    """ Tests with year month and day for dataset with a 00:00:00 time step """
+    """Tests with year month and day for dataset with a 00:00:00 time step"""
 
     test_path = (
         f"{MINI_ESGF_MASTER_DIR}/test_data/badc/cmip5/data/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon"

diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -83,37 +83,43 @@ def test_resolve_to_file_paths_urls(load_test_data):
 
 def test_parse_wps_input():
     obj = {
-        'operation': 'execute',
-        'version': '1.0.0',
-        'language': 'eng',
-        'identifier': 'subset',
-        'identifiers': 'subset',  # TODO: why identifierS?
-        'store_execute': True,
-        'status': True,
-        'lineage': True,
-        'inputs': {
-            'time': [{
-                'identifier': 'time',
-                'type': 'literal',
-                'data_type': 'string',
-                'allowed_values': [{'type': 'anyvalue'}],
-                'data': '1970/1980',
-            }],
-            'time_components': [{
-                'identifier': 'time',
-                'type': 'literal',
-                'data_type': 'string',
-                'allowed_values': [{'type': 'anyvalue'}],
-                'data': 'year:1970,1980|month=01,02,03',
-            }],
+        "operation": "execute",
+        "version": "1.0.0",
+        "language": "eng",
+        "identifier": "subset",
+        "identifiers": "subset",  # TODO: why identifierS?
+        "store_execute": True,
+        "status": True,
+        "lineage": True,
+        "inputs": {
+            "time": [
+                {
+                    "identifier": "time",
+                    "type": "literal",
+                    "data_type": "string",
+                    "allowed_values": [{"type": "anyvalue"}],
+                    "data": "1970/1980",
+                }
+            ],
+            "time_components": [
+                {
+                    "identifier": "time",
+                    "type": "literal",
+                    "data_type": "string",
+                    "allowed_values": [{"type": "anyvalue"}],
+                    "data": "year:1970,1980|month=01,02,03",
+                }
+            ],
         },
-        'outputs': {},
-        'raw': False
+        "outputs": {},
+        "raw": False,
     }
 
     request = WPSRequest()
     request.json = obj
 
     assert parse_wps_input(request.inputs, "time", default=None) == "1970/1980"
-    assert parse_wps_input(request.inputs, "time_components", default=None) == \
-        "year:1970,1980|month=01,02,03"
+    assert (
+        parse_wps_input(request.inputs, "time_components", default=None)
+        == "year:1970,1980|month=01,02,03"
+    )