Update dashboard (#195)

* update bokeh * use empty csv * update downloads usage * show exception when no records found * added ipsl
roocs · Nov 5, 2021 · 59a78a2 · 59a78a2
1 parent 8604804
commit 59a78a2
Show file tree

Hide file tree

Showing 7 changed files with 28 additions and 11 deletions.
diff --git a/environment.yml b/environment.yml
@@ -32,6 +32,6 @@ dependencies:
 - dask<2021.7.0
 - aiohttp
 # dashboard
-- bokeh
+- bokeh=2.4.1
 # tests
 - pytest
diff --git a/requirements.txt b/requirements.txt
@@ -21,5 +21,5 @@ pandas<1.3.0
 dask<2021.7.0
 aiohttp
 # dashboard
-bokeh
+bokeh>=2.4.1
 humanize
diff --git a/rook/dashboard/templates/base.html b/rook/dashboard/templates/base.html
@@ -9,8 +9,8 @@
     <!-- Bootstrap CSS -->
     <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-+0n0xVW2eSR5OomGNYDnhzAbDsOXxcvSN1TPprVMTNDbiYZCxYbOOl7+AMvyTG2x" crossorigin="anonymous">
     <!-- Bokeh CSS -->
-    <link href="http://cdn.pydata.org/bokeh/release/bokeh-2.3.2.min.css" rel="stylesheet">
-    <link href="http://cdn.pydata.org/bokeh/release/bokeh-widgets-2.3.2.min.css" rel="stylesheet">
+    <link href="http://cdn.pydata.org/bokeh/release/bokeh-2.4.1.min.css" rel="stylesheet">
+    <link href="http://cdn.pydata.org/bokeh/release/bokeh-widgets-2.4.1.min.css" rel="stylesheet">
 
     <title>Rook - Dashboard</title>
 </head>
@@ -20,9 +20,9 @@
     {% endblock %}
 
     <!-- Bokeh -->
-    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-2.3.2.min.js"></script>
-    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.3.2.min.js"></script>
-    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.3.2.min.js"></script>
+    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-2.4.1.min.js"></script>
+    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.1.min.js"></script>
+    <script src="https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.1.min.js"></script>
 
     <!-- Bootstrap Bundle with Popper -->
     <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.1/dist/js/bootstrap.bundle.min.js" integrity="sha384-gtEjrD/SeCtmISkJkNUaaKMoLD0//ElJ19smozuHV6z3Iehds+3Ulb9Bn9Plx0x4" crossorigin="anonymous"></script>

diff --git a/rook/processes/wps_dashboard.py b/rook/processes/wps_dashboard.py
@@ -23,7 +23,7 @@ def __init__(self):
                 min_occurs=1,
                 max_occurs=1,
                 default="local",
-                allowed_values=["local", "ceda", "dkrz", "all"],
+                allowed_values=["local", "ceda", "ipsl", "dkrz", "all"],
             ),
             LiteralInput(
                 "time",

diff --git a/rook/processes/wps_usage.py b/rook/processes/wps_usage.py
@@ -10,6 +10,11 @@
 
 LOGGER = logging.getLogger()
 
+EMPTY_CSV = """\
+remote_host_ip,ip_number,datetime,timezone,request_type,request,protocol,status_code,size,referer,user_agent
+127.0.0.1,1000000000,2021-01-01 12:00:00,+0200,GET,dummy.nc,HTTP/1.1,200,1000000,-,python
+""" # noqa
+
 
 class Usage(Process):
     def __init__(self):
@@ -64,17 +69,26 @@ def _handler(self, request, response):
         else:
             time = None
             time_start = time_end = None
+        # usage
         try:
             usage = WPSUsage()
             response.outputs["wpsusage"].file = usage.collect(
                 time_start=time_start, time_end=time_end, outdir=self.workdir
             )
             response.update_status("WPSUsage completed.", 50)
+        except Exception as e:
+            raise ProcessError(f"{e}")
+        # downloads
+        try:
             usage = Downloads()
-            response.outputs["downloads"].file = usage.collect(
+            downloads_csv = usage.collect(
                 time_start=time_start, time_end=time_end, outdir=self.workdir
             )
+            response.outputs["downloads"].file = downloads_csv
+        except Exception:
+            LOGGER.exception("downloads collection failed")
+            response.outputs["downloads"].data = EMPTY_CSV
+        finally:
             response.update_status("Downloads usage completed.", 90)
-        except Exception as e:
-            raise ProcessError(f"{e}")
+
         return response
diff --git a/rook/usage/combine.py b/rook/usage/combine.py
@@ -11,6 +11,7 @@
 
 URLS = {
     "local": config.get_config_value("server", "url"),
+    "ipsl": "http://copernicus-wps.ipsl.upmc.fr/wps",
     "dkrz": "http://rook3.cloud.dkrz.de/wps",
     "ceda": "http://rook-wps1.ceda.ac.uk/wps",
 }

diff --git a/rook/usage/downloads.py b/rook/usage/downloads.py
@@ -148,6 +148,8 @@ def parse(self, log_files, time_start=None, time_end=None, outdir=None):
                 if not record["request"].startswith(self.output_path):
                     continue
                 records.append(record)
+        if not records:
+            raise NotFoundError("could not find any records")
         df = pd.DataFrame(records)
         df = df.loc[
             df["request"].str.contains(rf"{self.output_path}/.*/.*\.nc", regex=True)