added initial dashboard (#182)

* added initial dashboard

environment.yml

@@ -5,7 +5,7 @@ channels:
 dependencies:
 - pip
 - python=3.7 # >=3.6,<3.8
-- pywps>=4.4.2,<4.5
+- pywps>=4.4.3,<4.5
 - jinja2
 - click
 - psutil
@@ -29,5 +29,7 @@ dependencies:
 - fsspec<0.9
 - pandas
 - aiohttp
+# dashboard
+- bokeh
 # tests
 - pytest

notebooks/dashboard.ipynb

@@ -0,0 +1,79 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "59109053-8451-421a-930c-534bfe75e2f8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from rook.dashboard import Dashboard"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7384ef24-299e-42c9-8a7a-fc990a02ce14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dash = Dashboard(\"/tmp\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1cd42832-c7fe-405a-b927-663d996fddff",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dash.load(\n",
+    "        \"http://rook3.cloud.dkrz.de:80/outputs/rook/9963ecd6-cde8-11eb-bb2b-fa163e466023/wps_requests.csv\",\n",
+    "        filter=\"orchestrate\",\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9fc29c58-067b-409b-b047-1c7ae1e5d3c8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dash.write()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3175d999-2052-4741-b19d-3ec371e0713c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import IPython\n",
+    "IPython.display.HTML(\"/tmp/dashboard.html\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/jobs.ipynb

@@ -0,0 +1,134 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "271ceeca-9ab5-4d02-9f4c-21ade5505401",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "url = \"http://rook3.cloud.dkrz.de:80/outputs/rook/9963ecd6-cde8-11eb-bb2b-fa163e466023/wps_requests.csv\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "085ee7da-e95d-4551-a4ab-431f66cd6a0f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import pandas_bokeh"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3a3f8791-56b2-464b-96ff-16d8d307228c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_csv(url, parse_dates=[4, 5])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e4fd8c38-1384-43e0-8da8-c9bb5a684629",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gdf = df.groupby(pd.Grouper(key=\"time_start\", freq=\"1D\")).count()\n",
+    "pdf = pd.DataFrame()\n",
+    "pdf[\"Date\"] = pd.to_datetime(gdf.index)\n",
+    "pdf[\"Jobs\"] = gdf.uuid.values\n",
+    "pdf.sort_values(by='Date', ascending=True)\n",
+    "plot = pdf.plot_bokeh(\n",
+    "        title=\"Jobs per day\",\n",
+    "        kind=\"line\",\n",
+    "        x=\"Date\",\n",
+    "        y=\"Jobs\",\n",
+    "        show_figure=False,\n",
+    "        return_html=True,\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a772e3d-d7ea-4952-a9c0-c01423935a94",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import IPython\n",
+    "IPython.display.HTML(plot)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7fb611a4-e1e4-490f-99bd-90ba4a7b7b4c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from bokeh.plotting import figure\n",
+    "from bokeh.models import ColumnDataSource"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "181c8f37-fadd-4c8e-a817-303e29ca4991",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plot = figure(title=\"Jobs per day\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0c986a68-a89b-4b2c-a538-5fad46bf6c28",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plot.varea(\n",
+    "    x=\"Date\", y1=\"Jobs\", y2=0,\n",
+    "    source=ColumnDataSource(pdf)\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "80a3722d-0b26-4991-8ef0-1c22f469e61b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from bokeh.io import show\n",
+    "show(plot)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

requirements.txt

@@ -1,4 +1,4 @@
-pywps>=4.4.2,<4.5
+pywps>=4.4.3,<4.5
 jinja2
 click
 psutil
@@ -22,3 +22,6 @@ intake
 fsspec<0.9
 pandas
 aiohttp
+# dashboard
+bokeh
+humanize

rook/dashboard/__init__.py

@@ -0,0 +1 @@
+from .base import Dashboard

rook/dashboard/base.py

@@ -0,0 +1,80 @@
+import os
+import pandas as pd
+import humanize
+
+from .plots import (
+    plot_duration,
+    plot_jobs_per_day,
+    plot_jobs_over_week_days,
+    plot_jobs_over_day_time,
+    plot_concurrent_jobs_per_day,
+    plot_errors_per_day,
+    table_of_errors,
+)
+
+from jinja2 import Environment, PackageLoader, select_autoescape
+
+env = Environment(
+    loader=PackageLoader("rook.dashboard"), autoescape=select_autoescape()
+)
+
+
+class Dashboard:
+    """
+    See dashboard examples:
+    * https://engineertodeveloper.com/big-bay-dam-monitoring-dashboard-part-5-bokeh-plots/
+    * https://towardsdatascience.com/https-medium-com-radecicdario-next-level-data-visualization-dashboard-app-with-bokeh-flask-c588c9398f98
+    """  # noqa
+
+    def __init__(self, output_dir=None):
+        self.df = None
+        self.output_dir = output_dir or os.path.curdir
+
+    def load(self, url, filter=None):
+        # read csv, parse start/end time
+        df = pd.read_csv(url, parse_dates=[4, 5])
+        # day of week
+        df["dayofweek"] = df["time_start"].dt.dayofweek
+        # hour
+        df["hour"] = df["time_start"].dt.hour
+        # finished jobs
+        df = df[df["status"].isin([4, 5])]
+        # filter
+        if filter:
+            df = df.loc[df["operation"] == "execute"].loc[df["identifier"] == filter]
+        # done
+        self.df = df
+
+    def write(self):
+        out = os.path.join(self.output_dir, "dashboard.html")
+        with open(out, "w") as f:
+            f.write(self.render())
+        return out
+
+    def render(self):
+        template = env.get_template("dashboard.html")
+        script_1, plot_1 = plot_jobs_per_day(self.df)
+        script_2, plot_2 = plot_jobs_over_week_days(self.df)
+        script_3, plot_3 = plot_jobs_over_day_time(self.df)
+        script_4, plot_4 = plot_concurrent_jobs_per_day(self.df)
+        script_41, plot_41 = plot_duration(self.df)
+        script_5, plot_5 = plot_errors_per_day(self.df)
+        script_errors, table_errors = table_of_errors(self.df)
+        return template.render(
+            time_start=humanize.naturaldate(self.df["time_start"].dt.date.values[0]),
+            time_end=humanize.naturaldate(self.df["time_end"].dt.date.values[-1]),
+            plot_1=plot_1,
+            script_1=script_1,
+            plot_2=plot_2,
+            script_2=script_2,
+            plot_3=plot_3,
+            script_3=script_3,
+            plot_4=plot_4,
+            script_4=script_4,
+            plot_41=plot_41,
+            script_41=script_41,
+            plot_5=plot_5,
+            script_5=script_5,
+            table_errors=table_errors,
+            script_errors=script_errors,
+        )