From ad79426fcd55dfb67f17ed977c99d92bd2b646a3 Mon Sep 17 00:00:00 2001 From: jbannister Date: Fri, 19 Nov 2021 18:09:18 +0000 Subject: [PATCH 1/8] First cut of new and improved index page for Notebooker --- notebooker/serialization/mongo.py | 30 +- notebooker/utils/results.py | 19 +- notebooker/web/report_hunter.py | 4 +- notebooker/web/routes/core.py | 15 +- notebooker/web/routes/index.py | 25 +- notebooker/web/static/gulpfile.js | 69 +-- notebooker/web/static/notebooker/index.js | 167 +----- .../static/notebooker/one_click_notebooks.css | 2 +- .../web/static/notebooker/result_listing.js | 181 ++++++ notebooker/web/static/package.json | 2 +- notebooker/web/static/semantic.json | 2 +- notebooker/web/static/yarn.lock | 537 ++++-------------- notebooker/web/templates/index.html | 19 + notebooker/web/templates/result_listing.html | 38 ++ notebooker/web/utils.py | 6 + setup.py | 2 + 16 files changed, 509 insertions(+), 609 deletions(-) create mode 100644 notebooker/web/static/notebooker/result_listing.js create mode 100644 notebooker/web/templates/result_listing.html diff --git a/notebooker/serialization/mongo.py b/notebooker/serialization/mongo.py index b982790f..c10eeb96 100644 --- a/notebooker/serialization/mongo.py +++ b/notebooker/serialization/mongo.py @@ -1,5 +1,7 @@ import datetime import json +from collections import Counter, defaultdict + from abc import ABC from logging import getLogger from typing import Any, AnyStr, Dict, List, Optional, Tuple, Union, Iterator @@ -300,6 +302,26 @@ def get_check_result( result = self.library.find_one({"job_id": job_id}, {"_id": 0}) return self._convert_result(result) + def _get_raw_results(self, base_filter, projection, limit): + if "status" in base_filter: + base_filter["status"].update({"$ne": JobStatus.DELETED.value}) + else: + base_filter["status"] = {"$ne": JobStatus.DELETED.value} + return self.library.find(base_filter, projection).sort("update_time", -1).limit(limit) + + def get_count_and_latest_time_per_report(self): + reports = list( + self._get_raw_results(base_filter={}, projection={"report_name": 1, "job_start_time": 1, "_id": 0}, limit=0) + ) + jobs_by_name = defaultdict(list) + for r in reports: + jobs_by_name[r["report_name"]].append(r) + output = {} + for report, all_runs in jobs_by_name.items(): + latest_start_time = max(r["job_start_time"] for r in all_runs) + output[report] = {"count": len(all_runs), "latest_run": latest_start_time} + return output + def get_all_results( self, since: Optional[datetime.datetime] = None, @@ -307,13 +329,13 @@ def get_all_results( mongo_filter: Optional[Dict] = None, load_payload: bool = True, ) -> Iterator[Union[NotebookResultComplete, NotebookResultError, NotebookResultPending]]: - base_filter = {"status": {"$ne": JobStatus.DELETED.value}} + base_filter = {} if mongo_filter: base_filter.update(mongo_filter) if since: base_filter.update({"update_time": {"$gt": since}}) projection = REMOVE_ID_PROJECTION if load_payload else REMOVE_PAYLOAD_FIELDS_AND_ID_PROJECTION - results = self.library.find(base_filter, projection).sort("update_time", -1).limit(limit) + results = self._get_raw_results(base_filter, projection, limit) for res in results: if res: converted_result = self._convert_result(res, load_payload=load_payload) @@ -404,8 +426,8 @@ def get_latest_successful_job_ids_for_name_all_params(self, report_name: str) -> return [result["job_id"] for result in results] - def n_all_results(self): - return self.library.find({"status": {"$ne": JobStatus.DELETED.value}}).count() + def n_all_results_for_report_name(self, report_name: str) -> int: + return self._get_raw_results({"report_name": report_name}, {}, 0).count() def delete_result(self, job_id: AnyStr) -> None: self.update_check_status(job_id, JobStatus.DELETED) diff --git a/notebooker/utils/results.py b/notebooker/utils/results.py index 74d89066..d451e043 100644 --- a/notebooker/utils/results.py +++ b/notebooker/utils/results.py @@ -1,7 +1,11 @@ +import datetime +from collections import defaultdict from datetime import datetime as dt from logging import getLogger from typing import Callable, Dict, Iterator, List, Mapping, Optional, Tuple +import babel.dates +import inflection from flask import url_for from notebooker import constants @@ -106,9 +110,10 @@ def get_all_result_keys( return all_keys -def get_all_available_results_json(serializer: MongoResultSerializer, limit: int) -> List[constants.NotebookResultBase]: +def get_all_available_results_json(serializer: MongoResultSerializer, limit: int, report_name: str = None) -> List[constants.NotebookResultBase]: json_output = [] - for result in serializer.get_all_results(limit=limit, load_payload=False): + mongo_filter = {"report_name": report_name} if report_name is not None else {} + for result in serializer.get_all_results(mongo_filter=mongo_filter, limit=limit, load_payload=False): output = result.saveable_output() output["result_url"] = url_for( "serve_results_bp.task_results", job_id=output["job_id"], report_name=output["report_name"] @@ -126,6 +131,16 @@ def get_all_available_results_json(serializer: MongoResultSerializer, limit: int return json_output +def get_count_and_latest_time_per_report(serializer: MongoResultSerializer): + reports = serializer.get_count_and_latest_time_per_report() + output = {} + for report_name, metadata in sorted(reports.items(), key=lambda x: x[1]["latest_run"], reverse=True): + metadata["report_name"] = report_name + metadata["time_diff"] = babel.dates.format_timedelta(datetime.datetime.utcnow() - metadata["latest_run"]) + output[inflection.titleize(report_name)] = metadata + return output + + def get_latest_successful_job_results_all_params( report_name: str, serializer: MongoResultSerializer, diff --git a/notebooker/web/report_hunter.py b/notebooker/web/report_hunter.py index 2a15515d..70994d76 100644 --- a/notebooker/web/report_hunter.py +++ b/notebooker/web/report_hunter.py @@ -36,8 +36,8 @@ def _report_hunter(webapp_config: WebappConfig, run_once: bool = False, timeout: ) now = datetime.datetime.now() cutoff = { - JobStatus.SUBMITTED: now - datetime.timedelta(minutes=SUBMISSION_TIMEOUT), - JobStatus.PENDING: now - datetime.timedelta(minutes=RUNNING_TIMEOUT), + JobStatus.SUBMITTED.value: now - datetime.timedelta(minutes=SUBMISSION_TIMEOUT), + JobStatus.PENDING.value: now - datetime.timedelta(minutes=RUNNING_TIMEOUT), } for result in all_pending: this_cutoff = cutoff.get(result.status) diff --git a/notebooker/web/routes/core.py b/notebooker/web/routes/core.py index 29f89963..cb017bba 100644 --- a/notebooker/web/routes/core.py +++ b/notebooker/web/routes/core.py @@ -1,7 +1,7 @@ from flask import Blueprint, jsonify, request import notebooker.version -from notebooker.utils.results import get_all_available_results_json +from notebooker.utils.results import get_all_available_results_json, get_count_and_latest_time_per_report from notebooker.web.utils import get_serializer, get_all_possible_templates, all_templates_flattened core_bp = Blueprint("core_bp", __name__) @@ -30,7 +30,18 @@ def all_available_results(): kick off a download, if requested. """ limit = int(request.args.get("limit", 50)) - return jsonify(get_all_available_results_json(get_serializer(), limit)) + report_name = request.args.get("report_name") + return jsonify(get_all_available_results_json(get_serializer(), limit, report_name=report_name)) + + +@core_bp.route("/core/get_all_templates_with_results") +def all_available_templates_with_results(): + """ + Core function for the index.html view which shows the templates which have results available. + + :returns: A JSON containing a list of template names with a count of how many results are in each. + """ + return jsonify(get_count_and_latest_time_per_report(get_serializer())) @core_bp.route("/core/all_possible_templates") diff --git a/notebooker/web/routes/index.py b/notebooker/web/routes/index.py index 244b25a3..b8cf62e2 100644 --- a/notebooker/web/routes/index.py +++ b/notebooker/web/routes/index.py @@ -1,5 +1,6 @@ import traceback +import inflection from flask import Blueprint, current_app, request, render_template, url_for, jsonify from notebooker.constants import JobStatus from notebooker.utils.results import get_all_result_keys @@ -10,6 +11,23 @@ @index_bp.route("/", methods=["GET"]) def index(): + """ + The index page which shows cards of each report which has at least one result in the database. + """ + username = request.headers.get("X-Auth-Username") + all_reports = get_all_possible_templates() + with current_app.app_context(): + result = render_template( + "index.html", + all_reports=all_reports, + donevalue=JobStatus.DONE, # needed so we can check if a result is available + username=username, + ) + return result + + +@index_bp.route("/result_listing/", methods=["GET"]) +def result_listing(report_name): """ The index page which returns a blank table which is async populated by /core/all_available_results. Async populating the table from a different URL means that we can lock down the "core" blueprint to @@ -19,12 +37,13 @@ def index(): all_reports = get_all_possible_templates() with current_app.app_context(): result = render_template( - "index.html", - all_jobs_url=url_for("core_bp.all_available_results"), + "result_listing.html", all_reports=all_reports, - n_results_available=get_serializer().n_all_results(), + n_results_available=get_serializer().n_all_results_for_report_name(report_name), donevalue=JobStatus.DONE, # needed so we can check if a result is available username=username, + report_name=report_name, + titleised_report_name=inflection.titleize(report_name) ) return result diff --git a/notebooker/web/static/gulpfile.js b/notebooker/web/static/gulpfile.js index ae713040..3c45edb4 100644 --- a/notebooker/web/static/gulpfile.js +++ b/notebooker/web/static/gulpfile.js @@ -1,34 +1,35 @@ -/******************************* - * Set-up - *******************************/ - -var - gulp = require('gulp'), - - // read user config to know what task to load - config = require('./tasks/config/user') -; - - -/******************************* - * Tasks - *******************************/ - -require('./tasks/collections/build')(gulp); -require('./tasks/collections/install')(gulp); - -gulp.task('default', gulp.series('watch')); - -/*-------------- - Docs ----------------*/ - -require('./tasks/collections/docs')(gulp); - -/*-------------- - RTL ----------------*/ - -if (config.rtl) { - require('./tasks/collections/rtl')(gulp); -} \ No newline at end of file +/******************************* + * Set-up + *******************************/ + +var + gulp = require('gulp'), + + // read user config to know what task to load + config = require('./tasks/config/user') +; + + +/******************************* + * Tasks + *******************************/ + +require('./tasks/collections/build')(gulp); +require('./tasks/collections/various')(gulp); +require('./tasks/collections/install')(gulp); + +gulp.task('default', gulp.series('watch')); + +/*-------------- + Docs +---------------*/ + +require('./tasks/collections/docs')(gulp); + +/*-------------- + RTL +---------------*/ + +if (config.rtl) { + require('./tasks/collections/rtl')(gulp); +} diff --git a/notebooker/web/static/notebooker/index.js b/notebooker/web/static/notebooker/index.js index 43303489..f774100c 100644 --- a/notebooker/web/static/notebooker/index.js +++ b/notebooker/web/static/notebooker/index.js @@ -1,43 +1,36 @@ -add_delete_callback = () => { - $('.deletebutton').click((clicked) => { - const to_delete = clicked.target.closest('button').id.split('_')[1]; - $('#deleteModal').modal({ - closable: true, - onDeny() { - return true; - }, - onApprove() { - $.ajax({ - type: 'POST', - url: `/delete_report/${to_delete}`, // We get this from loading.html, which comes from flask - dataType: 'json', - success(data, status, request) { - if (data.status === 'error') { - $('#errorMsg').text(data.content); - $('#errorPopup').show(); - } else { - window.location.reload(); - } - }, - error(xhr, error) { - }, - }); - }, - }).modal('show'); - }); -}; + + load_data = (limit) => { $.ajax({ - url: `/core/get_all_available_results?limit=${limit}`, + url: `/core/get_all_templates_with_results`, dataType: 'json', success: (result) => { - const table = $('#resultsTable').DataTable(); - table.clear(); - table.rows.add(result); - table.draw(); - $('#indexTableContainer').fadeIn(); - add_delete_callback(); + let $cardContainer = $('#cardContainer'); + $cardContainer.empty(); + for (let report in result) { + let stats = result[report]; + $cardContainer.append( + '' + + '
' + + '

' + report + '

\n' + + '
\n' + + ' Last ran ' + stats.time_diff + ' ago\n' + + '
' + + '
\n' + + '
\n' + + stats.count + + '
\n' + + '
\n' + + ' Runs\n' + + '
\n' + + '
' + + '
' + + '
' + + ' Original report name: ' + stats.report_name + '\n' + + '
' + + ''); + } }, error: (jqXHR, textStatus, errorThrown) => { $('#failedLoad').fadeIn(); @@ -47,109 +40,5 @@ load_data = (limit) => { $(document).ready(() => { - let columns = [ - { - title: 'Title', - name: 'title', - data: 'report_title', - }, - { - title: 'Report Template Name', - name: 'report_name', - data: 'report_name', - }, - { - title: 'Status', - name: 'status', - data: 'status', - }, - { - title: 'Start Time', - name: 'job_start_time', - data: 'job_start_time', - render: (dt) => { - const d = new Date(dt); - return d.toISOString().replace('T', ' ').slice(0, 19); - }, - }, - { - title: 'Completion Time', - name: 'job_finish_time', - data: 'job_finish_time', - render: (dt) => { - if (dt) { - const d = new Date(dt); - return d.toISOString().replace('T', ' ').slice(0, 19); - } - return ''; - }, - }, - { - title: 'Results', - name: 'result_url', - data: 'result_url', - render: (url) => `', - }, - { - title: 'PDF', - name: 'pdf_url', - data: 'pdf_url', - render: (url, type, row) => { - if (row.generate_pdf_output) { - return `'; - } - return ''; - }, - }] - var usingScheduler = undefined; - $.ajax({ - async: false, - url: '/scheduler/health', - success: () => { - usingScheduler = true; - }, - error: () => { - usingScheduler = false; - }, - }); - if (usingScheduler === true) { - columns = columns.concat([ - { - title: 'Scheduler Job', - name: 'scheduler_job_id', - data: 'scheduler_job_id', - render: (url, type, row) => { - if (row.scheduler_job_id) { - return ``; - } else { - return ''; - } - } - } - ]) - } - columns = columns.concat([ - { - title: 'Rerun', - name: 'rerun_url', - data: 'rerun_url', - render: (url, type, row) => `', - }, - { - title: 'Delete', - name: 'result_url', - data: 'result_url', - render: (url, type, row) => `${'', + }, + { + title: 'PDF', + name: 'pdf_url', + data: 'pdf_url', + render: (url, type, row) => { + if (row.generate_pdf_output) { + return `'; + } + return ''; + }, + }]) + var usingScheduler = undefined; + $.ajax({ + async: false, + url: '/scheduler/health', + success: () => { + usingScheduler = true; + }, + error: () => { + usingScheduler = false; + }, + }); + if (usingScheduler === true) { + columns = columns.concat([ + { + title: 'Scheduler Job', + name: 'scheduler_job_id', + data: 'scheduler_job_id', + render: (url, type, row) => { + if (row.scheduler_job_id) { + return ``; + } else { + return ''; + } + } + } + ]) + } + columns = columns.concat([ + { + title: 'Rerun', + name: 'rerun_url', + data: 'rerun_url', + render: (url, type, row) => `', + }, + { + title: 'Delete', + name: 'result_url', + data: 'result_url', + render: (url, type, row) => `${' - - - - diff --git a/notebooker/web/templates/result_listing.html b/notebooker/web/templates/result_listing.html index a645742d..58e35572 100644 --- a/notebooker/web/templates/result_listing.html +++ b/notebooker/web/templates/result_listing.html @@ -6,6 +6,7 @@ @@ -17,12 +18,21 @@

Couldn't load any reports!

Either we haven't run any reports yet, or you don't have permission to view any on this Notebooker instance. Please check with the instance admin. -

{{ titleised_report_name }} ({{ report_name }})

' + '
' + ' Original report name: ' + stats.report_name + '\n' + diff --git a/notebooker/web/static/notebooker/one_click_notebooks.css b/notebooker/web/static/notebooker/one_click_notebooks.css index 7d0d8fe4..1bb8ac83 100644 --- a/notebooker/web/static/notebooker/one_click_notebooks.css +++ b/notebooker/web/static/notebooker/one_click_notebooks.css @@ -33,6 +33,19 @@ code { clear:both; } +.ui.centered.grid { + padding: 1em; +} + +.moreResultsButton { + margin-bottom: 1em; + width: 100%; +} + +.ui.card { + color: #222222; +} + .ui.table { width: 99.5%; /* Because a border with overflow-y:auto causes a tiny scrollbar */ } diff --git a/notebooker/web/templates/index.html b/notebooker/web/templates/index.html index 16d8ae21..1c16fda5 100644 --- a/notebooker/web/templates/index.html +++ b/notebooker/web/templates/index.html @@ -20,14 +20,7 @@

Couldn't load any reports!

Loading...

-
-
- 1,234 -
-
- Runs -
-
+ Results are coming soon...
diff --git a/notebooker/web/templates/result_listing.html b/notebooker/web/templates/result_listing.html index 58e35572..21de1ccd 100644 --- a/notebooker/web/templates/result_listing.html +++ b/notebooker/web/templates/result_listing.html @@ -27,11 +27,13 @@

{{ titleised_report_name }} ({{ report_name {% if n_results_available != result_limit and n_results_available > result_limit %} +
- + Load all {{ n_results_available }} results
+
{% endif %} - {% if n_results_available != result_limit and n_results_available > result_limit %} -
-
- - Load all {{ n_results_available }} results +
+ {% if n_results_available != result_limit and n_results_available > result_limit %} +
+ +
+ {% endif %} +
+ + Manually run this report
- {% endif %}