RDISCROWD-5739 Admin Usage Dashboard (#824)

* add total projects, tasks, taskruns

* bump themes pointer

pybossa/cache/__init__.py

@@ -134,9 +134,9 @@ def cache(key_prefix, timeout=300, cache_group_keys=None):
     """
     Adding a random jitter to reduce DB load
     There are scheduled jobs refreshing cache. When refreshing happens, caches
-    could have the same TTL. Thus they could expires at the same time, and 
-    requests will hitting DB, causing a burst of DB load. By adding a random 
-    jitter, it reduces the possibility that caches expiring at the same time 
+    could have the same TTL. Thus they could expires at the same time, and
+    requests will hitting DB, causing a burst of DB load. By adding a random
+    jitter, it reduces the possibility that caches expiring at the same time
     and balanced the DB load to avoid many requests hitting the DB.
     """
     timeout += randrange(30)

pybossa/cache/site_stats.py

@@ -30,6 +30,15 @@
 
 session = db.slave_session
 
+def allow_all_time(func):
+    @wraps(func)
+    def wrapper(days=30):
+        if days == 'all':
+            days = 999999
+        return func(days=days)
+
+    return wrapper
+
 
 @memoize_with_l2_cache(timeout=ONE_DAY, timeout_l2=TWO_WEEKS,
                        key_prefix="site_n_auth_users")
@@ -66,13 +75,16 @@ def n_anon_users():
 
 @memoize_with_l2_cache(timeout=ONE_DAY, timeout_l2=TWO_WEEKS,
                        key_prefix="site_n_tasks")
-def n_tasks_site():
+def n_tasks_site(days='all'):
     """Return number of tasks in the server."""
-    sql = text('''SELECT COUNT(task.id) AS n_tasks FROM task''')
-    results = session.execute(sql)
-    for row in results:
-        n_tasks = row.n_tasks
-    return n_tasks or 0
+    sql = '''SELECT COUNT(task.id) AS n_tasks FROM task'''
+    if days != 'all':
+        sql += '''
+            WHERE
+                to_timestamp(task.created, 'YYYY-MM-DD"T"HH24:MI:SS.US') > current_timestamp - interval ':days days'
+        '''
+    data = {'days' : days}
+    return session.execute(text(sql), data).scalar()
 
 
 @memoize_with_l2_cache(timeout=ONE_DAY, timeout_l2=TWO_WEEKS,
@@ -90,13 +102,15 @@ def n_total_tasks_site():
 
 @memoize_with_l2_cache(timeout=ONE_DAY, timeout_l2=TWO_WEEKS,
                        key_prefix="site_n_task_runs")
-def n_task_runs_site():
+def n_task_runs_site(days="all"):
     """Return number of task runs in the server."""
-    sql = text('''SELECT COUNT(task_run.id) AS n_task_runs FROM task_run''')
-    results = session.execute(sql)
-    for row in results:
-        n_task_runs = row.n_task_runs
-    return n_task_runs or 0
+    sql = '''SELECT COUNT(task_run.id) AS n_task_runs FROM task_run'''
+    if days != 'all':
+        sql += '''
+            WHERE to_timestamp(task_run.finish_time, 'YYYY-MM-DD"T"HH24:MI:SS.US') > current_timestamp - interval ':days days'
+        '''
+    data = {'days' : days}
+    return session.execute(text(sql), data).scalar()
 
 
 @memoize_with_l2_cache(timeout=ONE_DAY, timeout_l2=TWO_WEEKS,
@@ -157,28 +171,20 @@ def get_top5_users_24_hours():
     return top5_users_24_hours
 
 
-def allow_all_time(func):
-    @wraps(func)
-    def wrapper(days=30):
-        if days == 'all':
-            days = 999999
-        return func(days=days)
-
-    return wrapper
-
-
 @memoize_with_l2_cache(timeout=ONE_WEEK, timeout_l2=TWO_WEEKS,
                        cache_group_keys=['number_of_created_jobs'])
-@allow_all_time
 def number_of_created_jobs(days=30):
-    """Number of created jobs"""
-    sql = text('''
-        SELECT COUNT(id) FROM project
-        WHERE
-        clock_timestamp() - to_timestamp(created, 'YYYY-MM-DD"T"HH24:MI:SS.US')
-            < interval ':days days';
-    ''')
-    return session.execute(sql, dict(days=days)).scalar()
+    """Number of created jobs by interval"""
+    sql = '''SELECT COUNT(id) FROM project'''
+    if days != 'all':
+        sql += '''
+            WHERE
+                clock_timestamp() - to_timestamp(created, 'YYYY-MM-DD"T"HH24:MI:SS.US')
+                < interval ':days days'
+                '''
+    data = {'days': days}
+
+    return session.execute(text(sql), data).scalar()
 
 
 @memoize_with_l2_cache(timeout=ONE_WEEK, timeout_l2=TWO_WEEKS,
@@ -237,7 +243,7 @@ def number_of_active_users(days=30):
     # TODO - revisit the SQL performance with index on finish_time in
     #  task_run table after DB engine upgrade
     sql = text('''
-        SELECT COUNT(DISTINCT(user_id)) as id 
+        SELECT COUNT(DISTINCT(user_id)) as id
         FROM task_run
         WHERE task_run.finish_time > CAST(CURRENT_DATE - INTERVAL ':days days' AS TEXT);
     ''')
@@ -305,11 +311,11 @@ def tasks_per_category():
     sql = text('''
         WITH tasks AS (
             SELECT SUM(n_tasks) AS n_tasks FROM project_stats ps
-            JOIN project p ON p.id = ps.project_id 
+            JOIN project p ON p.id = ps.project_id
             JOIN category c ON c.id = p.category_id
             WHERE n_tasks > 0
             GROUP BY c.id
-        ) 
+        )
         SELECT AVG(n_tasks) FROM tasks;
     ''')
     return session.execute(sql).scalar() or 'N/A'
@@ -413,6 +419,27 @@ def submission_chart():
     return dict(labels=labels, series=[series])
 
 
+@memoize_with_l2_cache(timeout=ONE_DAY, timeout_l2=TWO_WEEKS,
+                       key_prefix="n_projects_using_component")
+def n_projects_using_component(days='all', component=None):
+    """
+    Fetch total projects using component
+    """
+    component = '%' + component + '%'
+    sql = '''
+            SELECT count(project.id) AS n_projects
+            FROM project
+            LEFT JOIN "user" ON project.owner_id = "user".id
+            WHERE project.info->>'task_presenter' like :component
+        '''
+    if days != 'all':
+        sql += '''
+            AND to_timestamp(project.updated, 'YYYY-MM-DD"T"HH24:MI:SS.US') > current_timestamp - interval ':days days'
+        '''
+    data = {'days' : days, 'component' : component}
+    return session.execute(text(sql), data).scalar()
+
+
 def management_dashboard_stats_cached():
     stats_cached = all([sentinel.slave.smembers(get_cache_group_key(ms))
                         for ms in management_dashboard_stats])

pybossa/jobs.py

@@ -1355,6 +1355,23 @@ def delete_file(fname, container):
     from pybossa.core import uploader
     return uploader.delete_file(fname, container)
 
+def load_usage_dashboard_data(days):
+    timed_stats_funcs = [
+        (site_stats.number_of_created_jobs, "Projects"),
+        (site_stats.n_tasks_site, "Tasks"),
+        (site_stats.n_task_runs_site, "Taskruns"),
+    ]
+
+    # total tasks, taskruns, projects over a specified amount of time.
+    stats = OrderedDict()
+    for func, title in timed_stats_funcs:
+        stats[title] = func(days)
+
+    # component usage
+    for name, tag in current_app.config.get("USAGE_DASHBOARD_COMPONENTS", {}).items():
+        stats[name] = site_stats.n_projects_using_component(days=days, component=tag)
+
+    return stats
 
 def load_management_dashboard_data():
     # charts

pybossa/settings_local.py.tmpl

@@ -506,3 +506,24 @@ WIZARD_STEPS = OrderedDict([
 )
 
 MAX_IMAGE_UPLOAD_SIZE_MB = 5
+
+USAGE_DASHBOARD_COMPONENTS = {
+    'Annex' : 'annex-shell',
+    'DocX' : 'loadDocument',
+    'NLP Component' : 'text-tagging',
+    'Task Presenter' : 'task-presenter',
+    'All Buttons' : 'button-row',
+    'Submit Button' : 'submit-button',
+    'Submit and Leave Button' : 'submit-last-button',
+    'Cancel Button' : 'cancel-button',
+    'Task Timer' : 'task-timer',
+    'Conditional Display' : 'conditional-display',
+    'File Upload' : 'file-upload',
+    'Text Input' : 'text-input',
+    'Checkbox Input' : 'checkbox-input',
+    'Radio Group Input' : 'radio-group-input',
+    'Dropdown Input' : 'dropdown-input',
+    'Multiselect Input' : 'multi-select-input',
+    'Table' : 'table-element',
+    'Input Text Area' : 'input-text-area'
+}

pybossa/view/admin.py

@@ -50,7 +50,9 @@
 from pybossa.forms.admin_view_forms import *
 from pybossa.importers import BulkImportException
 from pybossa.jobs import get_dashboard_jobs, export_all_users, \
-    load_management_dashboard_data, perform_completed_tasks_cleanup
+    load_management_dashboard_data, \
+    load_usage_dashboard_data, \
+    perform_completed_tasks_cleanup
 from pybossa.jobs import get_management_dashboard_stats
 from pybossa.jobs import send_mail
 from pybossa.model import make_timestamp
@@ -578,6 +580,23 @@ def management_dashboard():
                            submission_chart=submission_chart)
 
 
+@blueprint.route('/usage_dashboard/')
+@login_required
+@admin_required
+def usage_dashboard():
+    days = str(request.args.get('days'))
+    if days == 'all':
+        pass
+    elif not days.isnumeric():
+        return redirect(url_for('admin.usage_dashboard', days=365))
+    else:
+        days = int(days)
+
+    stats = load_usage_dashboard_data(days)
+    return render_template('admin/usage_dashboard.html',
+                           stats=stats)
+
+
 @blueprint.route('/subadminusers', methods=['GET', 'POST'])
 @login_required
 @admin_required

test/test_admin.py

@@ -1337,6 +1337,33 @@ def test_admin_dashboard_admin_refresh_user_data_json(self, mock):
         for key in keys:
             assert key in data.keys(), data
 
+    @with_context
+    def test_usage_dashboard(self):
+        url = '/admin/usage_dashboard/?days=30'
+        self.register()
+        self.signin()
+        res = self.app.get(url, follow_redirects=True)
+        assert res.status_code == 200, 'expected 200 status code'
+        assert '1 Month' in str(res.data), 'expected 1 month in response html'
+
+    @with_context
+    def test_usage_dashboard_redirect_empty_params(self):
+        url = '/admin/usage_dashboard/'
+        self.register()
+        self.signin()
+        res = self.app.get(url, follow_redirects=True)
+        assert res.history and res.history[0], 'expected redirect'
+        assert 'admin/usage_dashboard/?days=365' in str(res.history[0].data)
+
+    @with_context
+    def test_usage_dashboard_redirect_bad_params(self):
+        url = '/admin/usage_dashboard/?months=some_nonsense'
+        self.register()
+        self.signin()
+        res = self.app.get(url, follow_redirects=True)
+        assert res.history and res.history[0], 'expected redirect'
+        assert 'admin/usage_dashboard/?days=365' in str(res.history[0].data)
+
     @with_context
     def test_announcement_json(self):
         """Test ADMIN JSON announcement"""