Merge pull request #83 from mozilla/b2g-app-summary

B2g app summary

datazilla/controller/admin/refdata/objectstore_refdata.py

@@ -29,6 +29,17 @@ def get_json_blob(project, id):
 
     return {}
 
+def get_json_blob_by_test_run_id(project, test_run_id):
+    """Based on the test_run_id passed in, return the JSON blob"""
+    ptm = PerformanceTestRefDataModel(project)
+    blob = ptm.get_object_json_blob_for_test_run([test_run_id])
+    ptm.disconnect()
+
+    if blob:
+        return blob[0]
+
+    return {}
+
 
 def get_error_detail_count(project, startdate, enddate):
     """Return counts attempting to parse some of the bad JSON to extract details."""

datazilla/controller/admin/testdata.py

@@ -233,5 +233,16 @@ def get_default_version(project, branch, product_name):
 
     return version
 
+def get_test_value_summary(project, test_ids, url, begin, now):
+
+    ptm = factory.get_ptm(project)
+
+    data = ptm.get_value_summary_by_test_ids(
+        test_ids, url, begin, now
+        )
+
+    ptm.disconnect()
+
+    return data
 
 

datazilla/model/base.py

@@ -776,13 +776,21 @@ def get_pages(self):
 
         proc = 'perftest.selects.get_pages'
 
-        pages_dict = self.sources["perftest"].dhub.execute(
+        pages_tuple = self.sources["perftest"].dhub.execute(
             proc=proc,
             debug_show=self.DEBUG,
-            key_column='url',
-            return_type='dict',
             )
 
+        pages_dict = {}
+        for page_data in pages_tuple:
+            if page_data['url'] not in pages_dict:
+                pages_dict[ page_data['url'] ] = {}
+                pages_dict[ page_data['url'] ]['test_ids'] = {}
+                pages_dict[ page_data['url'] ]['id'] = page_data['id']
+                pages_dict[ page_data['url'] ]['url'] = page_data['url']
+
+            pages_dict[ page_data['url'] ]['test_ids'][ page_data['test_id'] ] = True
+
         return pages_dict
 
 
@@ -958,6 +966,30 @@ def get_test_run_value_summary(self, test_run_id):
 
         return test_run_value_table
 
+    def get_value_summary_by_test_ids(
+        self, test_ids, url, begin_date, end_date
+        ):
+
+        data = []
+
+        if test_ids and url and begin_date and end_date:
+
+            proc = 'perftest.selects.get_value_summary_by_test_id'
+
+            r_string = ','.join( map( lambda t_id: '%s', test_ids ) )
+
+            test_ids.append( url )
+            test_ids.append( begin_date )
+            test_ids.append( end_date )
+
+            data = self.sources["perftest"].dhub.execute(
+                proc=proc,
+                debug_show=self.DEBUG,
+                placeholders=test_ids,
+                replace=[ r_string ]
+                )
+
+        return data
 
     def get_test_run_ids(
         self, branch, revisions, product_name=None, os_name=None,

datazilla/model/sql/perftest.json

@@ -524,6 +524,34 @@
 
          "host":"read_host"
 
+    },
+    "get_value_summary_by_test_id":{
+
+        "sql":"SELECT tv.test_run_id,
+                      tv.page_id,
+                      tr.revision,
+                      tr.test_id,
+                      tr.date_run,
+                      b.product_id,
+                      tr.test_id,
+                      m.operating_system_id,
+                      p.url,
+                      ROUND( AVG(tv.value), 2 ) AS avg,
+                      ROUND( MIN(tv.value), 2 ) AS min,
+                      ROUND( MAX(tv.value), 2 ) AS max,
+                      ROUND( STDDEV(tv.value), 2 ) AS 'std'
+              FROM test_value AS tv
+              LEFT JOIN pages AS p ON tv.page_id = p.id
+              LEFT JOIN value AS v ON tv.value_id = v.id
+              LEFT JOIN test_run AS tr ON tv.test_run_id = tr.id
+              LEFT JOIN machine AS m ON tr.machine_id = m.id
+              LEFT JOIN build AS b ON tr.build_id = b.id
+              WHERE tr.test_id IN (REP0) AND p.url = ? AND tr.date_run >= ? AND tr.date_run <= ?
+              GROUP BY tr.id, p.url
+              ORDER BY tr.id DESC",
+
+         "host":"read_host"
+
     },
     "get_page_values":{
 

datazilla/settings/local.sample.py

@@ -22,6 +22,9 @@
 # base URL
 DATAZILLA_URL               = os.environ.get("DATAZILLA_URL", "/")
 
+#pipe delimited list of allowed project names, defaults to \w+ if not set
+ALLOWED_PROJECTS            = os.environ.get("ALLOWED_PROJECTS", "")
+
 # This should always be False in production
 DEBUG = os.environ.get("DATAZILLA_DEBUG") is not None
 

datazilla/webapp/apps/datazilla/refdata/objectstore_views.py

@@ -35,7 +35,7 @@ def get_error_count(request, project):
 
 
 def get_json_blob(request, project, id):
-    """Return a count of all objectstore entries with error"""
+    """Return a json object for the objectstore id provided"""
 
     blob = objectstore_refdata.get_json_blob(project, id)
 
@@ -55,6 +55,28 @@ def get_json_blob(request, project, id):
     else:
         return HttpResponse("Id not found: {0}".format(id), status=404)
 
+def get_json_blob_by_test_run_id(request, project, test_run_id):
+    """Return a json object for the test_run_id provided"""
+
+    blob = objectstore_refdata.get_json_blob_by_test_run_id(
+        project, test_run_id)
+
+    if blob:
+
+        # If we don't have malformed json load it so we can return
+        # a single json data structure with all fields present including
+        # json_blob.  Malformed json will be returned as an escaped
+        # string.
+        try:
+            blob['json_blob'] = json.loads(blob['json_blob'])
+        except ValueError as e:
+            pass
+
+        return HttpResponse(json.dumps(blob), content_type=API_CONTENT_TYPE)
+
+    else:
+        return HttpResponse("Id not found: {0}".format(id), status=404)
+
 
 def get_db_size(request, project):
     """Return the size of the DB on disk in MB."""

datazilla/webapp/apps/datazilla/refdata/urls.py

@@ -10,6 +10,8 @@
 
     (r"^objectstore/json_blob/(?P<id>\d+)/?$", "objectstore_views.get_json_blob"),
 
+    (r"^objectstore/json_blob/test_run/(?P<test_run_id>\d+)/?$", "objectstore_views.get_json_blob_by_test_run_id"),
+
     (r"^objectstore/db_size/?$", "objectstore_views.get_db_size"),
 
     # perftest

datazilla/webapp/apps/datazilla/testdata/urls.py

@@ -5,6 +5,8 @@
 
     (r"^raw/(?P<branch>.+)/(?P<revision>\w+)/?$", "views.get_testdata"),
 
+    (r"^test_values/?$", "views.get_test_value_summary"),
+
     (r"^metrics/(?P<branch>.+)/(?P<revision>\w+)/pushlog/?$",
         "views.get_metrics_pushlog"),
 

datazilla/webapp/apps/datazilla/testdata/views.py

@@ -1,8 +1,10 @@
 import json
+import time
 
 from django.http import HttpResponse
 
 from datazilla.controller.admin import testdata
+from datazilla.model import utils
 
 REQUIRE_DAYS_AGO = """Invalid Request: Require days_ago parameter.
                     This specifies the number of days ago to use as the start
@@ -213,5 +215,29 @@ def get_application_log(request, project, revision):
         content_type=API_CONTENT_TYPE,
         )
 
+def get_test_value_summary(request, project):
 
+    test_ids = utils.get_id_list(request.GET['test_ids'])
+    page_name = request.GET.get("page_name", "")
+    range = request.GET.get("range", 30)
+
+    now = int(time.time())
+
+    begin = now - 2592000
+
+    if range == 60:
+        begin = now - 5184000
+    elif range == 90:
+        begin = now - 7776000
+
+    return HttpResponse(
+        json.dumps(testdata.get_test_value_summary(
+            project,
+            test_ids,
+            page_name,
+            begin,
+            now
+            )),
+        content_type=API_CONTENT_TYPE,
+        )
 

datazilla/webapp/apps/summary/b2g_app_urls.py

@@ -0,0 +1,7 @@
+from django.conf.urls.defaults import *
+from datazilla.webapp.apps.summary import summary_view
+
+urlpatterns = patterns('',
+                       #UI Application
+                       (r'^$', summary_view.apps_summary_page)
+                       )

datazilla/webapp/apps/summary/summary_view.py

@@ -18,3 +18,11 @@ def summary_page(request, project="", branch="", revision=""):
         'metrics.summary.html', template_context
         )
 
+def apps_summary_page(request, project="", branch="", revision=""):
+
+    #give template access to the DEBUG variable
+    template_context = { 'DEBUG':settings.DEBUG }
+
+    return render_to_response(
+        'apps.summary.html', template_context
+        )

datazilla/webapp/static/css/apps.css

@@ -0,0 +1,136 @@
+body {
+    font-family: helvetica, arial, sans-serif;
+    margin: 0;
+    font-size: 10px;
+    padding: 0;
+    border-top: 2px solid #676767;
+    min-width: 1120px;
+}
+div.app-toppanel {
+    -moz-box-sizing: border-box;
+    -webkit-box-sizing: border-box;
+    border: 0 solid #000;
+    background: -moz-linear-gradient(-90deg, #5CB2CB, #297E96);
+    background: -webkit-gradient(linear, left top, left bottom, from(#5CB2CB), to(#297E96));
+    height: 50px;
+    width: 1120px;
+    margin-top: 20px;
+    padding-top: 10px;
+}
+div.app-title {
+  background: url("/static/images/title.png") no-repeat scroll 0 center transparent;
+  position: absolute;
+  left: 15px;
+  width:400px;
+  height:100%;
+  top: 0;
+  z-index: 1;
+}
+a {
+    text-decoration: underline;
+    font-weight: bold;
+}
+
+div.app-container {
+    -moz-box-sizing: border-box;
+    -webkit-box-sizing: border-box;
+    border: 0 solid #000;
+    width: 1120px;
+    padding-top: 10px;
+    position: relative;
+    margin-left: auto;
+    margin-right: auto;
+    background:white;
+}
+div.app-list-controls {
+    position:absolute;
+    width: 150px;
+    height: 25px;
+    background: white;
+}
+div.app-list-controls-two {
+    position:absolute;
+    width: 150px;
+    height: 25px;
+    background: white;
+    top: 495px;
+}
+div.app-list {
+    position:absolute;
+    width: 150px;
+    height: 420px;
+    top:35px;
+    background: white;
+    overflow: auto;
+}
+div.app-list-two {
+    position:absolute;
+    width: 150px;
+    height: 380px;
+    top: 520px;
+    background: white;
+    overflow: auto;
+}
+div.app-control-container {
+    position:absolute;
+    width: 958px;
+    height: 25px;
+    background: white;
+    left: 160px;
+}
+div.app-control-container-two {
+    position:absolute;
+    width: 958px;
+    height: 25px;
+    background: white;
+    left: 160px;
+    top: 495px;
+}
+div.app-control-element {
+    padding: 5px;
+    float: left;
+}
+div.app-control-small-element {
+    margin-top: 2px;
+}
+div.app-graph-container {
+    position:absolute;
+    width: 958px;
+    height: 420px;
+    background: white;
+    top:35px;
+    left: 160px;
+}
+div.app-graph-container-two {
+    position:absolute;
+    width: 958px;
+    height: 380px;
+    background: white;
+    left: 160px;
+    top:520px;
+}
+div.app-hints {
+    position:absolute;
+    top: 470px;
+}
+div.app-detail-graph-container {
+    position:absolute;
+    width: 905px;
+    height: 25px;
+    background: white;
+    left: 30px;
+    top: 385px;
+
+    display:inline;
+}
+span.app-data {
+    margin-left:3px;
+}
+div.app-build-data{
+    width:100%;
+    float:left;
+}
+div.app-spinner {
+  background: transparent url(/static/images/spinner.gif) no-repeat center center;
+  height:420px;
+}