Permalink
Browse files

added group by revision and branch select functionality

  • Loading branch information...
jeads committed Feb 1, 2013
1 parent c753d09 commit 7346f0e03ca29208c28fd951d291c1ae4e1a38a1
@@ -1,4 +1,5 @@
from datazilla.model.refdata import PerformanceTestRefDataModel
+from datazilla.model.base import PerformanceTestModel
def get_error_count(project, startdate, enddate):
@@ -40,6 +41,19 @@ def get_json_blob_by_test_run_id(project, test_run_id):
return {}
+def get_json_blob_by_revisions(project, branch, gaia_revision, gecko_revision):
+
+ ptm = PerformanceTestModel(project)
+ test_run_ids = ptm.get_test_run_ids_by_revisions(
+ branch, gaia_revision, gecko_revision
+ )
+ ptm.disconnect()
+
+ ptrm = PerformanceTestRefDataModel(project)
+ blobs = ptrm.get_object_json_blob_for_test_run(test_run_ids)
+ ptm.disconnect()
+
+ return blobs
def get_error_detail_count(project, startdate, enddate):
"""Return counts attempting to parse some of the bad JSON to extract details."""
@@ -233,12 +233,12 @@ def get_default_version(project, branch, product_name):
return version
-def get_test_value_summary(project, test_ids, url, begin, now):
+def get_test_value_summary(project, branch, test_ids, url, begin, now):
ptm = factory.get_ptm(project)
data = ptm.get_value_summary_by_test_ids(
- test_ids, url, begin, now
+ branch, test_ids, url, begin, now
)
ptm.disconnect()
View
@@ -967,17 +967,18 @@ def get_test_run_value_summary(self, test_run_id):
return test_run_value_table
def get_value_summary_by_test_ids(
- self, test_ids, url, begin_date, end_date
+ self, branch, test_ids, url, begin_date, end_date
):
data = []
- if test_ids and url and begin_date and end_date:
+ if branch and test_ids and url and begin_date and end_date:
proc = 'perftest.selects.get_value_summary_by_test_id'
r_string = ','.join( map( lambda t_id: '%s', test_ids ) )
+ test_ids.append( branch )
test_ids.append( url )
test_ids.append( begin_date )
test_ids.append( end_date )
@@ -991,6 +992,25 @@ def get_value_summary_by_test_ids(
return data
+ def get_test_run_ids_by_revisions(
+ self, branch, revision, gecko_revision):
+ #TODO: This method is specific to the b2g project and should
+ # be placed in a derived class
+
+ proc = 'perftest.selects.get_test_run_ids_from_revisions'
+
+ data = self.sources["perftest"].dhub.execute(
+ proc=proc,
+ debug_show=self.DEBUG,
+ placeholders=[revision, gecko_revision, branch]
+ )
+
+ test_run_ids = []
+ for d in data:
+ test_run_ids.append(d['id'])
+
+ return test_run_ids
+
def get_test_run_ids(
self, branch, revisions, product_name=None, os_name=None,
os_version=None, branch_version=None, processor=None,
@@ -1234,6 +1254,9 @@ def load_test_data(self, data):
self._set_test_values(data, test_id, test_run_id)
self._set_test_aux_data(data, test_id, test_run_id)
+ # Make project specific changes
+ self._adapt_project_specific_data(data, test_run_id)
+
return test_run_id
@@ -1354,6 +1377,44 @@ def _adapt_production_data(self, data):
if ('mac' in os) and ('os x' in osversion):
data['test_build']['branch'] += '-Non-PGO'
+ def _adapt_project_specific_data(self, data, test_run_id):
+
+ ###
+ #TODO: This should be moved into a derived class
+ ###
+ if self.project == 'b2g':
+ ###
+ #b2g has two unique test run fields, gecko_revision and
+ #build_revision, they need to be loaded here
+ ###
+ self._update_b2g_test_run(data, test_run_id)
+
+ def _update_b2g_test_run(self, data, test_run_id):
+
+ if 'gecko_revision' in data['test_build']:
+
+ gecko_proc = 'perftest.inserts.set_gecko_revision'
+
+ gecko_revision = data['test_build']['gecko_revision']
+
+ test_dict = self.sources["perftest"].dhub.execute(
+ proc=gecko_proc,
+ debug_show=self.DEBUG,
+ placeholders=[ gecko_revision, test_run_id ]
+ )
+
+ if 'build_revision' in data['test_build']:
+
+ build_proc = 'perftest.inserts.set_build_revision'
+
+ build_revision = data['test_build']['build_revision']
+
+ test_dict = self.sources["perftest"].dhub.execute(
+ proc=build_proc,
+ debug_show=self.DEBUG,
+ placeholders=[ build_revision, test_run_id ]
+ )
+
def _set_test_aux_data(self, data, test_id, test_run_id):
"""Insert test aux data to db for given test_id and test_run_id."""
for aux_data, aux_values in data.get('results_aux', {}).items():
@@ -1109,7 +1109,7 @@ def log_msg(self, revision, test_run_id, msg_type, msg):
def get_application_log(self, revision):
- proc = 'perftest.inserts.get_application_log'
+ proc = 'perftest.selects.get_application_log'
placeholders = [ revision ]
@@ -333,7 +333,26 @@
"host":"master_host"
},
- "get_application_log":{
+ "set_gecko_revision":{
+
+ "sql":"UPDATE test_run
+ SET gecko_revision = ?
+ WHERE id = ?",
+
+ "host":"master_host"
+ },
+ "set_build_revision":{
+
+ "sql":"UPDATE test_run
+ SET build_revision = ?
+ WHERE id = ?",
+
+ "host":"master_host"
+ }
+ },
+ "selects":{
+
+ "get_application_log":{
"sql":"SELECT `revision`,
`test_run_id`,
`msg_type`,
@@ -343,9 +362,8 @@
WHERE `revision` = ?",
"host":"read_host"
- }
- },
- "selects":{
+ },
+
"get_test_run_summary":{
"sql":"SELECT tr.id AS 'test_run_id',
@@ -530,10 +548,11 @@
"sql":"SELECT tv.test_run_id,
tv.page_id,
tr.revision,
+ tr.gecko_revision,
+ tr.build_revision,
tr.test_id,
tr.date_run,
b.product_id,
- tr.test_id,
m.operating_system_id,
p.url,
ROUND( AVG(tv.value), 2 ) AS avg,
@@ -546,13 +565,25 @@
LEFT JOIN test_run AS tr ON tv.test_run_id = tr.id
LEFT JOIN machine AS m ON tr.machine_id = m.id
LEFT JOIN build AS b ON tr.build_id = b.id
- WHERE tr.test_id IN (REP0) AND p.url = ? AND tr.date_run >= ? AND tr.date_run <= ?
- GROUP BY tr.id, p.url
- ORDER BY tr.id DESC",
+ LEFT JOIN product AS pr ON b.product_id = pr.id
+ WHERE tr.test_id IN (REP0) AND pr.branch = ? AND p.url = ? AND tr.date_run >= ? AND tr.date_run <= ?
+ GROUP BY tr.revision, tr.gecko_revision, tr.test_id, p.url
+ ORDER BY tr.date_run, tr.id DESC",
"host":"read_host"
},
+ "get_test_run_ids_from_revisions":{
+
+ "sql":"SELECT tr.id
+ FROM test_run AS tr
+ LEFT JOIN build AS b ON tr.build_id = b.id
+ LEFT JOIN product AS pr ON b.product_id = pr.id
+ WHERE tr.revision = ? AND tr.gecko_revision = ? AND pr.branch = ?
+ ORDER BY tr.id ASC",
+
+ "host":"read_host"
+ },
"get_page_values":{
"sql":"SELECT tv.id AS test_value_id,
@@ -33,6 +33,27 @@ def get_error_count(request, project):
)
return HttpResponse(json.dumps(stats), content_type=API_CONTENT_TYPE)
+def get_json_blob_by_revisions(request, project):
+
+ branch = request.GET.get("branch")
+ gaia_revision = request.GET.get("gaia_revision")
+ gecko_revision = request.GET.get("gecko_revision")
+
+ blobs = objectstore_refdata.get_json_blob_by_revisions(
+ project, branch, gaia_revision, gecko_revision)
+
+ if blobs:
+ try:
+ for index, b in enumerate(blobs):
+ blobs[index]['json_blob'] = json.loads(b['json_blob'])
+ except ValueError as e:
+ pass
+
+ return HttpResponse(json.dumps(blobs), content_type=API_CONTENT_TYPE)
+ else:
+ return HttpResponse(
+ "gaia revision, {0}, and gecko revision, {1} not found".format(gaia_revision, gecko_revision),
+ status=404)
def get_json_blob(request, project, id):
"""Return a json object for the objectstore id provided"""
@@ -8,9 +8,12 @@
(r"^objectstore/error_list/?$", "objectstore_views.get_error_list"),
+ (r"^objectstore/json_blob/revisions/?$", "objectstore_views.get_json_blob_by_revisions"),
+
(r"^objectstore/json_blob/(?P<id>\d+)/?$", "objectstore_views.get_json_blob"),
- (r"^objectstore/json_blob/test_run/(?P<test_run_id>\d+)/?$", "objectstore_views.get_json_blob_by_test_run_id"),
+ (r"^objectstore/json_blob/test_run/(?P<test_run_id>\d+)/?$",
+ "objectstore_views.get_json_blob_by_test_run_id"),
(r"^objectstore/db_size/?$", "objectstore_views.get_db_size"),
@@ -217,6 +217,7 @@ def get_application_log(request, project, revision):
def get_test_value_summary(request, project):
+ branch = request.GET['branch']
test_ids = utils.get_id_list(request.GET['test_ids'])
page_name = request.GET.get("page_name", "")
range = request.GET.get("range", 30)
@@ -232,11 +233,7 @@ def get_test_value_summary(request, project):
return HttpResponse(
json.dumps(testdata.get_test_value_summary(
- project,
- test_ids,
- page_name,
- begin,
- now
+ project, branch, test_ids, page_name, begin, now
)),
content_type=API_CONTENT_TYPE,
)
@@ -19,6 +19,7 @@ var AppsPage = new Class( {
this.gaiaHrefBase = "https://github.com/mozilla-b2g/gaia/commit/";
this.geckoHrefBase = "http://git.mozilla.org/?p=releases/gecko.git;a=commit;h=";
+ this.buildHrefBase = "https://github.com/mozilla-b2g/platform_build/commit/";
},
setRefData: function(){
@@ -20,6 +20,7 @@ var GraphControlsComponent = new Class({
this.appLookup = {};
this.testLookup = {};
+ this.branchLookup = {};
this.model.getApps(this, this.initializeAppList);
@@ -75,15 +76,32 @@ var GraphControlsComponent = new Class({
this.appLookup[ seriesDatum.id ] = seriesDatum;
}
- this.model.getTests(this, this.initializeTestList);
+ this.model.getBranches(this, this.initializeBranchList);
},
+ initializeBranchList: function(data){
+
+ var keys = _.keys(data).sort();
+
+ var branch = "";
+ var i = 0;
+
+ for(i = 0; i<keys.length; i++){
+
+ branch = data[ keys[i] ].branch;
+ if(!this.branchLookup[branch]){
+ this.branchLookup[branch] = true;
+ this.view.addBranch(branch);
+ }
+ }
+
+ //Make sure apps and branches are loaded before tests are initialized
+ this.model.getTests(this, this.initializeTestList);
+ },
initializeTestList: function(data){
var sortOrder = this.view.getAlphabeticalSortKeys(data);
- sortOrder.reverse();
-
for(var i=0; i<sortOrder.length; i++){
var seriesDatum = data[ sortOrder[i] ];
@@ -154,10 +172,12 @@ var GraphControlsView = new Class({
this.appSeriesContainerSel = '#app_series';
this.testSeriesContainerSel = '#test_series';
+ this.branchSelectMenuSel = '#app_branch';
+
this.idRegex = /^.*_(\d+)$/;
this.colors = [
- '#0b3b40', '#99911c', '#a66247', '#7989b3', '#8bccc4', '#f2eba5', '#ff622e', '#cc8bc7', '#2254bf', '#22bf90', '#666345', '#ffbead', '#481059', '#2c96f2', '#10593d', '#ffc42e', '#4c1a0e', '#a35dd9', '#add9ff', '#196612', '#332409', '#604e73', '#103859', '#3dbf22', '#7f5717', '#462eff', '#1e84a6', '#88b379', '#f2cea5', '#1e1ea6', '#2cdbf2', '#d4f22c', '#e58729', '#091233', '#a61e88', '#ff2ee3',
+ '#0b3b40', '#99911c', '#a66247', '#7989b3', '#8bccc4', '#a35dd9', '#ff622e', '#cc8bc7', '#2254bf', '#22bf90', '#666345', '#ffbead', '#481059', '#2c96f2', '#10593d', '#ffc42e', '#4c1a0e', '#add9ff', '#196612', '#332409', '#604e73', '#103859', '#3dbf22', '#7f5717', '#462eff', '#1e84a6', '#88b379', '#f2cea5', '#1e1ea6', '#2cdbf2', '#d4f22c', '#e58729', '#091233', '#a61e88', '#ff2ee3', '#f2eba5'
];
this.testColor = '#5CB2CB';
@@ -255,6 +275,12 @@ var GraphControlsView = new Class({
}
}
return id;
+ },
+ addBranch: function(branch){
+ var optionEl = $('<option></option>');
+ $(optionEl).attr('value', branch);
+ $(optionEl).text(branch);
+ $(this.branchSelectMenuSel).append(optionEl);
}
});
var GraphControlsModel = new Class({
@@ -271,6 +297,20 @@ var GraphControlsModel = new Class({
},
+ getBranches: function(context, fnSuccess){
+
+ var uri = '/' + APPS_PAGE.refData.project + '/refdata/perftest/ref_data/products';
+
+ jQuery.ajax( uri, {
+ accepts:'application/json',
+ dataType:'json',
+ cache:false,
+ type:'GET',
+ data:data,
+ context:context,
+ success:fnSuccess,
+ });
+ },
getApps: function(context, fnSuccess){
var uri = '/' + APPS_PAGE.refData.project + '/refdata/perftest/ref_data/tests';
Oops, something went wrong.

0 comments on commit 7346f0e

Please sign in to comment.