From 17b6d359ae757cfa18543eca283cefe54592aac9 Mon Sep 17 00:00:00 2001 From: Jonathan Eads Date: Fri, 25 Jul 2014 11:48:12 -0700 Subject: [PATCH 1/2] Remove job property names from resultset web service data to improve page load performance (Bugs 1032437 1032216 1032448 1042621) --- treeherder/webapp/api/resultset.py | 30 ++++++----- treeherder/webapp/api/utils.py | 80 ++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 13 deletions(-) diff --git a/treeherder/webapp/api/resultset.py b/treeherder/webapp/api/resultset.py index 5faf4302f2d..2aaed27284f 100644 --- a/treeherder/webapp/api/resultset.py +++ b/treeherder/webapp/api/resultset.py @@ -7,7 +7,8 @@ from treeherder.model.derived import DatasetNotFoundError from treeherder.webapp.api.utils import (UrlQueryFilter, with_jobs, oauth_required, get_option, - to_timestamp) + to_timestamp, get_job_value_list, + JOB_PROPERTY_RETURN_KEY) PLATFORM_ORDER = { "linux32": 0, @@ -60,7 +61,6 @@ def list(self, request, project, jm): GET method for list of ``resultset`` records with revisions """ - # make a mutable copy of these params filter_params = request.QUERY_PARAMS.copy() @@ -113,7 +113,9 @@ def list(self, request, project, jm): ) if with_jobs: - results = self.get_resultsets_with_jobs(jm, objs, full, {}) + debug = request.QUERY_PARAMS.get('debug', None) + results = self.get_resultsets_with_jobs( + jm, objs, full, {}, debug) else: results = objs @@ -122,7 +124,8 @@ def list(self, request, project, jm): return Response({ 'meta': meta, - 'results': results + 'results': results, + 'job_property_names': JOB_PROPERTY_RETURN_KEY }) @with_jobs @@ -136,7 +139,8 @@ def retrieve(self, request, project, jm, pk=None): objs = jm.get_result_set_list(0, 1, full, filter.conditions) if objs: - rs = self.get_resultsets_with_jobs(jm, objs, full, {}) + debug = request.QUERY_PARAMS.get('debug', None) + rs = self.get_resultsets_with_jobs(jm, objs, full, {}, debug) return Response(rs[0]) else: return Response("No resultset with id: {0}".format(pk), 404) @@ -151,7 +155,7 @@ def revisions(self, request, project, jm, pk=None): return Response(objs) @staticmethod - def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs): + def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs, debug): """Convert db result of resultsets in a list to JSON""" # Fetch the job results all at once, then parse them out in memory. @@ -219,21 +223,21 @@ def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs): by_job_type = sorted(list(jg_group), key=job_type_grouper) + job_list = [] groups.append({ "symbol": jg_symbol, "name": by_job_type[0]["job_group_name"], - "jobs": by_job_type + "jobs": job_list }) # build the uri ref for each job for job in by_job_type: - job["id"] = job["job_id"] - del(job["job_id"]) - del(job["option_collection_hash"]) - job["platform_option"] = platform_option - job["resource_uri"] = reverse("jobs-detail", - kwargs={"project": jm.project, "pk": job["id"]}) + job_list.append( + get_job_value_list( + job, platform_option, jm.project, debug + ) + ) if job["state"] == "completed": job_counts[job["result"]] += 1 diff --git a/treeherder/webapp/api/utils.py b/treeherder/webapp/api/utils.py index 3e41a101466..4898d66342f 100644 --- a/treeherder/webapp/api/utils.py +++ b/treeherder/webapp/api/utils.py @@ -6,10 +6,54 @@ import oauth2 as oauth from django.conf import settings from rest_framework.response import Response +from rest_framework.reverse import reverse from treeherder.model.derived import JobsModel from treeherder.etl.oauth_utils import OAuthCredentials +# To add a new property to the job object returned, +# where the database column name is identical to +# the property name, just add the column name to this +# structure. +JOB_PROPERTIES = { + "submit_timestamp": 0, + "machine_name": 1, + "job_group_symbol": 2, + "job_group_name": 3, + "platform_option": 4, + "job_type_description": 5, + "result_set_id": 6, + "result": 7, + "id": 8, + "machine_platform_architecture": 9, + "end_timestamp": 10, + "build_platform": 11, + "job_guid": 12, + "job_type_name": 13, + "platform": 14, + "state": 15, + "running_eta": 16, + "pending_eta": 17, + "build_os": 18, + "who": 19, + "failure_classification_id": 20, + "job_type_symbol": 21, + "reason": 22, + "job_group_description": 23, + "job_coalesced_to_guid": 24, + "machine_platform_os": 25, + "start_timestamp": 26, + "build_architecture": 27, + "build_platform_id": 28, + "resource_uri": 29, + "option_collection_hash": 30 +} + +# This list can maps the array indexes to the +# corresponding property names +JOB_PROPERTY_RETURN_KEY = [None]*len(JOB_PROPERTIES) +for k, v in JOB_PROPERTIES.iteritems(): + JOB_PROPERTY_RETURN_KEY[v] = k class UrlQueryFilter(object): """ @@ -213,3 +257,39 @@ def to_timestamp(datestr): datestr, "%Y-%m-%d" ).timetuple()) + +def get_job_value_list(job, platform_option, project, debug): + + if debug: + # If debug is specified return a dictionary for each + # job where the key is the full property name + job_values = {} + else: + # By default don't return all of the job property names + # with each job to reduce the size of the data structure + # returned + job_values = [None]*len(JOB_PROPERTIES) + + for p in JOB_PROPERTIES: + + key = JOB_PROPERTIES[p] + if debug: + key = p + + if p == "id": + job_values[key] = job["job_id"] + elif p == "platform_option": + job_values[key] = platform_option + elif p == "resource_uri": + job_values[key] = reverse( + "jobs-detail", + kwargs={"project": project, "pk": job["job_id"]} + ) + else: + job_values[key] = job[p] + + return job_values + + + + From 3a349b8d0459d81a3862123412646e21ed7db7c4 Mon Sep 17 00:00:00 2001 From: Jonathan Eads Date: Fri, 25 Jul 2014 11:48:12 -0700 Subject: [PATCH 2/2] Remove job property names from resultset web service data to improve page load performance (Bugs 1032437 1032216 1032448 1042621) --- tests/webapp/api/test_resultset_api.py | 4 +- treeherder/webapp/api/resultset.py | 30 +++++----- treeherder/webapp/api/utils.py | 80 ++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 15 deletions(-) diff --git a/tests/webapp/api/test_resultset_api.py b/tests/webapp/api/test_resultset_api.py index 78ebbce91eb..127a932c43d 100644 --- a/tests/webapp/api/test_resultset_api.py +++ b/tests/webapp/api/test_resultset_api.py @@ -54,7 +54,7 @@ def test_resultset_list_full_false(webapp, eleven_jobs_processed, jm): """ resp = webapp.get( reverse("resultset-list", kwargs={"project": jm.project}), - {"full": False} + {"full": False, "debug": True} ) results = resp.json['results'] @@ -81,7 +81,7 @@ def test_resultset_list_full_false(webapp, eleven_jobs_processed, jm): assert(meta == { u'count': 10, - u'filter_params': {u'full': u'False'}, + u'filter_params': {u'full': u'False', u'debug': u'True'}, u'repository': u'test_treeherder' }) diff --git a/treeherder/webapp/api/resultset.py b/treeherder/webapp/api/resultset.py index 5faf4302f2d..2aaed27284f 100644 --- a/treeherder/webapp/api/resultset.py +++ b/treeherder/webapp/api/resultset.py @@ -7,7 +7,8 @@ from treeherder.model.derived import DatasetNotFoundError from treeherder.webapp.api.utils import (UrlQueryFilter, with_jobs, oauth_required, get_option, - to_timestamp) + to_timestamp, get_job_value_list, + JOB_PROPERTY_RETURN_KEY) PLATFORM_ORDER = { "linux32": 0, @@ -60,7 +61,6 @@ def list(self, request, project, jm): GET method for list of ``resultset`` records with revisions """ - # make a mutable copy of these params filter_params = request.QUERY_PARAMS.copy() @@ -113,7 +113,9 @@ def list(self, request, project, jm): ) if with_jobs: - results = self.get_resultsets_with_jobs(jm, objs, full, {}) + debug = request.QUERY_PARAMS.get('debug', None) + results = self.get_resultsets_with_jobs( + jm, objs, full, {}, debug) else: results = objs @@ -122,7 +124,8 @@ def list(self, request, project, jm): return Response({ 'meta': meta, - 'results': results + 'results': results, + 'job_property_names': JOB_PROPERTY_RETURN_KEY }) @with_jobs @@ -136,7 +139,8 @@ def retrieve(self, request, project, jm, pk=None): objs = jm.get_result_set_list(0, 1, full, filter.conditions) if objs: - rs = self.get_resultsets_with_jobs(jm, objs, full, {}) + debug = request.QUERY_PARAMS.get('debug', None) + rs = self.get_resultsets_with_jobs(jm, objs, full, {}, debug) return Response(rs[0]) else: return Response("No resultset with id: {0}".format(pk), 404) @@ -151,7 +155,7 @@ def revisions(self, request, project, jm, pk=None): return Response(objs) @staticmethod - def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs): + def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs, debug): """Convert db result of resultsets in a list to JSON""" # Fetch the job results all at once, then parse them out in memory. @@ -219,21 +223,21 @@ def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs): by_job_type = sorted(list(jg_group), key=job_type_grouper) + job_list = [] groups.append({ "symbol": jg_symbol, "name": by_job_type[0]["job_group_name"], - "jobs": by_job_type + "jobs": job_list }) # build the uri ref for each job for job in by_job_type: - job["id"] = job["job_id"] - del(job["job_id"]) - del(job["option_collection_hash"]) - job["platform_option"] = platform_option - job["resource_uri"] = reverse("jobs-detail", - kwargs={"project": jm.project, "pk": job["id"]}) + job_list.append( + get_job_value_list( + job, platform_option, jm.project, debug + ) + ) if job["state"] == "completed": job_counts[job["result"]] += 1 diff --git a/treeherder/webapp/api/utils.py b/treeherder/webapp/api/utils.py index 3e41a101466..d72caf0a267 100644 --- a/treeherder/webapp/api/utils.py +++ b/treeherder/webapp/api/utils.py @@ -6,10 +6,54 @@ import oauth2 as oauth from django.conf import settings from rest_framework.response import Response +from rest_framework.reverse import reverse from treeherder.model.derived import JobsModel from treeherder.etl.oauth_utils import OAuthCredentials +# To add a new property to the job object returned, +# where the database column name is identical to +# the property name, just add the column name to this +# structure. +JOB_PROPERTIES = { + "submit_timestamp": 0, + "machine_name": 1, + "job_group_symbol": 2, + "job_group_name": 3, + "platform_option": 4, + "job_type_description": 5, + "result_set_id": 6, + "result": 7, + "id": 8, + "machine_platform_architecture": 9, + "end_timestamp": 10, + "build_platform": 11, + "job_guid": 12, + "job_type_name": 13, + "platform": 14, + "state": 15, + "running_eta": 16, + "pending_eta": 17, + "build_os": 18, + "who": 19, + "failure_classification_id": 20, + "job_type_symbol": 21, + "reason": 22, + "job_group_description": 23, + "job_coalesced_to_guid": 24, + "machine_platform_os": 25, + "start_timestamp": 26, + "build_architecture": 27, + "build_platform_id": 28, + "resource_uri": 29, + "option_collection_hash": 30 +} + +# This list can maps the array indexes to the +# corresponding property names +JOB_PROPERTY_RETURN_KEY = [None]*len(JOB_PROPERTIES) +for k, v in JOB_PROPERTIES.iteritems(): + JOB_PROPERTY_RETURN_KEY[v] = k class UrlQueryFilter(object): """ @@ -213,3 +257,39 @@ def to_timestamp(datestr): datestr, "%Y-%m-%d" ).timetuple()) + +def get_job_value_list(job, platform_option, project, debug): + + if debug: + # If debug is specified return a dictionary for each + # job where the key is the full property name + job_values = {} + else: + # By default don't return all of the job property names + # with each job to reduce the size of the data structure + # returned + job_values = [None]*len(JOB_PROPERTIES) + + for p in JOB_PROPERTIES: + + key = JOB_PROPERTIES[p] + if debug: + key = p + + if p == "id": + job_values[key] = job["job_id"] + elif p == "platform_option": + job_values[key] = platform_option + elif p == "resource_uri": + job_values[key] = reverse( + "jobs-detail", + kwargs={"project": project, "pk": job["job_id"]} + ) + else: + job_values[key] = job.get(p, None) + + return job_values + + + +