Minimize job data

tests/webapp/api/test_resultset_api.py

@@ -54,7 +54,7 @@ def test_resultset_list_full_false(webapp, eleven_jobs_processed, jm):
     """
     resp = webapp.get(
         reverse("resultset-list", kwargs={"project": jm.project}),
-                {"full": False}
+                {"full": False, "debug": True}
     )
 
     results = resp.json['results']
@@ -81,7 +81,7 @@ def test_resultset_list_full_false(webapp, eleven_jobs_processed, jm):
 
     assert(meta == {
         u'count': 10,
-        u'filter_params': {u'full': u'False'},
+        u'filter_params': {u'full': u'False', u'debug': u'True'},
         u'repository':
         u'test_treeherder'
     })

treeherder/webapp/api/resultset.py

@@ -7,7 +7,8 @@
 from treeherder.model.derived import DatasetNotFoundError
 from treeherder.webapp.api.utils import (UrlQueryFilter, with_jobs,
                                          oauth_required, get_option,
-                                         to_timestamp)
+                                         to_timestamp, get_job_value_list,
+                                         JOB_PROPERTY_RETURN_KEY)
 
 PLATFORM_ORDER = {
     "linux32": 0,
@@ -60,7 +61,6 @@ def list(self, request, project, jm):
         GET method for list of ``resultset`` records with revisions
 
         """
-
         # make a mutable copy of these params
         filter_params = request.QUERY_PARAMS.copy()
 
@@ -113,7 +113,9 @@ def list(self, request, project, jm):
         )
 
         if with_jobs:
-            results = self.get_resultsets_with_jobs(jm, objs, full, {})
+            debug = request.QUERY_PARAMS.get('debug', None)
+            results = self.get_resultsets_with_jobs(
+                jm, objs, full, {}, debug)
         else:
             results = objs
 
@@ -122,7 +124,8 @@ def list(self, request, project, jm):
 
         return Response({
             'meta': meta,
-            'results': results
+            'results': results,
+            'job_property_names': JOB_PROPERTY_RETURN_KEY
         })
 
     @with_jobs
@@ -136,7 +139,8 @@ def retrieve(self, request, project, jm, pk=None):
 
         objs = jm.get_result_set_list(0, 1, full, filter.conditions)
         if objs:
-            rs = self.get_resultsets_with_jobs(jm, objs, full, {})
+            debug = request.QUERY_PARAMS.get('debug', None)
+            rs = self.get_resultsets_with_jobs(jm, objs, full, {}, debug)
             return Response(rs[0])
         else:
             return Response("No resultset with id: {0}".format(pk), 404)
@@ -151,7 +155,7 @@ def revisions(self, request, project, jm, pk=None):
         return Response(objs)
 
     @staticmethod
-    def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs):
+    def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs, debug):
         """Convert db result of resultsets in a list to JSON"""
 
         # Fetch the job results all at once, then parse them out in memory.
@@ -219,21 +223,21 @@ def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs):
 
                     by_job_type = sorted(list(jg_group), key=job_type_grouper)
 
+                    job_list = []
                     groups.append({
                         "symbol": jg_symbol,
                         "name": by_job_type[0]["job_group_name"],
-                        "jobs": by_job_type
+                        "jobs": job_list
                     })
 
                     # build the uri ref for each job
                     for job in by_job_type:
-                        job["id"] = job["job_id"]
-                        del(job["job_id"])
-                        del(job["option_collection_hash"])
 
-                        job["platform_option"] = platform_option
-                        job["resource_uri"] = reverse("jobs-detail",
-                            kwargs={"project": jm.project, "pk": job["id"]})
+                        job_list.append(
+                            get_job_value_list(
+                                job, platform_option, jm.project, debug
+                            )
+                        )
 
                         if job["state"] == "completed":
                             job_counts[job["result"]] += 1

treeherder/webapp/api/utils.py

@@ -6,10 +6,54 @@
 import oauth2 as oauth
 from django.conf import settings
 from rest_framework.response import Response
+from rest_framework.reverse import reverse
 
 from treeherder.model.derived import JobsModel
 from treeherder.etl.oauth_utils import OAuthCredentials
 
+# To add a new property to the job object returned,
+# where the database column name is identical to
+# the property name, just add the column name to this
+# structure.
+JOB_PROPERTIES = {
+    "submit_timestamp": 0,
+    "machine_name": 1,
+    "job_group_symbol": 2,
+    "job_group_name": 3,
+    "platform_option": 4,
+    "job_type_description": 5,
+    "result_set_id": 6,
+    "result": 7,
+    "id": 8,
+    "machine_platform_architecture": 9,
+    "end_timestamp": 10,
+    "build_platform": 11,
+    "job_guid": 12,
+    "job_type_name": 13,
+    "platform": 14,
+    "state": 15,
+    "running_eta": 16,
+    "pending_eta": 17,
+    "build_os": 18,
+    "who": 19,
+    "failure_classification_id": 20,
+    "job_type_symbol": 21,
+    "reason": 22,
+    "job_group_description": 23,
+    "job_coalesced_to_guid": 24,
+    "machine_platform_os": 25,
+    "start_timestamp": 26,
+    "build_architecture": 27,
+    "build_platform_id": 28,
+    "resource_uri": 29,
+    "option_collection_hash": 30
+}
+
+# This list can maps the array indexes to the
+# corresponding property names
+JOB_PROPERTY_RETURN_KEY = [None]*len(JOB_PROPERTIES)
+for k, v in JOB_PROPERTIES.iteritems():
+    JOB_PROPERTY_RETURN_KEY[v] = k
 
 class UrlQueryFilter(object):
     """
@@ -213,3 +257,39 @@ def to_timestamp(datestr):
         datestr,
         "%Y-%m-%d"
         ).timetuple())
+
+def get_job_value_list(job, platform_option, project, debug):
+
+    if debug:
+        # If debug is specified return a dictionary for each
+        # job where the key is the full property name
+        job_values = {}
+    else:
+        # By default don't return all of the job property names
+        # with each job to reduce the size of the data structure
+        # returned
+        job_values = [None]*len(JOB_PROPERTIES)
+
+    for p in JOB_PROPERTIES:
+
+        key = JOB_PROPERTIES[p]
+        if debug:
+            key = p
+
+        if p == "id":
+            job_values[key] = job["job_id"]
+        elif p == "platform_option":
+            job_values[key] = platform_option
+        elif p == "resource_uri":
+            job_values[key] = reverse(
+                "jobs-detail",
+                kwargs={"project": project, "pk": job["job_id"]}
+                )
+        else:
+            job_values[key] = job.get(p, None)
+
+    return job_values
+
+
+
+