Merge pull request #292 from mozilla/server-hidden-jobs

Bug 1074851 - Implement server side job filtering

treeherder/model/derived/jobs.py

@@ -17,7 +17,8 @@
 from django.core.cache import cache
 
 from treeherder.model.models import (Datasource,
-                                     ReferenceDataSignatures)
+                                     ReferenceDataSignatures,
+                                     ExclusionProfile)
 
 from treeherder.model import utils
 
@@ -64,6 +65,13 @@ class JobsModel(TreeherderModelBase):
     INCOMPLETE_STATES = ["running", "pending"]
     STATES = INCOMPLETE_STATES + ["completed", "coalesced"]
 
+    # based on an exclusion profile, which jobs to return
+    EXCLUSION_STATES = [
+        'excluded', # only jobs that are excluded
+        'included', # only jobs that are NOT excluded
+        'all' # both included and excluded jobs
+    ]
+
     # indexes of specific items in the ``job_placeholder`` objects
     JOB_PH_JOB_GUID = 0
     JOB_PH_COALESCED_TO_GUID = 2
@@ -1109,7 +1117,11 @@ def get_result_set_details(self, result_set_ids):
 
         return aggregate_details
 
-    def get_result_set_job_list(self, result_set_ids, full=True, **kwargs):
+    def get_result_set_job_list(self,
+                                result_set_ids,
+                                full=True,
+                                exclusion_state='included',
+                                **kwargs):
         """
         Retrieve a list of ``jobs`` and results for a result_set.
 
@@ -1131,6 +1143,22 @@ def get_result_set_job_list(self, result_set_ids, full=True, **kwargs):
         if "job_type_name" in kwargs:
             repl.append(" AND jt.`name` = '{0}'".format(kwargs["job_type_name"]))
 
+        if exclusion_state != 'all':
+            def_excl = ExclusionProfile.objects.filter(is_default=True)
+            if len(def_excl):
+                try:
+                    signatures = def_excl[0].flat_exclusion[self.project]
+                    # NOT IN if it's 'included' so we don't see excluded jobs
+                    # just IN if it's 'excluded' so we ONLY see excluded jobs
+                    negation = "NOT " if exclusion_state == 'included' else ''
+                    repl.append(" AND j.signature {0}IN ('{1}')".format(
+                                negation,
+                                "', '".join(signatures)
+                                ))
+                except KeyError:
+                    # this repo/project has no hidden signatures
+                    pass
+
         if full:
             proc = "jobs.selects.get_result_set_job_list_full"
         else:
@@ -1148,7 +1176,7 @@ def get_result_set_job_list(self, result_set_ids, full=True, **kwargs):
             signatures.add(job['signature'])
 
         reference_signature_names = self.refdata_model.get_reference_data_signature_names(
-            list(signatures));
+            list(signatures))
 
         return {
             'job_list':data,
@@ -2077,7 +2105,7 @@ def get_signatures_from_properties(self, props):
             ' AND '.join(['COALESCE(SUM(`property`=%s AND `value`=%s), 0) > 0']
                             * len(props))]
 
-        # convert to 1 dimentional list
+        # convert to 1 dimensional list
         props = [el for x in props.items() for el in x]
         props.extend(props)
 

treeherder/model/models.py

@@ -16,7 +16,7 @@
 from django.conf import settings
 from django.core.cache import cache
 from django.db import models
-from django.db.models import Max
+from django.db.models import Max, Q
 from django.contrib.auth.models import User
 from warnings import filterwarnings, resetwarnings
 
@@ -579,27 +579,62 @@ def save(self, *args, **kwargs):
 
         # update the old default profile
         if self.is_default:
-            ExclusionProfile.objects.filter(is_default=True).exclude(id=self.id).update(is_default=False)
+            ExclusionProfile.objects.filter(is_default=True).exclude(
+                id=self.id).update(is_default=False)
 
     def update_flat_exclusions(self):
-        # prepare the nested defaultdict structure for the flat exclusions
-        # options should be stored in a set but sets are not serializable.
-        # using a list instead
-        job_types_constructor = lambda: defaultdict(dict)
-        platform_constructor = lambda: defaultdict(job_types_constructor)
-        flat_exclusions = defaultdict(platform_constructor)
-
+        oc_lookup = OptionCollection.objects.all().select_related(
+            "options__name")
+
+        def get_oc_hash(option):
+            return oc_lookup.get(option__name=option).option_collection_hash
+
+        # this is necessary because the ``job_types`` come back in the form of
+        # ``Mochitest (18)`` or ``Reftest IPC (Ripc)`` so we must split these
+        # back out.
+        # same deal for ``platforms``
+        # todo: update if/when chunking policy changes
+        # when we change chunking, we will likely only get back the name,
+        # so we'll just compare that to the ``job_type_name`` field.
+        def split_combo(combos):
+            list1 = []
+            list2 = []
+            for combo in combos:
+                first, sep, second = combo.rpartition(' (')
+                list1.append(first)
+                list2.append(second.rstrip(')'))
+            return list1, list2
+
+        query = None
         for exclusion in self.exclusions.all().select_related("info"):
-            # create a set of combinations for each property in the exclusion
-            combo = tuple(itertools.product(exclusion.info['repos'], exclusion.info['platforms'],
-                                            exclusion.info['job_types'], exclusion.info['options']))
-            for repo, platform, job_type, option in combo:
-                flat_exclusions[repo][platform][job_type][option] = 1
-
-        if cmp(self.flat_exclusion, flat_exclusions) != 0:
-            self.flat_exclusion = flat_exclusions
-            super(ExclusionProfile, self).save(force_insert=False, force_update=True)
-
+            info = exclusion.info
+            option_collection_hashes = map(get_oc_hash, info['options'])
+            job_type_names, job_type_symbols = split_combo(info['job_types'])
+            platform_names, platform_arch = split_combo(info['platforms'])
+            new_query = Q(repository__in=info['repos'],
+                          machine_platform__in=platform_names,
+                          job_type_name__in=job_type_names,
+                          job_type_symbol__in=job_type_symbols,
+                          option_collection_hash__in=option_collection_hashes)
+            query = (query | new_query) if query else new_query
+
+        self.flat_exclusion = {}
+
+        if query:
+            signatures = ReferenceDataSignatures.objects.filter(query).values_list(
+                'repository', 'signature')
+
+            self.flat_exclusion = defaultdict(list)
+
+            # group the signatures by repo, so the queries don't have to be
+            # so long when getting jobs
+            for repo,sig in signatures:
+                self.flat_exclusion[repo].append(sig)
+
+        super(ExclusionProfile, self).save(
+            force_insert=False,
+            force_update=True
+            )
 
     class Meta:
         db_table = 'exclusion_profile'

treeherder/model/sql/jobs.json

@@ -849,6 +849,7 @@
                     ON jt.`job_group_id` = jg.id
                   WHERE j.`result_set_id` IN (REP1)
                   REP2
+                  REP3
                   ",
             "host": "read_host"
         },
@@ -897,6 +898,7 @@
                     ON jt.`job_group_id` = jg.id
                   WHERE j.`result_set_id` IN (REP1)
                   REP2
+                  REP3
                   ",
             "host": "read_host"
         },

treeherder/webapp/api/resultset.py

@@ -178,17 +178,27 @@ def revisions(self, request, project, jm, pk=None):
     def get_resultset_jobs(self, request, project, jm, pk=None):
 
         result_set_ids = request.QUERY_PARAMS.getlist('result_set_ids') or []
+        exclusion_state = request.QUERY_PARAMS.get('exclusion_state', 'included')
+
+        # default to only included, if an invalid value passed in
+        if exclusion_state not in jm.EXCLUSION_STATES:
+            exclusion_state = 'included'
+
         debug = request.QUERY_PARAMS.get('debug', None)
 
         filter_params = request.QUERY_PARAMS.copy()
+        if 'result_set_ids' in filter_params:
+            del filter_params['result_set_ids']
+        if 'exclusion_state' in filter_params:
+            del filter_params['exclusion_state']
 
         # adapt the result_set_ids to the get_result_set_list
         # return structure
         objs = []
         map(lambda r_id:objs.append({'id':int(r_id)}), result_set_ids)
 
         results = self.get_resultsets_with_jobs(
-                jm, objs, True, filter_params, debug, 'id')
+                jm, objs, True, filter_params, debug, exclusion_state, 'id')
 
         meta = {}
         meta['count'] = len(results)
@@ -203,12 +213,10 @@ def get_resultset_jobs(self, request, project, jm, pk=None):
 
     @staticmethod
     def get_resultsets_with_jobs(
-        jm, rs_list, full, filter_kwargs, debug, sort_key='push_timestamp'):
+        jm, rs_list, full, filter_kwargs, debug, exclusion_state='included',
+        sort_key='push_timestamp'):
         """Convert db result of resultsets in a list to JSON"""
 
-        if 'result_set_ids' in filter_kwargs:
-            del filter_kwargs['result_set_ids']
-
         rs_map = {}
         for rs in rs_list:
             rs_map[rs["id"]] = rs
@@ -219,6 +227,7 @@ def get_resultsets_with_jobs(
         job_list = jm.get_result_set_job_list(
             rs_map.keys(),
             full,
+            exclusion_state,
             **filter_kwargs
         )