Bug 1398386 - Use signatures to set tier on buildbot jobs only

* Makes TaskCluster jobs independent of profiles to set Tier.  We take
  whatever tier we are given by TaskCluster for the job.
* Removes use of exclusion profiles for Buildbot jobs.  Jobs that
  should be Tier-2 and Tier-3 are hard-coded in the Treeherder code
  by their job signature.

tests/etl/test_job_ingestion.py

@@ -4,13 +4,10 @@
 
 from tests import test_utils
 from tests.sample_data_generator import job_data
-from treeherder.etl.jobs import (_get_lower_tier_signatures,
-                                 _remove_existing_jobs,
+from treeherder.etl.jobs import (_remove_existing_jobs,
                                  store_job_data)
 from treeherder.etl.push import store_push_data
-from treeherder.model.models import (ExclusionProfile,
-                                     Job,
-                                     JobExclusion,
+from treeherder.model.models import (Job,
                                      JobLog,
                                      Push)
 
@@ -273,44 +270,30 @@ def test_remove_existing_jobs_one_existing_one_new(test_repository, failure_clas
     assert Job.objects.count() == 1
 
 
-def test_new_job_in_exclusion_profile(test_repository, failure_classifications, sample_data,
-                                      sample_push, mock_log_parser,
-                                      test_sheriff, push_stored):
-    for job in sample_data.job_data[:2]:
-        job["revision"] = push_stored[0]["revision"]
-
-    job = sample_data.job_data[1]
-    platform = job["job"]["machine_platform"]["platform"]
-    arch = job["job"]["machine_platform"]["architecture"]
-
-    job_exclusion = JobExclusion.objects.create(
-        name="jobex",
-        info={
-            "platforms": ["{} ({})".format(platform, arch)],
-            "repos": [test_repository.name],
-            "option_collections": ["opt"],
-            "option_collection_hashes": ["102210fe594ee9b33d82058545b1ed14f4c8206e"],
-            "job_types": ["B2G Emulator Image Build (B)"]},
-        author=test_sheriff,
-    )
-    exclusion_profile = ExclusionProfile.objects.create(
-        name="Tier-2",
-        is_default=False,
-        author=test_sheriff,
-    )
-    exclusion_profile.exclusions.add(job_exclusion)
-    store_job_data(test_repository, sample_data.job_data[:2])
+def test_ingest_buildbot_tier1_job(test_repository, sample_data, sample_push,
+                                   failure_classifications, mock_log_parser):
+    """Tier is set to 1 if no lower_tier_signatures is used (ie: TaskCluster)"""
+    job_data = sample_data.job_data[:1]
+    store_push_data(test_repository, sample_push)
+    store_job_data(test_repository, job_data)
+    job = Job.objects.all().first()
+    assert job.tier == 1
 
-    # We check all the jobs applying the exclusion profile
-    # If we find the excluded job, there is a problem
-    excluded_signatures = ExclusionProfile.objects.get_signatures_for_project(
-        test_repository.name, exclusion_profile.name)
-    obtained = Job.objects.all().exclude(
-        signature__signature__in=excluded_signatures)
-    assert job['job']['job_guid'] not in [ob.guid for ob in obtained]
-    lower_tier_signatures = _get_lower_tier_signatures(test_repository)
-    assert len(lower_tier_signatures) == 1
-    assert lower_tier_signatures[0]['tier'] == 2
+
+def test_ingest_buildbot_tier2_job(test_repository, sample_data, sample_push,
+                                   failure_classifications, mock_log_parser):
+    """Tier is set to 2 if it matches the signature object"""
+    job_data = sample_data.job_data[:1]
+    test_utils.do_job_ingestion(test_repository, job_data, sample_push)
+    job = Job.objects.all().first()
+    lower_tier_signatures = {
+            job.signature.signature: 2
+    }
+    job_data_2 = copy.deepcopy(job_data)
+    job_data_2[0]['job']['job_guid'] = "foo"
+    store_job_data(test_repository, job_data_2, lower_tier_signatures)
+    job2 = Job.objects.get(guid="foo")
+    assert job2.tier == 2
 
 
 def test_ingesting_skip_existing(test_repository, failure_classifications, sample_data,

treeherder/etl/buildapi.py

@@ -21,6 +21,36 @@
     'complete': 'processed_buildapi_complete',
 }
 
+# Signature sets are for
+# autoland
+# mozilla-beta
+# mozilla-central
+# mozilla-esr52
+# mozilla-inbound
+# mozilla-release
+# try
+# and all project repos
+TIER_SIGNATURES = {
+    # mozilla-central
+    '49f148889483d2b918968dc58a3dc128e0cf3bad': 2,
+    'eca4450a7589f585abb6417e2ec50ec9f9222f30': 2,
+    'a639fc8e9c851c23d8757ca4cc0bdf4f47191a8d': 2,
+    'ece1657e1df7a7eccfad512ec42ca95c737c9967': 2,
+    'd5d4966a6291e0d5038feb53706ce94815f2940f': 2,
+    '168c3f4f8b6e86fa6c72a90a7594ca740773754a': 2,
+    '68a10491197ebdb2e1fe7421948602e610756648': 2,
+    '42e9a1ff1501fbd9c5bfc4ee2db3014ef9e03764': 2,
+    'a4cc9dc681d9dee8d2b2e12473bd108e567f9a6e': 2,
+    'd727b862b1e0d15051c2cf70d433b98405b9d24e': 2,
+    '7910e346829fdaf579fba02e7bdd5bdb484b3328': 2,
+    # mozilla-esr52
+    '4226b6b3015e09de57c2d553584eba5057b7425e': 3,
+    '4df5d64f9501123193b0b3cf4d2e9b3c84bc2076': 3,
+    # mozilla-release
+    '285c5e0ad301703887bde886e4fb539e34678692': 3,
+    'effdff3f7ec01e57e53e97cdda9e3bbf0f243e44': 3,
+}
+
 
 class Builds4hTransformerMixin(object):
 
@@ -448,7 +478,8 @@ def store_jobs(job_collections, chunk_size):
                 repository = Repository.objects.get(
                     name=repository_name)
                 collection.validate()
-                store_job_data(repository, collection.get_collection_data())
+                store_job_data(repository, collection.get_collection_data(),
+                               TIER_SIGNATURES)
             except Exception:
                 newrelic.agent.record_exception()
                 errors.append({

treeherder/etl/jobs.py

@@ -12,7 +12,6 @@
                                      store_job_artifacts)
 from treeherder.etl.common import get_guid_root
 from treeherder.model.models import (BuildPlatform,
-                                     ExclusionProfile,
                                      FailureClassification,
                                      Job,
                                      JobDuration,
@@ -28,8 +27,6 @@
                                      ReferenceDataSignatures,
                                      TaskclusterMetadata)
 
-LOWER_TIERS = [2, 3]
-
 logger = logging.getLogger(__name__)
 
 
@@ -40,28 +37,6 @@ def _get_number(s):
         return 0
 
 
-def _get_lower_tier_signatures(repository):
-    # get the lower tier data signatures for this project.
-    # if there are none, then just return an empty list
-    # this keeps track of them order (2, then 3) so that the latest
-    # will have precedence.  If a job signature is in both Tier-2 and
-    # Tier-3, then it will end up in Tier-3.
-    lower_tier_signatures = []
-    for tier_num in LOWER_TIERS:
-        try:
-            signatures = ExclusionProfile.objects.get_signatures_for_project(
-                repository.name, "Tier-{}".format(tier_num))
-            lower_tier_signatures.append({
-                'tier': tier_num,
-                'signatures': signatures
-            })
-        except ExclusionProfile.DoesNotExist:
-            # no exclusion profile for this tier
-            pass
-
-    return lower_tier_signatures
-
-
 def _remove_existing_jobs(data):
     """
     Remove jobs from data where we already have them in the same state.
@@ -207,24 +182,15 @@ def _load_job(repository, job_datum, push_id, lower_tier_signatures):
             'option_collection_hash': option_collection_hash
         })
 
-    if created:
-        # A new ReferenceDataSignature has been added, so we need
-        # to reload lower tier exclusions
-        lower_tier_signatures = _get_lower_tier_signatures(repository)
-
     tier = job_datum.get('tier') or 1
-    # job tier signatures override the setting from the job structure
-    # Check the signatures list for any supported lower tiers that have
-    # an active exclusion profile.
 
     result = job_datum.get('result', 'unknown')
 
-    # As stated elsewhere, a job will end up in the lowest tier where its
-    # signature belongs.  So if a signature is in Tier-2 and Tier-3, it
-    # will end up in 3.
-    for tier_info in lower_tier_signatures:
-        if signature_hash in tier_info["signatures"]:
-            tier = tier_info["tier"]
+    # Job tier signatures override the setting from the job structure
+    # Check the signatures list for any supported lower tiers that have
+    # an active exclusion profile.
+    if lower_tier_signatures and signature_hash in lower_tier_signatures:
+        tier = lower_tier_signatures[signature_hash]
 
     try:
         duration = JobDuration.objects.values_list(
@@ -416,7 +382,7 @@ def _schedule_log_parsing(job, job_logs, result):
                            args=[job.id, job_log_ids, priority])
 
 
-def store_job_data(repository, data):
+def store_job_data(repository, data, lower_tier_signatures=None):
     """
     Store job data instances into jobs db
 
@@ -484,8 +450,6 @@ def store_job_data(repository, data):
 
     coalesced_job_guid_placeholders = []
 
-    lower_tier_signatures = _get_lower_tier_signatures(repository)
-
     for datum in data:
         try:
             # TODO: this might be a good place to check the datum against