Merge 3da5813 into a81b624

mozci/mozci.py

@@ -15,7 +15,7 @@
 import logging
 
 from mozci.platforms import determine_upstream_builder, is_downstream
-from mozci.sources import allthethings, buildapi, buildjson, pushlog
+from mozci.sources import allthethings, buildapi, buildjson, pushlog, treeherder
 from mozci.utils.misc import _all_urls_reachable
 from mozci.utils.transfer import path_to_file
 
@@ -55,7 +55,7 @@ def _unique_build_request(buildername, revision):
         return True
 
 
-def _status_summary(jobs):
+def _status_summary(jobs, status_provider="buildapi"):
     """Return the number of successful, pending and running jobs."""
     assert type(jobs) == list
     successful = 0
@@ -64,7 +64,10 @@ def _status_summary(jobs):
     coalesced = 0
 
     for job in jobs:
-        status = buildapi.BuildapiJobStatus(job).get_status()
+        if status_provider == "treeherder":
+            status = treeherder.query_job_status(job)
+        else:
+            status = buildapi.BuildapiJobStatus(job).get_status()
         if status == buildapi.PENDING:
             pending += 1
         if status == buildapi.RUNNING:
@@ -367,7 +370,8 @@ def trigger_job(revision, buildername, times=1, files=None, dry_run=False,
     return list_of_requests
 
 
-def trigger_range(buildername, revisions, times=1, dry_run=False, files=None):
+def trigger_range(buildername, revisions, status_provider="buildapi", times=1,
+                  dry_run=False, files=None):
     """Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
     repo_name = query_repo_name_from_buildername(buildername)
     LOG.info("We want to have %s job(s) of %s on revisions %s" %
@@ -384,9 +388,14 @@ def trigger_range(buildername, revisions, times=1, dry_run=False, files=None):
                  (times, buildername, rev))
 
         # 1) How many potentially completed jobs can we get for this buildername?
-        jobs = buildapi.query_jobs_schedule(repo_name, rev)
-        matching_jobs = _matching_jobs(buildername, jobs)
-        successful_jobs, pending_jobs, running_jobs = _status_summary(matching_jobs)[0:3]
+        if status_provider == "treeherder":
+            jobs = treeherder.query_jobs_schedule(repo_name, rev)
+            matching_jobs = treeherder._matching_jobs(buildername, jobs)
+        else:
+            jobs = buildapi.query_jobs_schedule(repo_name, rev)
+            matching_jobs = _matching_jobs(buildername, jobs)
+        successful_jobs, pending_jobs, running_jobs = _status_summary(matching_jobs,
+                                                                      status_provider)[0:3]
 
         potential_jobs = pending_jobs + running_jobs + successful_jobs
         LOG.debug("We found %d pending jobs, %d running jobs and %d successful_jobs." %
@@ -405,6 +414,10 @@ def trigger_range(buildername, revisions, times=1, dry_run=False, files=None):
             # If a job matching what we want already exists, we can
             # use the retrigger API in self-serve to retrigger that
             # instead of creating a new arbitrary job
+            if status_provider == "treeherder":
+                jobs = buildapi.query_jobs_schedule(repo_name, rev)
+                matching_jobs = _matching_jobs(buildername, jobs)
+
             if len(matching_jobs) > 0:
                 request_id = matching_jobs[0]["requests"][0]["request_id"]
                 buildapi.make_retrigger_request(
@@ -450,7 +463,7 @@ def trigger(builder, revision, files=[], dry_run=False, extra_properties=None):
                                           extra_properties)
 
 
-def backfill_revlist(buildername, revisions):
+def backfill_revlist(buildername, revisions, status_provider="buildapi"):
     """
     Find the last known good job for that buildername iterating through the list of revisions.
 
@@ -465,9 +478,14 @@ def backfill_revlist(buildername, revisions):
     LOG.info("We want to find a successful job for '%s' in this range: [%s:%s]" %
              (buildername, revisions[0], revisions[-1]))
     for rev in revisions:
-        jobs = buildapi.query_jobs_schedule(repo_name, rev)
-        matching_jobs = _matching_jobs(buildername, jobs)
-        successful_jobs = _status_summary(matching_jobs)[0]
+        if status_provider == "treeherder":
+            jobs = treeherder.query_jobs_schedule(repo_name, rev)
+            matching_jobs = treeherder._matching_jobs(buildername, jobs)
+        else:
+            jobs = buildapi.query_jobs_schedule(repo_name, rev)
+            matching_jobs = _matching_jobs(buildername, jobs)
+        successful_jobs = _status_summary(matching_jobs, status_provider)[0]
+
         if successful_jobs > 0:
             LOG.info("The last successful job for buildername '%s' is on %s" %
                      (buildername, rev))

mozci/scripts/trigger.py

@@ -63,6 +63,12 @@ def parse_args(argv=None):
                         dest="debug",
                         help="set debug for logging.")
 
+    parser.add_argument("--status-provider",
+                        metavar="[buildapi|treeherder]",
+                        dest="status_provider",
+                        default="buildapi",
+                        help="Get job status from buildapi or treeherder.")
+
     parser.add_argument("--file",
                         action="append",
                         dest="files",
@@ -127,7 +133,7 @@ def sanitize_buildernames(buildernames):
 
 
 def determine_revlist(repo_url, buildername, rev, back_revisions,
-                      delta, from_rev, backfill, skips, max_revisions):
+                      delta, from_rev, backfill, skips, max_revisions, status_provider):
     """Determine which revisions we need to trigger."""
     if back_revisions:
         push_info = query_revision_info(repo_url, rev)
@@ -160,7 +166,7 @@ def determine_revlist(repo_url, buildername, rev, back_revisions,
                                      start_id=start_id,
                                      end_id=end_id)
 
-        revlist = backfill_revlist(buildername, revlist)
+        revlist = backfill_revlist(buildername, revlist, status_provider)
 
     else:
         revlist = [rev]
@@ -197,12 +203,14 @@ def main():
             from_rev=options.from_rev,
             backfill=options.backfill,
             skips=options.skips,
-            max_revisions=options.max_revisions)
+            max_revisions=options.max_revisions,
+            status_provider=options.status_provider)
 
         try:
             trigger_range(
                 buildername=buildername,
                 revisions=revlist,
+                status_provider=options.status_provider,
                 times=options.times,
                 dry_run=options.dry_run,
                 files=options.files

mozci/scripts/triggerbyfilters.py

@@ -56,6 +56,12 @@ def parse_args(argv=None):
                         dest="debug",
                         help="set debug for logging.")
 
+    parser.add_argument("--status-provider",
+                        metavar="[buildapi|treeherder]",
+                        dest="status_provider",
+                        default="buildapi",
+                        help="Get job status from buildapi or treeherder.")
+
     options = parser.parse_args(argv)
     return options
 
@@ -93,6 +99,7 @@ def main():
         trigger_range(
             buildername=buildername,
             revisions=[options.rev],
+            status_provider=options.status_provider,
             times=options.times,
             dry_run=options.dry_run,
         )

mozci/sources/treeherder.py

@@ -0,0 +1,63 @@
+import logging
+
+from thclient import TreeherderClient
+
+
+LOG = logging.getLogger('mozci')
+PENDING, RUNNING, COALESCED, UNKNOWN = range(-4, 0)
+SUCCESS, WARNING, FAILURE, SKIPPED, EXCEPTION, RETRY, CANCELLED = range(7)
+
+
+def query_jobs_schedule(repo_name, revision):
+    """
+    Return all jobs for a given revision
+    """
+    treeherder_client = TreeherderClient()
+    results = treeherder_client.get_resultsets(repo_name, revision=revision)
+    revision_id = results[0]["id"]
+    all_jobs = treeherder_client.get_jobs(repo_name, count=2000, result_set_id=revision_id)
+    return all_jobs
+
+
+def _matching_jobs(buildername, all_jobs):
+    """
+    Return all jobs that matched the criteria.
+    """
+    matching_jobs = []
+    for j in all_jobs:
+        if j["ref_data_name"] == buildername:
+            matching_jobs.append(j)
+
+    return matching_jobs
+
+
+def query_job_status(job):
+    """
+    Helper to determine the scheduling status of a job from treeherder.
+    """
+    if job["result"] == "unknown":
+        if job["state"] == "pending":
+            return PENDING
+        elif job["state"] == "running":
+            return RUNNING
+        else:
+            return UNKNOWN
+
+    if job["job_coalesced_to_guid"] is not None:
+        return COALESCED
+
+    if job["state"] == "completed":
+        if job["result"] == "success":
+            return SUCCESS
+        elif job["result"] == "busted" or job["result"] == "testfailed":
+            return FAILURE
+        elif job["result"] == "skipped":
+            return SKIPPED
+        elif job["result"] == "exception":
+            return EXCEPTION
+        elif job["result"] == "retry":
+            return RETRY
+        elif job["result"] == "usercancel":
+            return CANCELLED
+
+    raise Exception("Unexpected status")

setup.py

@@ -16,7 +16,8 @@
         'bugsy>=0.4.0',
         'progressbar>=2.3',
         'requests>=2.5.1',
-        'keyring>=5.3'
+        'keyring>=5.3',
+        'treeherder-client>=1.3'
     ],
 
     # Meta-data for upload to PyPI