Merge branch 'master' of github.com:armenzg/mozilla_ci_tools

mozci/mozci.py

@@ -8,7 +8,11 @@
 
 import logging
 
-from buildapi_client import make_retrigger_request, trigger_arbitrary_job
+from buildapi_client import (
+    BuildapiDown,
+    make_retrigger_request,
+    trigger_arbitrary_job
+)
 
 from mozci import repositories
 from mozci.errors import (
@@ -80,6 +84,7 @@ def set_query_source(query_source="buildapi"):
     """ Function to set the global QUERY_SOURCE """
     global QUERY_SOURCE
     assert query_source in ('buildapi', 'treeherder')
+    LOG.info('Setting {} as our query source'.format(query_source))
     if query_source == "treeherder":
         source_class = TreeherderApi
     else:
@@ -92,9 +97,9 @@ def _unique_build_request(buildername, revision):
     if is_upstream(buildername) and \
        revision in SCHEDULING_MANAGER and \
        buildername in SCHEDULING_MANAGER[revision]:
-        LOG.debug("We have already scheduled the build '%s' for "
-                  "revision %s during this session. We don't allow "
-                  "multiple requests." % (buildername, revision))
+        LOG.info("We have already scheduled the build '%s' for "
+                 "revision %s during this session. We don't allow "
+                 "multiple requests." % (buildername, revision))
         return False
     else:
         return True
@@ -214,6 +219,7 @@ def determine_trigger_objective(revision, buildername, trigger_build_if_missing=
             continue
 
         # Successful or failed jobs may have the files we need
+        # Bug 1314930 - TreeherderApi() cannot always reach for the files
         files = _find_files(job)
 
         if not files or not _all_urls_reachable(files.values()):
@@ -414,7 +420,7 @@ def trigger_job(revision, buildername, times=1, files=None, dry_run=False,
     list_of_requests = []
     repo_url = repositories.query_repo_url(repo_name)
     if len(revision) != 40:
-        LOG.warning('We should not be using revisions less than 40 chars ({}).'.format(revision))
+        LOG.info('We are going to convert the revision into 40 chars ({}).'.format(revision))
         push_info = query_push_by_revision(repo_url, revision)
         revision = push_info.changesets[0].node
         assert len(revision) == 40, 'This should have been a 40 char revision.'
@@ -423,7 +429,6 @@ def trigger_job(revision, buildername, times=1, files=None, dry_run=False,
         return list_of_requests
 
     LOG.info("==> We want to trigger '%s' a total of %d time(s)." % (buildername, times))
-    LOG.info("")  # Extra line to help visual of logs
 
     if VALIDATE and not valid_builder(buildername):
         LOG.error("The builder %s requested is invalid" % buildername)
@@ -612,6 +617,9 @@ def trigger_talos_jobs_for_build(buildername, revision, times, dry_run=False):
                 times=times,
                 dry_run=dry_run
             )
+        except BuildapiDown:
+            LOG.exception('Buildapi is down. We will not try anymore.')
+            return FAILURE
         except:
             LOG.exception('We failed to trigger {}; Let us try the rest.'.format(buildername))
             failed_builders += '%s\n' % buildername
@@ -634,15 +642,18 @@ def trigger_all_talos_jobs(repo_name, revision, times, priority=0, dry_run=False
         pgo = True
     buildernames = build_talos_buildernames_for_repo(repo_name, pgo)
     for buildername in buildernames:
-        trigger_range(buildername=buildername,
-                      revisions=[revision],
-                      times=times,
-                      dry_run=dry_run,
-                      extra_properties={'mozci_request': {
-                                        'type': 'trigger_all_talos_jobs',
-                                        'times': times,
-                                        'priority': priority}
-                                        })
+        trigger_job(buildername=buildername,
+                    revision=revision,
+                    times=times,
+                    dry_run=dry_run,
+                    trigger_build_if_missing=True,
+                    extra_properties={
+                        'mozci_request': {
+                            'type': 'trigger_all_talos_jobs',
+                            'times': times,
+                            'priority': priority
+                        }
+                    })
 
 
 def manual_backfill(revision, buildername, dry_run=False):

scripts/trigger.py

@@ -13,7 +13,9 @@
     query_builders,
     query_repo_name_from_buildername,
     query_repo_url_from_buildername,
-    set_query_source
+    set_query_source,
+    trigger_all_talos_jobs,
+    trigger_talos_jobs_for_build,
 )
 from mozci.query_jobs import BuildApi, COALESCED, TreeherderApi
 from mozci.repositories import query_repo_url
@@ -28,11 +30,19 @@
 from mozci.platforms import filter_buildernames
 from mozci.query_jobs import WARNING, SUCCESS
 
+ACTIONS = {
+    'trigger-all-talos': {
+        'help': 'This will trigger all talos jobs for a revision. This will also '
+                'trigger the builds that the talos jobs depend on.'
+    },
+}
+
 
 def parse_args(argv=None):
     """Parse command line options."""
     parser = ArgumentParser()
 
+    parser.add_argument('action', help='Available actions: ' + ','.join(ACTIONS.keys()))
     # Required arguments
     parser.add_argument('-b', "--buildername",
                         dest="buildernames",
@@ -162,6 +172,12 @@ def parse_args(argv=None):
                         dest="failed_jobs",
                         help="trigger failed jobs for particular revision")
 
+    # Mode 6: Use --trigger-talos-for-build to trigger talos jobs for a particular build
+    parser.add_argument('--trigger-talos-for-build',
+                        action="store_true",
+                        dest="trigger_talos_for_build",
+                        help="trigger all talos jobs for a particular build")
+
     options = parser.parse_args(argv)
     return options
 
@@ -292,6 +308,11 @@ def main():
     else:
         LOG = setup_logging(logging.INFO)
 
+    if options.action == 'trigger-all-talos':
+        trigger_all_talos_jobs(options.repo_name, options.rev, options.times,
+                               dry_run=options.dry_run)
+        sys.exit(0)
+
     validate_options(options)
     if not options.dry_run and not valid_credentials():
         sys.exit(-1)
@@ -358,7 +379,8 @@ def main():
         return
 
     # Mode #3: Trigger jobs based on revision list modifiers
-    if not (options.includes or options.exclude or options.failed_jobs):
+    if not (options.includes or options.exclude or options.failed_jobs or
+            options.trigger_talos_for_build):
         job_names = options.buildernames
 
     # Mode 4 - Schedule every builder matching --includes and does not match --exclude.
@@ -409,6 +431,15 @@ def main():
             revision=revision,
             status=WARNING)
 
+    elif options.trigger_talos_for_build:
+        trigger_talos_jobs_for_build(
+            buildername=options.buildernames[0],
+            revision=revision,
+            times=2,
+            dry_run=options.dry_run,
+        )
+        exit(0)
+
     for buildername in job_names:
         revlist = determine_revlist(
             repo_url=repo_url,

setup.py

@@ -2,7 +2,7 @@
 
 
 required = [
-    "buildapi_client",
+    "buildapi_client>=0.6.1",
     "ijson",
     "keyring",
     "progressbar",
@@ -16,7 +16,7 @@
 
 setup(
     name='mozci',
-    version='0.49.1.dev0',
+    version='0.50.1.dev0',
     packages=find_packages(),
     install_requires=required + ['pytest-runner'],
     tests_require=required + ['mock', 'pytest'],

test/helpers.py

@@ -34,9 +34,5 @@ def _get_graph_result():
 
 
 ALLTHETHINGS = _get_allthethings()
-
-
 SETA_RESULT = _get_SETA()
-
-
 GRAPH_RESULT = _get_graph_result()