Issue 339 - Schedule task graphs without auxiliary commands + dry_run…

… + single builder

ci_manager.py
-------------
* Rename TaskclusterSchedulingClient to TaskClusterBuildbotManager
* Move graph scheduling logic to the tc module
* Support passing along *args and **kwargs (for dry_run support)
* TaskClusterBuildbotManager can now schedule a single builder.

buildbot_to_taskcluster.py
--------------------------
* Shorter help text on one of the options
* Pass the dry_run value
* We can now schedule single builders

buildbot_bridge.py
------------------
* generate_graph_from_builders() can now receive *args and **kwargs
* Added scheduler:create-task-graph scope to the task_graph template
* Added schedule_graph() method which calls the tc client's scheduler

authentication.py
-----------------
* Use path_to_file to have the credentials on the same spot as every other auxiliary file
* Store password as 'ldap' instead of 'mozci' in the keyring

mozci/ci_manager.py

@@ -77,11 +77,8 @@ def cancel_all(self, repo_name, revision, *args, **kwargs):
 
 class TaskclusterManager(BaseCIManager):
 
-    def schedule_graph(self, task_graph):
-        # XXX: We should call the tc client
-        # scheduler = taskcluster.Scheduler(options)
-        # scheduler.createTaskGraph(taskGraphId, payload) -> result
-        return task_graph
+    def schedule_graph(self, task_graph, *args, **kwargs):
+        return tc.schedule_graph(task_graph, *args, **kwargs)
 
     def schedule_arbitrary_job(self, repo_name, revision, uuid, *args, **kwargs):
         pass
@@ -104,8 +101,8 @@ class TaskClusterBuildbotManager(TaskclusterManager):
     def schedule_graph(self, repo_name, revision, builders_graph, *args, **kwargs):
         """ It schedules a task graph for buildbot jobs through TaskCluster.
 
-        Given a graph of builders a TaskCluster graph will be generated which
-        the Buildbot bridge will use to schedule Buildbot jobs.
+        Given a graph of Buildbot builders a TaskCluster graph will be generated
+        which the Buildbot bridge will use to schedule Buildbot jobs.
 
         NOTE: All builders in the graph must contain the same repo_name.
         NOTE: The revision must be a valid one for the implied repo_name from
@@ -124,21 +121,22 @@ def schedule_graph(self, repo_name, revision, builders_graph, *args, **kwargs):
         :rtype: dict
 
         """
-        graph = buildbot_bridge.generate_task_graph(
+        task_graph = buildbot_bridge.generate_task_graph(
             repo_name=repo_name,
             revision=revision,
             builders_graph=builders_graph,
-            *args,
-            **kwargs
         )
-        return super(TaskClusterBuildbotManager, self).schedule_graph(graph)
+        return super(TaskClusterBuildbotManager, self).schedule_graph(
+            task_graph=task_graph, *args, **kwargs)
 
     def schedule_arbitrary_job(self, repo_name, revision, uuid, *args, **kwargs):
-        task = buildbot_bridge.schedule_arbitrary_builder(
+        task_graph = buildbot_bridge.generate_graph_from_builder(
+            repo_name=repo_name,
             revision=revision,
             buildername=uuid,
             *args, **kwargs
         )
-        return super(TaskClusterBuildbotManager, self).schedule_arbitrary_task(task)
+        return super(TaskClusterBuildbotManager, self).schedule_graph(
+            task_graph=task_graph, *args, **kwargs)
 
 # End of TaskClusterBuildbotManager

mozci/scripts/misc/buildbot_to_taskcluster.py

@@ -35,8 +35,16 @@ def main():
                         type=str,
                         help="12-char representing a push.")
 
-    parser.add_argument('builders_graph',
-                        metavar='builders_graph',
+    parser.add_argument("-b", "--builder",
+                        action="store",
+                        dest="builder",
+                        type=str,
+                        help="Use this if you just want to schedule one builder instead "
+                        "of a graph.")
+
+    parser.add_argument("-g", "--graph",
+                        action="store",
+                        dest="builders_graph",
                         help='Graph of builders in the form of: '
                              'dict(builder: [dep_builders].')
 
@@ -48,13 +56,20 @@ def main():
         setup_logging()
 
     mgr = TaskClusterBuildbotManager()
-    # XXX: test what happens when we have a bad graph
-    mgr.schedule_graph(
-        repo_name=options.repo_name,
-        revision=options.revision,
-        builders_graph=ast.literal_eval(options.builders_graph),
-        #  dry_run=options.dry_run
-    )
+    if options.builder:
+        mgr.schedule_arbitrary_job(
+            repo_name=options.repo_name,
+            revision=options.revision,
+            uuid=options.builder,
+            dry_run=options.dry_run
+        )
+    else:
+        mgr.schedule_graph(
+            repo_name=options.repo_name,
+            revision=options.revision,
+            builders_graph=ast.literal_eval(options.builders_graph),
+            dry_run=options.dry_run
+        )
 
 if __name__ == "__main__":
     main()

mozci/scripts/misc/taskcluster_retrigger.py

@@ -6,7 +6,7 @@
 
 from argparse import ArgumentParser
 
-from mozci.scheduling import TaskclusterSchedulingClient
+from mozci.ci_manager import TaskClusterBuildbotManager
 from mozci.utils.misc import setup_logging
 
 
@@ -35,7 +35,7 @@ def main():
     else:
         LOG = setup_logging()
 
-    sch = TaskclusterSchedulingClient()
+    sch = TaskClusterBuildbotManager()
     for t_id in options.task_ids:
         ret_code = sch.retrigger(uuid=t_id, dry_run=options.dry_run)
         if ret_code < 0:

mozci/sources/buildbot_bridge.py

@@ -103,6 +103,23 @@ def _validate_builders_graph(repo_name, builders_graph):
     return result
 
 
+def generate_graph_from_builder(repo_name, revision, buildername, *args, **kwargs):
+    """Return TaskCluster graph based on buildername.
+
+    :param repo_name The name of a repository e.g. mozilla-inbound, alder et al.
+    :type repo_name: str
+    :param revision: push revision
+    :type revision: str
+    :param buildername: Buildbot buildername
+    :type revision: str
+
+    :returns: return None or a valid taskcluster task graph.
+    :rtype: dict
+
+    """
+    return generate_task_graph(repo_name, revision, {buildername: []})
+
+
 def generate_task_graph(repo_name, revision, builders_graph):
     """Return TaskCluster graph based on builders_graph.
 
@@ -129,6 +146,7 @@ def generate_task_graph(repo_name, revision, builders_graph):
     task_graph = {
         'scopes': [
             'queue:define-task:buildbot-bridge/buildbot-bridge',
+            'scheduler:create-task-graph',
         ],
         'tasks': [],
         'metadata': dict(metadata.items() + {'name': 'task graph local'}.items())

mozci/sources/tc.py

@@ -87,3 +87,24 @@ def retrigger_task(task_id, dry_run=False):
         new_task_id = -1
 
     return new_task_id
+
+
+def schedule_graph(task_graph, dry_run=False):
+    """ It schedules a TaskCluster graph and returns its id.
+
+    :param task_graph: It is a TaskCluster graph as defined in here:
+        http://docs.taskcluster.net/scheduler/api-docs/#createTaskGraph
+    :type task_graph: json
+    :param dry_run: It does not schedule the graph
+    :type dry_run: bool
+    :returns: task graph id.
+    :rtype: int
+
+    """
+    graph_id = taskcluster_client.slugId()
+    scheduler = taskcluster_client.Scheduler()
+    if dry_run:
+        LOG.info("DRY-RUN: We have not scheduled the graph.")
+    else:
+        scheduler.createTaskGraph(graph_id, task_graph)
+    return graph_id

mozci/utils/authentication.py

@@ -11,7 +11,9 @@
 import logging
 import os
 
-CREDENTIALS_PATH = os.path.expanduser("~/.mozilla/credentials.cfg")
+from mozci.utils.transfer import path_to_file
+
+CREDENTIALS_PATH = path_to_file("credentials.cfg")
 DIRNAME = os.path.dirname(CREDENTIALS_PATH)
 LOG = logging.getLogger('mozci')
 AUTH = None
@@ -50,7 +52,7 @@ def get_credentials():
     if content is not None:
         LOG.debug("Loading LDAP user from %s" % CREDENTIALS_PATH)
         https_username = content[0].strip()
-        https_password = keyring.get_password("mozci", https_username)
+        https_password = keyring.get_password("ldap", https_username)
         if https_password is None or https_password == "":
             https_password = getpass.getpass(
                 "Input LDAP password for user %s: " % https_username)