forked from mozilla/mozilla_ci_tools
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request mozilla#308 from armenzg/taskcluster_retrigger
Bug 1149738 - Add the ability to retrigger TaskCluster tasks + Log messages from all loggers instead of just mozci messages. r=chmanchester
- Loading branch information
Showing
11 changed files
with
201 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
""" | ||
This module allow us to interact with the various scheduling systems | ||
in a very generic manner. | ||
Defined in here: | ||
* BaseSchedulingClient | ||
* TaskclusterSchedulingClient | ||
""" | ||
from __future__ import absolute_import | ||
from abc import ABCMeta, abstractmethod | ||
|
||
from mozci.sources import tc | ||
|
||
|
||
class BaseSchedulingClient: | ||
""" Base class for common scheduling methods. """ | ||
|
||
__metaclass__ = ABCMeta | ||
|
||
@abstractmethod | ||
def retrigger(self, uuid, **kwargs): | ||
pass | ||
|
||
|
||
class TaskclusterSchedulingClient(BaseSchedulingClient): | ||
|
||
def retrigger(self, uuid, **kwargs): | ||
return tc.retrigger_task(task_id=uuid, **kwargs) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
''' | ||
taskcluster_retrigger.py allows you to retrigger a task from TaskCluster | ||
past its deadline. | ||
''' | ||
import logging | ||
|
||
from argparse import ArgumentParser | ||
|
||
from mozci.scheduling import TaskclusterSchedulingClient | ||
from mozci.utils.misc import setup_logging | ||
|
||
|
||
def main(): | ||
parser = ArgumentParser() | ||
parser.add_argument("--debug", | ||
action="store_true", | ||
dest="debug", | ||
help="set debug for logging.") | ||
|
||
parser.add_argument("--dry-run", | ||
action="store_true", | ||
dest="dry_run", | ||
help="Dry run. No real actions are taken.") | ||
|
||
parser.add_argument('task_ids', | ||
metavar='task_id', | ||
type=str, | ||
nargs='+', | ||
help='Task IDs to work with.') | ||
|
||
options = parser.parse_args() | ||
|
||
if options.debug: | ||
LOG = setup_logging(logging.DEBUG) | ||
else: | ||
LOG = setup_logging() | ||
|
||
sch = TaskclusterSchedulingClient() | ||
for t_id in options.task_ids: | ||
ret_code = sch.retrigger(uuid=t_id, dry_run=options.dry_run) | ||
if ret_code < 0: | ||
LOG.warning("We could not retrigger task %s" % t_id) | ||
|
||
if __name__ == "__main__": | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
""" | ||
This module allow us to interact with taskcluster through the taskcluster | ||
client. | ||
""" | ||
import datetime | ||
import json | ||
import logging | ||
import traceback | ||
|
||
import taskcluster as taskcluster_client | ||
|
||
LOG = logging.getLogger('mozci') | ||
TASKCLUSTER_TOOLS_HOST = 'https://tools.taskcluster.net' | ||
|
||
|
||
def retrigger_task(task_id, dry_run=False): | ||
""" Given a task id (our uuid) we query it and build | ||
a new task based on the old one which we schedule on TaskCluster. | ||
We don't call the rerun API since we can't rerun a task past | ||
its deadline, instead we create a new task with a new taskGroupId, | ||
expiration, creation and deadline values. | ||
task_id (int) - ID that identifies a task on Taskcluster | ||
dry_run (bool) - Default to False. If True, it won't trigger | ||
a task. | ||
returns - 0 for dry_run case, -1 for any failure or the task id (int) | ||
of a succesful retriggered task. | ||
http://docs.taskcluster.net/queue/api-docs/#createTask | ||
""" | ||
one_year = 365 | ||
new_task_id = 0 | ||
|
||
try: | ||
queue = taskcluster_client.Queue() | ||
task = queue.task(task_id) | ||
|
||
LOG.debug("Original task: (Limit 1024 char)") | ||
LOG.debug(str(json.dumps(task))[:1024]) | ||
new_task_id = taskcluster_client.slugId() | ||
|
||
artifacts = task['payload'].get('artifacts', {}) | ||
for artifact, definition in artifacts.iteritems(): | ||
definition['expires'] = taskcluster_client.fromNow('%s days' % one_year) | ||
|
||
# The task group will be identified by the ID of the only | ||
# task in the group | ||
task['taskGroupId'] = new_task_id | ||
# https://bugzilla.mozilla.org/show_bug.cgi?id=1190660 | ||
# TC workers create public logs which are 365 days; if the task expiration | ||
# date is the same or less than that we won't have logs for the task | ||
task['expires'] = taskcluster_client.fromNow('%s days' % (one_year + 1)) | ||
now = datetime.datetime.utcnow() | ||
tomorrow = now + datetime.timedelta(hours=24) | ||
task['created'] = taskcluster_client.stringDate(now) | ||
task['deadline'] = taskcluster_client.stringDate(tomorrow) | ||
|
||
LOG.debug("Contents of new task: (Limit 1024 char)") | ||
LOG.debug(str(task)[:1024]) | ||
|
||
if not dry_run: | ||
LOG.info("Attempting to schedule new task with task_id: {}".format(new_task_id)) | ||
result = queue.createTask(new_task_id, task) | ||
LOG.debug(json.dumps(result)) | ||
LOG.info("{}/task-inspector/#{}".format(TASKCLUSTER_TOOLS_HOST, new_task_id)) | ||
else: | ||
LOG.info("Dry-run mode: Nothing was retriggered.") | ||
|
||
except taskcluster_client.exceptions.TaskclusterRestFailure as e: | ||
traceback.print_exc() | ||
new_task_id = -1 | ||
|
||
except taskcluster_client.exceptions.TaskclusterAuthFailure as e: | ||
# Hack until we fix it in the issue | ||
if str(e) == "Authorization Failed": | ||
LOG.error("The taskclaster client that you specified is lacking " | ||
"the right set of scopes.") | ||
LOG.error("Run this same command with --debug and you will see " | ||
"the missing scopes (the output comes from the " | ||
"taskcluster python client)") | ||
elif str(e) == "Authentication Error": | ||
LOG.error("Make sure that you create permanent credentials and you " | ||
"set these environment variables: TASKCLUSTER_CLIENT_ID & " | ||
"TASKCLUSTER_ACCESS_TOKEN") | ||
new_task_id = -1 | ||
|
||
return new_task_id |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters