Skip to content

Commit

Permalink
start using the new workerPoolId format
Browse files Browse the repository at this point in the history 
per
https://github.com/taskcluster/taskcluster-rfcs/blob/master/rfcs/0145-workerpoolid-taskqueueid.md

Fixes mozilla-releng#378
  • Loading branch information
@escapewindow
escapewindow committed Aug 7, 2019
1 parent 77d2cc5 commit 25bde55
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 43 deletions.
87 changes: 55 additions & 32 deletions scriptworker/constants.py
View file
Expand Up @@ -117,12 +117,12 @@
}), ),

# scriptworker identification
"scriptworker_worker_types": (
"balrogworker-v1",
"beetmoverworker-v1",
"pushapk-v1",
"signing-linux-v1",
"treescriptworker-v1"
"scriptworker_worker_pools": (
"scriptworker-prov-v1/balrogworker-v1",
"scriptworker-prov-v1/beetmoverworker-v1",
"scriptworker-prov-v1/pushapk-v1",
"scriptworker-prov-v1/signing-linux-v1",
"scriptworker-prov-v1/treescriptworker-v1"
),
"scriptworker_provisioners": (
"scriptworker-prov-v1",
Expand All @@ -135,34 +135,57 @@
),

# decision task cot
"valid_decision_worker_types": (
"gecko-1-decision",
"gecko-2-decision",
"gecko-3-decision",
# gecko-focus was for mozilla-mobile releases (bug 1455290) for more details.
# TODO: Remove it once not used anymore
"gecko-focus",
"mobile-1-decision",
# We haven't had the need for mobile-2-decision yet
# https://bugzilla.mozilla.org/show_bug.cgi?id=1512631#c6
"mobile-3-decision",
"app-services-1-decision",
"app-services-3-decision",
),
"valid_decision_worker_pools": frozendict({
"by-cot-product": frozendict({
"firefox": (
"aws-provisioner-v1/gecko-1-decision",
"aws-provisioner-v1/gecko-2-decision",
"aws-provisioner-v1/gecko-3-decision",
),
"thunderbird": (
"aws-provisioner-v1/gecko-1-decision",
"aws-provisioner-v1/gecko-2-decision",
"aws-provisioner-v1/gecko-3-decision",
),
"mobile": (
# gecko-focus was for mozilla-mobile releases (bug 1455290) for more details.
# TODO: Remove it once not used anymore
"aws-provisioner-v1/gecko-focus",
"aws-provisioner-v1/mobile-1-decision",
# We haven't had the need for mobile-2-decision yet
# https://bugzilla.mozilla.org/show_bug.cgi?id=1512631#c6
"aws-provisioner-v1/mobile-3-decision",
),
"application-services": (
"aws-provisioner-v1/app-services-1-decision",
"aws-provisioner-v1/app-services-3-decision",
),
}),
}),

# docker-image cot
"valid_docker_image_worker_types": (
"taskcluster-images", # TODO: Remove this image once docker-images is the only valid worker type
"gecko-images", # legacy
"gecko-1-images",
"gecko-2-images",
"gecko-3-images",

"mobile-1-images", # there is no mobile level 2.
"mobile-3-images",
"app-services-1-images",
"app-services-3-images",
),
"valid_docker_image_worker_pools": frozendict({
"by-cot-product": frozendict({
"firefox": (
"aws-provisioner-v1/gecko-1-images",
"aws-provisioner-v1/gecko-2-images",
"aws-provisioner-v1/gecko-3-images",
),
"thunderbird": (
"aws-provisioner-v1/gecko-1-images",
"aws-provisioner-v1/gecko-2-images",
"aws-provisioner-v1/gecko-3-images",
),
"mobile": (
"aws-provisioner-v1/mobile-1-images", # there is no mobile level 2.
"aws-provisioner-v1/mobile-3-images",
),
"application-services": (
"aws-provisioner-v1/app-services-1-images",
"aws-provisioner-v1/app-services-3-images",
),
}),
}),


# for trace_back_to_*_tree. These repos have access to restricted scopes;
Expand Down
16 changes: 8 additions & 8 deletions scriptworker/cot/verify.py
View file
Expand Up @@ -51,7 +51,7 @@
get_branch,
get_triggered_by,
get_task_id,
get_worker_type,
get_worker_pool_id,
is_try_or_pull_request,
is_action,
)
Expand Down Expand Up @@ -343,7 +343,7 @@ def guess_worker_impl(link):
worker_impls.append("docker-worker")
if task['provisionerId'] in link.context.config['scriptworker_provisioners']:
worker_impls.append("scriptworker")
if task['workerType'] in link.context.config['scriptworker_worker_types']:
if get_worker_pool_id(task) in link.context.config['scriptworker_worker_pools']:
worker_impls.append("scriptworker")
if task['payload'].get("mounts") is not None:
worker_impls.append("generic-worker")
Expand Down Expand Up @@ -1656,9 +1656,9 @@ async def verify_parent_task(chain, link):
CoTError: on chain of trust verification error.
"""
worker_type = get_worker_type(link.task)
if worker_type not in chain.context.config['valid_decision_worker_types']:
raise CoTError("{} is not a valid decision workerType!".format(worker_type))
pool = get_worker_pool_id(link.task)
if pool not in chain.context.config['valid_decision_worker_pools']:
raise CoTError("{} is not a valid decision workerPoolId!".format(pool))
if chain is not link:
# make sure all tasks generated from this parent task match the published
# task-graph.json. Not applicable if this link is the ChainOfTrust object,
Expand Down Expand Up @@ -1713,9 +1713,9 @@ async def verify_docker_image_task(chain, link):
"""
errors = []
# workerType
worker_type = get_worker_type(link.task)
if worker_type not in chain.context.config['valid_docker_image_worker_types']:
errors.append("{} is not a valid docker-image workerType!".format(worker_type))
worker_pool = get_worker_pool_id(link.task)
if worker_pool not in chain.context.config['valid_docker_image_worker_pools']:
errors.append("{} is not a valid docker-image workerPoolId!".format(worker_pool))
raise_on_errors(errors)


Expand Down
32 changes: 32 additions & 0 deletions scriptworker/task.py
View file
Expand Up @@ -273,6 +273,38 @@ def get_worker_type(task):
return task['workerType']


# get_provisioner_id {{{1
def get_provisioner_id(task):
"""Given a task dict, return the provisionerId.
Args:
task (dict): the task dict.
Returns:
str: the provisionerId.
"""
return task['provisionerId']


# get_worker_pool_id {{{1
def get_worker_pool_id(task):
"""Given a task dict, return the worker pool id.
This corresponds to `{provisioner_id}/{workerType}`.
Args:
task (dict): the task dict.
Returns:
str: the workerPoolId.
"""
return "{}/{}".format(
get_provisioner_id(task), get_worker_type(task)
)


# get_project {{{1
def get_project(valid_vcs_rules, source_url):
"""Given vcs rules and a source_url, return the project.
Expand Down
12 changes: 9 additions & 3 deletions scriptworker/test/test_cot_verify.py
View file
Expand Up @@ -515,6 +515,9 @@ def test_raise_on_errors(errors, raises):
), (
{'payload': {}, 'provisionerId': 'test-dummy-provisioner', 'workerType': 'test-dummy-myname', 'scopes': ["x"]},
'scriptworker', False
), (
{'payload': {}, 'provisionerId': 'scriptworker-prov-v1', 'workerType': 'signing-linux-v1', 'scopes': ["x"]},
'scriptworker', False
), (
{'payload': {'mounts': [], 'osGroups': []}, 'provisionerId': '', 'workerType': '', 'scopes': []},
'generic-worker', False
Expand Down Expand Up @@ -2080,7 +2083,8 @@ def task_graph(*args, **kwargs):
makedirs(os.path.dirname(path))
touch(path)
chain.links = [parent_link, build_link]
parent_link.task['workerType'] = chain.context.config['valid_decision_worker_types'][0]
parent_link.task['provisionerId'] = chain.context.config['valid_decision_worker_pools'][0].split("/")[0]
parent_link.task['workerType'] = chain.context.config['valid_decision_worker_pools'][0].split("/")[1]
mocker.patch.object(cotverify, 'load_json_or_yaml', new=task_graph)
mocker.patch.object(cotverify, 'verify_parent_task_definition', new=defn_fn)
if raises:
Expand Down Expand Up @@ -2123,7 +2127,8 @@ def task_graph(*args, **kwargs):
@pytest.mark.asyncio
async def test_verify_parent_task_missing_graph(chain, decision_link, build_link, mocker):
chain.links = [decision_link, build_link]
decision_link.task['workerType'] = chain.context.config['valid_decision_worker_types'][0]
decision_link.task['provisionerId'] = chain.context.config['valid_decision_worker_pools'][0].split("/")[0]
decision_link.task['workerType'] = chain.context.config['valid_decision_worker_pools'][0].split("/")[1]
with pytest.raises(CoTError):
await cotverify.verify_parent_task(chain, decision_link)

Expand Down Expand Up @@ -2155,7 +2160,8 @@ async def test_verify_partials_task_noop(chain, build_link):
# verify_docker_image_task {{{1
@pytest.mark.asyncio
async def test_verify_docker_image_task(chain, docker_image_link):
docker_image_link.task['workerType'] = chain.context.config['valid_docker_image_worker_types'][0]
docker_image_link.task['provisionerId'] = chain.context.config['valid_docker_image_worker_pools'][0].split("/")[0]
docker_image_link.task['workerType'] = chain.context.config['valid_docker_image_worker_pools'][0].split("/")[1]
await cotverify.verify_docker_image_task(chain, docker_image_link)


Expand Down

0 comments on commit 25bde55

Please sign in to comment.