Merge pull request mozilla-releng#380 from escapewindow/worker-pool-id

Worker pool
escapewindow · Aug 7, 2019 · 8740487 · 8740487
2 parents da3f572 + 62123eb
commit 8740487
Show file tree

Hide file tree

Showing 12 changed files with 140 additions and 63 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,22 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [24.0.0] - 2019-08-07
+### Added
+- Added `scripts/pin.sh` and `scripts/pin-helper.sh`
+- Added `scriptworker_worker_pools`, `valid_decision_worker_pools`, and `valid_docker_image_pools`
+- Added `get_worker_pool_id` and `get_provisioner_id`
+
+### Changed
+- We now pin dependencies via `scripts/pin.sh`
+- Our scriptworker, decision, and docker-image workerType allowlisting now goes by worker-pool-id, constrained by `cot_product`
+- Our integration tasks use workerTypes that follow the new workerType name restrictions.
+
+### Removed
+- Removed `scripts/pip` and `scripts/Dockerfile` in favor of the new `pin.sh`
+- Removed `scriptworker_worker_types`, `valid_decision_worker_types`, and `valid_docker_image_types`
+- Removed `taskcluster-images` as a valid docker-image workerType
+
 ## [23.6.2] - 2019-07-26
 
 ### Added

diff --git a/requirements/base.py36.txt b/requirements/base.py36.txt
@@ -191,10 +191,10 @@ taskcluster-urls==11.0.0 \
     --hash=sha256:2aceab7cf5b1948bc197f2e5e50c371aa48181ccd490b8bada00f1e3baf0c5cc \
     --hash=sha256:74bd2110b5daaebcec5e1d287bf137b61cb8cf6b2d8f5f2b74183e32bc4e7c87 \
     # via taskcluster
-taskcluster==15.0.0 \
-    --hash=sha256:3d5f080cf6c27772983dfa3945ad558d1f7d35f5bcdd483cfd2b704aa2180f85 \
-    --hash=sha256:4e4a0348cbe29bf54a7354e67968bc0d8a00e2376d9be37c82b6d0095d9d9341 \
-    --hash=sha256:78d30dc38526ce2cce051b0acecc40021130f8a4d4829cc1a11836ca40a48cd9
+taskcluster==16.0.0 \
+    --hash=sha256:2846eb270a9c563ff2e0fbfc69999bbcc4b9e33979206d5433fd1eeba11f6785 \
+    --hash=sha256:8ac7b25c28138b4c1b01614e272f5558ff2af79d4e01430c32ca407838923467 \
+    --hash=sha256:f2ae5cd1f89f6ca3ab869b71b35158e762decc904f5bf483a4ea4843f75a7ef5
 typing-extensions==3.7.4 \
     --hash=sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95 \
     --hash=sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87 \

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -188,10 +188,10 @@ taskcluster-urls==11.0.0 \
     --hash=sha256:2aceab7cf5b1948bc197f2e5e50c371aa48181ccd490b8bada00f1e3baf0c5cc \
     --hash=sha256:74bd2110b5daaebcec5e1d287bf137b61cb8cf6b2d8f5f2b74183e32bc4e7c87 \
     # via taskcluster
-taskcluster==15.0.0 \
-    --hash=sha256:3d5f080cf6c27772983dfa3945ad558d1f7d35f5bcdd483cfd2b704aa2180f85 \
-    --hash=sha256:4e4a0348cbe29bf54a7354e67968bc0d8a00e2376d9be37c82b6d0095d9d9341 \
-    --hash=sha256:78d30dc38526ce2cce051b0acecc40021130f8a4d4829cc1a11836ca40a48cd9
+taskcluster==16.0.0 \
+    --hash=sha256:2846eb270a9c563ff2e0fbfc69999bbcc4b9e33979206d5433fd1eeba11f6785 \
+    --hash=sha256:8ac7b25c28138b4c1b01614e272f5558ff2af79d4e01430c32ca407838923467 \
+    --hash=sha256:f2ae5cd1f89f6ca3ab869b71b35158e762decc904f5bf483a4ea4843f75a7ef5
 urllib3==1.25.3 \
     --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \
     --hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232 \

diff --git a/requirements/test.py36.txt b/requirements/test.py36.txt
@@ -58,9 +58,9 @@ filelock==3.0.12 \
     --hash=sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59 \
     --hash=sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836 \
     # via tox
-flake8-docstrings==1.3.0 \
-    --hash=sha256:4e0ce1476b64e6291520e5570cf12b05016dd4e8ae454b8a8a9a48bc5f84e1cd \
-    --hash=sha256:8436396b5ecad51a122a2c99ba26e5b4e623bf6e913b0fea0cb6c2c4050f91eb
+flake8-docstrings==1.3.1 \
+    --hash=sha256:3ad372b641f4c8e70c7465f067aed4ff8bf1e9347fce14f9eb71ed816db36257 \
+    --hash=sha256:d8d72ccd5807c1ab9ff1466cb9bece0c4d94b8669e9bc4f472abc80dbc5d399e
 flake8-polyfill==1.0.2 \
     --hash=sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9 \
     --hash=sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda \

diff --git a/requirements/test.txt b/requirements/test.txt
@@ -58,9 +58,9 @@ filelock==3.0.12 \
     --hash=sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59 \
     --hash=sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836 \
     # via tox
-flake8-docstrings==1.3.0 \
-    --hash=sha256:4e0ce1476b64e6291520e5570cf12b05016dd4e8ae454b8a8a9a48bc5f84e1cd \
-    --hash=sha256:8436396b5ecad51a122a2c99ba26e5b4e623bf6e913b0fea0cb6c2c4050f91eb
+flake8-docstrings==1.3.1 \
+    --hash=sha256:3ad372b641f4c8e70c7465f067aed4ff8bf1e9347fce14f9eb71ed816db36257 \
+    --hash=sha256:d8d72ccd5807c1ab9ff1466cb9bece0c4d94b8669e9bc4f472abc80dbc5d399e
 flake8-polyfill==1.0.2 \
     --hash=sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9 \
     --hash=sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda \

diff --git a/scriptworker/constants.py b/scriptworker/constants.py
@@ -117,12 +117,12 @@
     }), ),
 
     # scriptworker identification
-    "scriptworker_worker_types": (
-        "balrogworker-v1",
-        "beetmoverworker-v1",
-        "pushapk-v1",
-        "signing-linux-v1",
-        "treescriptworker-v1"
+    "scriptworker_worker_pools": (
+        "scriptworker-prov-v1/balrogworker-v1",
+        "scriptworker-prov-v1/beetmoverworker-v1",
+        "scriptworker-prov-v1/pushapk-v1",
+        "scriptworker-prov-v1/signing-linux-v1",
+        "scriptworker-prov-v1/treescriptworker-v1"
     ),
     "scriptworker_provisioners": (
         "scriptworker-prov-v1",
@@ -135,34 +135,57 @@
     ),
 
     # decision task cot
-    "valid_decision_worker_types": (
-        "gecko-1-decision",
-        "gecko-2-decision",
-        "gecko-3-decision",
-        # gecko-focus was for mozilla-mobile releases (bug 1455290) for more details.
-        # TODO: Remove it once not used anymore
-        "gecko-focus",
-        "mobile-1-decision",
-        # We haven't had the need for mobile-2-decision yet
-        # https://bugzilla.mozilla.org/show_bug.cgi?id=1512631#c6
-        "mobile-3-decision",
-        "app-services-1-decision",
-        "app-services-3-decision",
-    ),
+    "valid_decision_worker_pools": frozendict({
+        "by-cot-product": frozendict({
+            "firefox": (
+                "aws-provisioner-v1/gecko-1-decision",
+                "aws-provisioner-v1/gecko-2-decision",
+                "aws-provisioner-v1/gecko-3-decision",
+            ),
+            "thunderbird": (
+                "aws-provisioner-v1/gecko-1-decision",
+                "aws-provisioner-v1/gecko-2-decision",
+                "aws-provisioner-v1/gecko-3-decision",
+            ),
+            "mobile": (
+                # gecko-focus was for mozilla-mobile releases (bug 1455290) for more details.
+                # TODO: Remove it once not used anymore
+                "aws-provisioner-v1/gecko-focus",
+                "aws-provisioner-v1/mobile-1-decision",
+                # We haven't had the need for mobile-2-decision yet
+                # https://bugzilla.mozilla.org/show_bug.cgi?id=1512631#c6
+                "aws-provisioner-v1/mobile-3-decision",
+            ),
+            "application-services": (
+                "aws-provisioner-v1/app-services-1-decision",
+                "aws-provisioner-v1/app-services-3-decision",
+            ),
+        }),
+    }),
 
     # docker-image cot
-    "valid_docker_image_worker_types": (
-        "taskcluster-images",   # TODO: Remove this image once docker-images is the only valid worker type
-        "gecko-images",  # legacy
-        "gecko-1-images",
-        "gecko-2-images",
-        "gecko-3-images",
-
-        "mobile-1-images",  # there is no mobile level 2.
-        "mobile-3-images",
-        "app-services-1-images",
-        "app-services-3-images",
-    ),
+    "valid_docker_image_worker_pools": frozendict({
+        "by-cot-product": frozendict({
+            "firefox": (
+                "aws-provisioner-v1/gecko-1-images",
+                "aws-provisioner-v1/gecko-2-images",
+                "aws-provisioner-v1/gecko-3-images",
+            ),
+            "thunderbird": (
+                "aws-provisioner-v1/gecko-1-images",
+                "aws-provisioner-v1/gecko-2-images",
+                "aws-provisioner-v1/gecko-3-images",
+            ),
+            "mobile": (
+                "aws-provisioner-v1/mobile-1-images",  # there is no mobile level 2.
+                "aws-provisioner-v1/mobile-3-images",
+            ),
+            "application-services": (
+                "aws-provisioner-v1/app-services-1-images",
+                "aws-provisioner-v1/app-services-3-images",
+            ),
+        }),
+    }),
 
 
     # for trace_back_to_*_tree.  These repos have access to restricted scopes;

diff --git a/scriptworker/cot/verify.py b/scriptworker/cot/verify.py
@@ -51,7 +51,7 @@
     get_branch,
     get_triggered_by,
     get_task_id,
-    get_worker_type,
+    get_worker_pool_id,
     is_try_or_pull_request,
     is_action,
 )
@@ -343,7 +343,7 @@ def guess_worker_impl(link):
         worker_impls.append("docker-worker")
     if task['provisionerId'] in link.context.config['scriptworker_provisioners']:
         worker_impls.append("scriptworker")
-    if task['workerType'] in link.context.config['scriptworker_worker_types']:
+    if get_worker_pool_id(task) in link.context.config['scriptworker_worker_pools']:
         worker_impls.append("scriptworker")
     if task['payload'].get("mounts") is not None:
         worker_impls.append("generic-worker")
@@ -1656,9 +1656,9 @@ async def verify_parent_task(chain, link):
         CoTError: on chain of trust verification error.
 
     """
-    worker_type = get_worker_type(link.task)
-    if worker_type not in chain.context.config['valid_decision_worker_types']:
-        raise CoTError("{} is not a valid decision workerType!".format(worker_type))
+    pool = get_worker_pool_id(link.task)
+    if pool not in chain.context.config['valid_decision_worker_pools']:
+        raise CoTError("{} is not a valid decision workerPoolId!".format(pool))
     if chain is not link:
         # make sure all tasks generated from this parent task match the published
         # task-graph.json. Not applicable if this link is the ChainOfTrust object,
@@ -1713,9 +1713,9 @@ async def verify_docker_image_task(chain, link):
     """
     errors = []
     # workerType
-    worker_type = get_worker_type(link.task)
-    if worker_type not in chain.context.config['valid_docker_image_worker_types']:
-        errors.append("{} is not a valid docker-image workerType!".format(worker_type))
+    worker_pool = get_worker_pool_id(link.task)
+    if worker_pool not in chain.context.config['valid_docker_image_worker_pools']:
+        errors.append("{} is not a valid docker-image workerPoolId!".format(worker_pool))
     raise_on_errors(errors)
 
 

diff --git a/scriptworker/task.py b/scriptworker/task.py
@@ -273,6 +273,38 @@ def get_worker_type(task):
     return task['workerType']
 
 
+# get_provisioner_id {{{1
+def get_provisioner_id(task):
+    """Given a task dict, return the provisionerId.
+
+    Args:
+        task (dict): the task dict.
+
+    Returns:
+        str: the provisionerId.
+
+    """
+    return task['provisionerId']
+
+
+# get_worker_pool_id {{{1
+def get_worker_pool_id(task):
+    """Given a task dict, return the worker pool id.
+
+    This corresponds to `{provisioner_id}/{workerType}`.
+
+    Args:
+        task (dict): the task dict.
+
+    Returns:
+        str: the workerPoolId.
+
+    """
+    return "{}/{}".format(
+        get_provisioner_id(task), get_worker_type(task)
+    )
+
+
 # get_project {{{1
 def get_project(valid_vcs_rules, source_url):
     """Given vcs rules and a source_url, return the project.

diff --git a/scriptworker/test/test_cot_verify.py b/scriptworker/test/test_cot_verify.py
@@ -515,6 +515,9 @@ def test_raise_on_errors(errors, raises):
 ), (
     {'payload': {}, 'provisionerId': 'test-dummy-provisioner', 'workerType': 'test-dummy-myname', 'scopes': ["x"]},
     'scriptworker', False
+), (
+    {'payload': {}, 'provisionerId': 'scriptworker-prov-v1', 'workerType': 'signing-linux-v1', 'scopes': ["x"]},
+    'scriptworker', False
 ), (
     {'payload': {'mounts': [], 'osGroups': []}, 'provisionerId': '', 'workerType': '', 'scopes': []},
     'generic-worker', False
@@ -2080,7 +2083,8 @@ def task_graph(*args, **kwargs):
             makedirs(os.path.dirname(path))
             touch(path)
         chain.links = [parent_link, build_link]
-        parent_link.task['workerType'] = chain.context.config['valid_decision_worker_types'][0]
+        parent_link.task['provisionerId'] = chain.context.config['valid_decision_worker_pools'][0].split("/")[0]
+        parent_link.task['workerType'] = chain.context.config['valid_decision_worker_pools'][0].split("/")[1]
         mocker.patch.object(cotverify, 'load_json_or_yaml', new=task_graph)
         mocker.patch.object(cotverify, 'verify_parent_task_definition', new=defn_fn)
         if raises:
@@ -2123,7 +2127,8 @@ def task_graph(*args, **kwargs):
 @pytest.mark.asyncio
 async def test_verify_parent_task_missing_graph(chain, decision_link, build_link, mocker):
     chain.links = [decision_link, build_link]
-    decision_link.task['workerType'] = chain.context.config['valid_decision_worker_types'][0]
+    decision_link.task['provisionerId'] = chain.context.config['valid_decision_worker_pools'][0].split("/")[0]
+    decision_link.task['workerType'] = chain.context.config['valid_decision_worker_pools'][0].split("/")[1]
     with pytest.raises(CoTError):
         await cotverify.verify_parent_task(chain, decision_link)
 
@@ -2155,7 +2160,8 @@ async def test_verify_partials_task_noop(chain, build_link):
 # verify_docker_image_task {{{1
 @pytest.mark.asyncio
 async def test_verify_docker_image_task(chain, docker_image_link):
-    docker_image_link.task['workerType'] = chain.context.config['valid_docker_image_worker_types'][0]
+    docker_image_link.task['provisionerId'] = chain.context.config['valid_docker_image_worker_pools'][0].split("/")[0]
+    docker_image_link.task['workerType'] = chain.context.config['valid_docker_image_worker_pools'][0].split("/")[1]
     await cotverify.verify_docker_image_task(chain, docker_image_link)
 
 

diff --git a/scriptworker/test/test_integration.py b/scriptworker/test/test_integration.py
@@ -64,7 +64,7 @@ def read_integration_creds():
 
 
 def build_config(override, basedir):
-    randstring = slugid.nice()[0:6]
+    randstring = slugid.nice().lower().replace('_', '').replace('-', '')[:6]
     config = get_unfrozen_copy(DEFAULT_CONFIG)
     ED25519_DIR = os.path.join(os.path.dirname(__file__), "data", "ed25519")
     config.update({

diff --git a/scriptworker/version.py b/scriptworker/version.py
@@ -52,7 +52,7 @@ def get_version_string(version):
 
 # 1}}}
 # Semantic versioning 2.0.0  http://semver.org/
-__version__ = (23, 6, 1)
+__version__ = (24, 0, 0)
 __version_string__ = get_version_string(__version__)
 
 

diff --git a/version.json b/version.json
@@ -1,8 +1,8 @@
 {
     "version": [
-        23,
-        6,
-        2
+        24,
+        0,
+        0
     ],
-    "version_string": "23.6.2"
+    "version_string": "24.0.0"
 }