Merge 0dfa6d9 into ccc1b31

mozilla-releng · Nov 10, 2016 · 27db038 · 27db038
2 parents ccc1b31 + 0dfa6d9
commit 27db038
Show file tree

Hide file tree

Showing 33 changed files with 1,838 additions and 461 deletions.
diff --git a/.gitignore b/.gitignore
@@ -61,6 +61,7 @@ target/
 
 # Vim
 *.swp
+*.swo
 
 # TODO cleanup
 secrets.json
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,11 +3,22 @@ All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
+### Added
+- `scriptworker.cot.verify` now verifies the chain of trust for the graph.
+- `scriptworker.exceptions.CoTError` now marks chain of trust validation errors.
+- `scriptworker.task.get_task_id`, `scriptworker.task.get_run_id`, `scriptworker.task.get_decision_task_id`, `scriptworker.task.get_worker_type`
+
 ### Changed
 - config files are now yaml, to enable comments.  `config_example.json` and `cot_config_example.json` have been consolidated into `scriptworker.yaml.tmpl`.  `context.cot_config` items now live in `context.config`.
+- `validate_artifact_url` now takes a list of dictionaries as rules, leading to more configurable url checking.
+- `scriptworker.cot` is now `scriptworker.cot.generate`.  The `get_environment` function has been renamed to `get_cot_environment`.
+- `scriptworker.gpg.get_body` now takes a `verify_sig` kwarg.
+- `download_artifacts` now takes `valid_artifact_task_ids` as a kwarg.
 
 ### Removed
 - removed all references to `cot_config`
+- removed the credential update, since puppet restarts scriptworker on config change.
+- removed aiohttp max connections
 
 ## [0.9.0] - 2016-11-01
 ### Added

diff --git a/requirements-prod.txt b/requirements-prod.txt
@@ -14,16 +14,16 @@ PyYAML==3.12 \
     --hash=sha512:13733054619053893f4a8d75c1da020a87e1f697b007ab182be06b5b941b49c4cb3dc6f9ff131be6bc10fc727ad7334fa5401346f3a77140aed0f86280532fce \
     --hash=sha512:5ff411ddbbf733ad52334015a04eb061998803ff94ad474ea2d534a713aa302cdfadde908ff2bb1dbd4f13e1a1fb99786f8dd1be1e061fcb6a7c0f471e41591f \
     --hash=sha512:4ae6c5843688f45751ddb14a8b6f16f58b1f8a4805be430afa4035857b1826be95a8422395d64ac88282f9edd57f35fee5d1b6a4dab9093d47491d300d2079f3
-aiohttp==1.0.5 \
-    --hash=sha512:520bbc3283de9f215ff57e3ab2e0f9255caef3bff65dd47d6ef846845732ec0ddb32488897241a0a03839e38569cb5a083300318df0e431677eeea1676c981d0 \
-    --hash=sha512:fd67c2a32b355e38aec5447af1decfa3efe91511a3d196c29c0f2b80fe208f11a89502e724468c3f23eb081875ced3044f1712db5f46cebe2abdc36bb4f88207 \
-    --hash=sha512:c18665e6ac0f71e8fcac35a66937f0226c8e4af4ddca2b15a27a77da3cf7172d3fe3f08ba202ca934aae27ea7676b8f9a0c2ee52acbb9d2f3bf922eafd89aab0 \
-    --hash=sha512:196a2f2604741b457a37529598c334aa6544923f979c4cf7394325a41e193953b926371332c06f30969cdc62ba9b19009f4bae8cb4bdcaa20f95410115a814af \
-    --hash=sha512:975f16c5c2eb3821ccbbd465acd6113b18c64a50f879143760e8fc457eb40548c29bccc215b55348c78e9de25c625a4aa94698e034fea3020a7d603ab8cf8264 \
-    --hash=sha512:d034ac08cc5c68753915df4dbe3e894a9fc09d423aff51c43d44eaafb0d12be84f8c21438ae29f6b828db9b586b952547becbd79d2209b4fbfd66d8dac4dad33 \
-    --hash=sha512:c0e1c4c65d2e3d4fce0c072e9876be0a39fee2e5ea727d5fdc9de361356f0744a5fe0a6b02e99358d2322144fa5dffbba1d966e24c4622070ffaf5f389f42198 \
-    --hash=sha512:a15ae9cd0d9409f6005b044249448fb079734517b1f2c93b898693657c2f2af389b924b30b36b4c03f91a599b2e85ffffa1e9ae458f6809f0a169e2ab245e8ca \
-    --hash=sha512:db5661b77b90eaecf04d80cebc1023ec7930bd639aed075d24291f897784d040ccc3fc5f9db310a3886f1661a0d163aaf42a8c5549b566027e890df42bb1bc0d
+aiohttp==1.1.1 \
+    --hash=sha512:21954987936b1670af2b1bc376d9a0cf2d4c94f967938d5710c6e99037bc4ef43b56b0b0740243e4909d78b0b2a96c9f645937a7d474fc2425f5c05d53655a4d \
+    --hash=sha512:ffe30c7d0417169df890615dfed123f99dec6627c324a4ac2ee27059c0aeee532b79543e4c06a084d6d86bde18c45c463ab8a994007b4866b77c9716c3d7d77c \
+    --hash=sha512:6803bf65657742e31da68bb5a50ee1a05204b9cc594e0f1cfcc56f2de902462327eb67cc9042b23e20a454a1f37e4f32d9dcb4b47639018098cd9cb26e0cca8e \
+    --hash=sha512:389d2e10b0f224ee52f6ff0c8080a909b33c3aad0d6ae1b87a7d717e6daf8376486e6df748e31c99966f736ad697b3e7460d2f1aa7a5b04c66d30ee1c98d7c48 \
+    --hash=sha512:d7fd98bbbef1f12005a2ead2f62fa26db0d7133276199d52d3958da3e35db02fafdf5ccd11bf96fee7adc021967e71dc8833d4eab0ad8de268b47310cdc60b6d \
+    --hash=sha512:39321b3d4d5f0754f1cc858dc2cbdf45073d88461ad8a21f5f46405e84461c874ba9bd3b0170cce9fd97ab717caf16f78850bb2573d7c55f6b5cbccbd440e9e2 \
+    --hash=sha512:f9ab0bcbfc4b4f50c70344b21a3741fcd5e842112636fd0c6985472de10cb532bcfd54280ff537ea8189b275d7e29afec813b99dff9ed8adb363ba1803bf03a3 \
+    --hash=sha512:96a4484d27eb22abe62dadb3177e5d2af28cd42c81dcc388c0f35f5942b9a3136a58f5074967db884dffe1ef518ccfa0714937787e76fee58403fc3907948235 \
+    --hash=sha512:8aadcc62129c40aa155dca5719a25849900320e7b92a31d9962b50855d3a2b7fb2b12fddbb2084e52cd941d0d19197bfaab4146efa86450c7348e5d0c4a840c9
 arrow==0.8.0 \
     --hash=sha512:b6c01970d408e1169d042f593859577eef9961a2e7d6e0d5d01ddbdc001f806ca191cf152bd2d4060a877aeabee6754f06c3b91fbca53ee0a135a9355d08b347
 async-timeout==1.1.0 \
@@ -79,3 +79,13 @@ taskcluster==0.3.4 \
 virtualenv==15.0.3 \
     --hash=sha512:34486ae905ea83c092418efaf6abae05cdd2d36c32f452ad25775452412c1b6c97de7dccb0f634a3f8a7b1768ed9975b5f16a79fe711ac1cecffd0a46e245583 \
     --hash=sha512:f417dbefe8864be9028ab9efd69844b94550dc7f739688b00c32c15ad4f84a2b0d4885564ed0d0e0b43960a328fe81092cebfb8dd0d4b10a804b42122d9c9621
+yarl==0.5.3 \
+    --hash=sha512:6283e6104ca8c399ece4a3bcf0701361c5d4a1b9bd1f175d9ea1a798e387fabe3e71935441acb3a5362fce82defa7e1eabd370121b6ab69797a1a2a93b4a10e1 \
+    --hash=sha512:d00560013883df51551384c9e957e0cfa4a3b2bef726ff19ddd07d48d25c6d9d419c999cfcf834c45b191708983eea4401a231d62bbddadae98757ba6a4f32ce \
+    --hash=sha512:5d6f8fddb0a78cc610292d06c6bf1a805f68f86182efc4cca28d62c708dae3360c93d28764d3f2dc47c7be5089a169920eca0e71cb757c16fdf3ff7f017a277d \
+    --hash=sha512:a3648ab98dfe9f87fb8f07f89b0932bd4caa308f2da843f02f20d826dca7932d5f0d9449ab3aa643f4ee01d5e380ab5347014df5769ee5b81b697771d9119efc \
+    --hash=sha512:077447b58925d6b3f4703e72615b4845a60844afa0f2abbba675879e8f1cf3a2c7aa103b73c2452e3a7696937c54b8574b685c9bc88ff8ed07f1810e6b3a3f16 \
+    --hash=sha512:68242d020a07179e5a2245b8cf1a0df30f8aa61b0094ea52bc1fb7ee27d48ebe98247f3b391249cf0b52bd5960ed714a71c016c347869541ef71441494592ad8 \
+    --hash=sha512:7298a78b61f8d5d1032445db0c5c88785bf8c6e80df7202dc8edd48b8188df83dbea3311be7fad9547f8211751f96deb945f7614c43af24f01c6fc8acefcd3df \
+    --hash=sha512:658196a3ebd1e79ae6e76354b6cb43f460736a1fa8fe548f3e22666a7c1dcda557f28f22996d640f316ed0f050815889381be97eb59b4ccf305e68c2b40a335d \
+    --hash=sha512:f5bbee7677a86fd808c66df12f5e9ac6e82f6b383154eefb101bf05c540355bf6fbbdd3a42d61c90d80c56854c2a3496d6c05d312c47cd7ddbd08c171f48c5fc
diff --git a/requirements-test-prod.txt b/requirements-test-prod.txt
@@ -14,16 +14,16 @@ PyYAML==3.12 \
     --hash=sha512:13733054619053893f4a8d75c1da020a87e1f697b007ab182be06b5b941b49c4cb3dc6f9ff131be6bc10fc727ad7334fa5401346f3a77140aed0f86280532fce \
     --hash=sha512:5ff411ddbbf733ad52334015a04eb061998803ff94ad474ea2d534a713aa302cdfadde908ff2bb1dbd4f13e1a1fb99786f8dd1be1e061fcb6a7c0f471e41591f \
     --hash=sha512:4ae6c5843688f45751ddb14a8b6f16f58b1f8a4805be430afa4035857b1826be95a8422395d64ac88282f9edd57f35fee5d1b6a4dab9093d47491d300d2079f3
-aiohttp==1.0.5 \
-    --hash=sha512:520bbc3283de9f215ff57e3ab2e0f9255caef3bff65dd47d6ef846845732ec0ddb32488897241a0a03839e38569cb5a083300318df0e431677eeea1676c981d0 \
-    --hash=sha512:fd67c2a32b355e38aec5447af1decfa3efe91511a3d196c29c0f2b80fe208f11a89502e724468c3f23eb081875ced3044f1712db5f46cebe2abdc36bb4f88207 \
-    --hash=sha512:c18665e6ac0f71e8fcac35a66937f0226c8e4af4ddca2b15a27a77da3cf7172d3fe3f08ba202ca934aae27ea7676b8f9a0c2ee52acbb9d2f3bf922eafd89aab0 \
-    --hash=sha512:196a2f2604741b457a37529598c334aa6544923f979c4cf7394325a41e193953b926371332c06f30969cdc62ba9b19009f4bae8cb4bdcaa20f95410115a814af \
-    --hash=sha512:975f16c5c2eb3821ccbbd465acd6113b18c64a50f879143760e8fc457eb40548c29bccc215b55348c78e9de25c625a4aa94698e034fea3020a7d603ab8cf8264 \
-    --hash=sha512:d034ac08cc5c68753915df4dbe3e894a9fc09d423aff51c43d44eaafb0d12be84f8c21438ae29f6b828db9b586b952547becbd79d2209b4fbfd66d8dac4dad33 \
-    --hash=sha512:c0e1c4c65d2e3d4fce0c072e9876be0a39fee2e5ea727d5fdc9de361356f0744a5fe0a6b02e99358d2322144fa5dffbba1d966e24c4622070ffaf5f389f42198 \
-    --hash=sha512:a15ae9cd0d9409f6005b044249448fb079734517b1f2c93b898693657c2f2af389b924b30b36b4c03f91a599b2e85ffffa1e9ae458f6809f0a169e2ab245e8ca \
-    --hash=sha512:db5661b77b90eaecf04d80cebc1023ec7930bd639aed075d24291f897784d040ccc3fc5f9db310a3886f1661a0d163aaf42a8c5549b566027e890df42bb1bc0d
+aiohttp==1.1.1 \
+    --hash=sha512:21954987936b1670af2b1bc376d9a0cf2d4c94f967938d5710c6e99037bc4ef43b56b0b0740243e4909d78b0b2a96c9f645937a7d474fc2425f5c05d53655a4d \
+    --hash=sha512:ffe30c7d0417169df890615dfed123f99dec6627c324a4ac2ee27059c0aeee532b79543e4c06a084d6d86bde18c45c463ab8a994007b4866b77c9716c3d7d77c \
+    --hash=sha512:6803bf65657742e31da68bb5a50ee1a05204b9cc594e0f1cfcc56f2de902462327eb67cc9042b23e20a454a1f37e4f32d9dcb4b47639018098cd9cb26e0cca8e \
+    --hash=sha512:389d2e10b0f224ee52f6ff0c8080a909b33c3aad0d6ae1b87a7d717e6daf8376486e6df748e31c99966f736ad697b3e7460d2f1aa7a5b04c66d30ee1c98d7c48 \
+    --hash=sha512:d7fd98bbbef1f12005a2ead2f62fa26db0d7133276199d52d3958da3e35db02fafdf5ccd11bf96fee7adc021967e71dc8833d4eab0ad8de268b47310cdc60b6d \
+    --hash=sha512:39321b3d4d5f0754f1cc858dc2cbdf45073d88461ad8a21f5f46405e84461c874ba9bd3b0170cce9fd97ab717caf16f78850bb2573d7c55f6b5cbccbd440e9e2 \
+    --hash=sha512:f9ab0bcbfc4b4f50c70344b21a3741fcd5e842112636fd0c6985472de10cb532bcfd54280ff537ea8189b275d7e29afec813b99dff9ed8adb363ba1803bf03a3 \
+    --hash=sha512:96a4484d27eb22abe62dadb3177e5d2af28cd42c81dcc388c0f35f5942b9a3136a58f5074967db884dffe1ef518ccfa0714937787e76fee58403fc3907948235 \
+    --hash=sha512:cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e
 arrow==0.8.0 \
     --hash=sha512:b6c01970d408e1169d042f593859577eef9961a2e7d6e0d5d01ddbdc001f806ca191cf152bd2d4060a877aeabee6754f06c3b91fbca53ee0a135a9355d08b347
 async-timeout==1.1.0 \
@@ -122,9 +122,9 @@ pytest-asyncio==0.5.0 \
 pytest-cov==2.4.0 \
     --hash=sha512:dc11d4098c98baed9c0a6a97bce9cf353c4af0e4f2e755e25826efcde079f8f67305ebc14041705b339d7b87dbfa11b03ad351e78b69768e959e6c9bba3c275b \
     --hash=sha512:288e621b3ef47bb3cefa64ce082da0471ef39b0d5fc8f59ad58487139a58ae5705b57c0cd8e48f41ea7230b923f6282f03577030bf973315cb544a4fe0a3ca60
-pytest-mock==1.3.0 \
-    --hash=sha512:5e871a7631cfa56e4264254545f9150c4ea30043d6699405bf003c37357933bec97efd887e768a45995e42af71ce046df287e9ac321ac712e38ae81755ca51e2 \
-    --hash=sha512:5a457d16f1b6fe2e2f92d09ad129ffc3e12263e31d02a63b44dbbf50a367cfde5076a425e81f9f09e340a688970abf3a9839ab7038953ba7b438c5f7f09a3c2d
+pytest-mock==1.4.0 \
+    --hash=sha512:1663cb3176cf08772230509d96f210081715ba9a4bb9310ffadf3a1e6ce22071d881524d2e44a8b3a9530ac5f95911d396e001ff808b4b599f6ef0bbf9baf99d \
+    --hash=sha512:351813d18f59528de0e92ec498e97d31998581b7ba3c9d02fcb8ca4804ee825a4fce760a041b64f56f259fa6f36dd4b84624e635da5b9b8e0354ff0760cede53
 python-dateutil==2.5.3 \
     --hash=sha512:413b935321f0a65fd8e8ba49990acd5bed60b9bcd614837a26c1cf52ecbf268543a97791dfa7b91ebf183866a8de7be302ca7d18d10352e897994f60bce5df62 \
     --hash=sha512:d8e28dad57ea85663962f4518faea0eb551248ce5dbb0fb1071aa16d3cfa11e303beb4e6f3477ed02cceb42c328c324fa7183bf70beedbe14e579f0b3e8aa3b9 \
@@ -148,3 +148,13 @@ tox==2.4.1 \
 virtualenv==15.0.3 \
     --hash=sha512:34486ae905ea83c092418efaf6abae05cdd2d36c32f452ad25775452412c1b6c97de7dccb0f634a3f8a7b1768ed9975b5f16a79fe711ac1cecffd0a46e245583 \
     --hash=sha512:f417dbefe8864be9028ab9efd69844b94550dc7f739688b00c32c15ad4f84a2b0d4885564ed0d0e0b43960a328fe81092cebfb8dd0d4b10a804b42122d9c9621
+yarl==0.5.3 \
+    --hash=sha512:6283e6104ca8c399ece4a3bcf0701361c5d4a1b9bd1f175d9ea1a798e387fabe3e71935441acb3a5362fce82defa7e1eabd370121b6ab69797a1a2a93b4a10e1 \
+    --hash=sha512:d00560013883df51551384c9e957e0cfa4a3b2bef726ff19ddd07d48d25c6d9d419c999cfcf834c45b191708983eea4401a231d62bbddadae98757ba6a4f32ce \
+    --hash=sha512:5d6f8fddb0a78cc610292d06c6bf1a805f68f86182efc4cca28d62c708dae3360c93d28764d3f2dc47c7be5089a169920eca0e71cb757c16fdf3ff7f017a277d \
+    --hash=sha512:a3648ab98dfe9f87fb8f07f89b0932bd4caa308f2da843f02f20d826dca7932d5f0d9449ab3aa643f4ee01d5e380ab5347014df5769ee5b81b697771d9119efc \
+    --hash=sha512:077447b58925d6b3f4703e72615b4845a60844afa0f2abbba675879e8f1cf3a2c7aa103b73c2452e3a7696937c54b8574b685c9bc88ff8ed07f1810e6b3a3f16 \
+    --hash=sha512:68242d020a07179e5a2245b8cf1a0df30f8aa61b0094ea52bc1fb7ee27d48ebe98247f3b391249cf0b52bd5960ed714a71c016c347869541ef71441494592ad8 \
+    --hash=sha512:7298a78b61f8d5d1032445db0c5c88785bf8c6e80df7202dc8edd48b8188df83dbea3311be7fad9547f8211751f96deb945f7614c43af24f01c6fc8acefcd3df \
+    --hash=sha512:658196a3ebd1e79ae6e76354b6cb43f460736a1fa8fe548f3e22666a7c1dcda557f28f22996d640f316ed0f050815889381be97eb59b4ccf305e68c2b40a335d \
+    --hash=sha512:f5bbee7677a86fd808c66df12f5e9ac6e82f6b383154eefb101bf05c540355bf6fbbdd3a42d61c90d80c56854c2a3496d6c05d312c47cd7ddbd08c171f48c5fc
diff --git a/scriptworker.yaml.tmpl b/scriptworker.yaml.tmpl
@@ -44,6 +44,8 @@ verbose: true
 sign_chain_of_trust: false
 verify_chain_of_trust: false
 verify_cot_signature: false
+# Chain of Trust job type, e.g. signing
+cot_job_type: signing
 
 
 #-----------------------------------------------------------------------------------------------

diff --git a/scriptworker/client.py b/scriptworker/client.py
@@ -5,14 +5,12 @@
 scriptworker.exceptions and scriptworker.constants, or other standalone
 modules, to avoid circular imports.
 """
-import json
 import jsonschema
 import os
-import re
-from urllib.parse import urlparse, unquote
 
-from scriptworker.constants import DEFAULT_CONFIG, STATUSES
+from scriptworker.constants import STATUSES
 from scriptworker.exceptions import ScriptWorkerTaskException
+from scriptworker.utils import load_json, match_url_regex
 
 
 def get_task(config):
@@ -27,16 +25,10 @@ def get_task(config):
     Raises:
         ScriptWorkerTaskException: on error.
     """
-    try:
-        path = os.path.join(config['work_dir'], "task.json")
-        with open(path, "r") as fh:
-            contents = json.load(fh)
-            return contents
-    except (OSError, ValueError) as exc:
-        raise ScriptWorkerTaskException(
-            "Can't read task from {}!\n{}".format(path, str(exc)),
-            exit_code=STATUSES['internal-error']
-        )
+    path = os.path.join(config['work_dir'], "task.json")
+    message = "Can't read task from {}!\n%(exc)s".format(path)
+    contents = load_json(path, is_path=True, message=message)
+    return contents
 
 
 def validate_json_schema(data, schema, name="task"):
@@ -62,27 +54,16 @@ def validate_json_schema(data, schema, name="task"):
         )
 
 
-def validate_artifact_url(config, url):
+def validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url):
     """Ensure a URL fits in given scheme, netloc, and path restrictions.
 
-    If `valid_artifact_schemes`, `valid_artifact_netlocs`, and/or
-    `valid_artifact_path_regexes` are defined in `config` but are `None`,
-    skip that check.
-
-    If any are missing from `config`, fall back to the values in
-    `DEFAULT_CONFIG`.
-
-    If `valid_artifact_path_regexes` is not None, the url path should
-    match one.  Each regex should define a `filepath`, which is what we'll
-    return.
-
-    Otherwise, if we pass all checks, return the unmodified path.
-
     If we fail any checks, raise a ScriptWorkerTaskException with
     `malformed-payload`.
 
     Args:
-        config (dict): the running config.
+        valid_artifact_rules (tuple): the tests to run, with `schemas`, `netlocs`,
+            and `path_regexes`.
+        valid_artifact_task_ids (list): the list of valid task IDs to download from.
         url (str): the url of the artifact.
 
     Returns:
@@ -91,50 +72,21 @@ def validate_artifact_url(config, url):
     Raises:
         ScriptWorkerTaskException: on failure to validate.
     """
-    messages = []
-    validate_config = {}
-    for key in (
-        'valid_artifact_schemes', 'valid_artifact_netlocs',
-        'valid_artifact_path_regexes', 'valid_artifact_task_ids',
-    ):
-        if key in config:
-            validate_config[key] = config[key]
-        else:
-            validate_config[key] = DEFAULT_CONFIG[key]
-    parts = urlparse(url)
-    path = unquote(parts.path)
-    return_value = path
-    # scheme whitelisted?
-    if validate_config['valid_artifact_schemes'] is not None and \
-            parts.scheme not in validate_config['valid_artifact_schemes']:
-        messages.append('Invalid scheme: {}!'.format(parts.scheme))
-    # netloc whitelisted?
-    if validate_config['valid_artifact_netlocs'] is not None and \
-            parts.netloc not in validate_config['valid_artifact_netlocs']:
-        messages.append('Invalid netloc: {}!'.format(parts.netloc))
-    # check the paths
-    for regex in validate_config.get('valid_artifact_path_regexes') or []:
-        m = re.search(regex, path)
-        if m is None:
-            continue
-        path_info = m.groupdict()
+
+    def callback(match):
+        path_info = match.groupdict()
         # make sure we're pointing at a valid task ID
         if 'taskId' in path_info and \
-                path_info['taskId'] not in validate_config['valid_artifact_task_ids']:
-            messages.append('Invalid taskId: {}!'.format(path_info['taskId']))
-            break
+                path_info['taskId'] not in valid_artifact_task_ids:
+            return
         if 'filepath' not in path_info:
-            messages.append('Invalid regex {}!'.format(regex))
-            break
-        return_value = path_info['filepath']
-        break
-    else:
-        if validate_config.get('valid_artifact_path_regexes'):
-            messages.append('Invalid path: {}!'.format(path))
-
-    if messages:
+            return
+        return path_info['filepath']
+
+    filepath = match_url_regex(valid_artifact_rules, url, callback)
+    if filepath is None:
         raise ScriptWorkerTaskException(
-            "Can't validate url {}\n{}".format(url, messages),
+            "Can't validate url {}".format(url),
             exit_code=STATUSES['malformed-payload']
         )
-    return return_value.lstrip('/')
+    return filepath.lstrip('/')
diff --git a/scriptworker/config.py b/scriptworker/config.py
@@ -9,7 +9,6 @@
 import argparse
 from copy import deepcopy
 from frozendict import frozendict
-import json
 import logging
 import os
 import re
@@ -19,6 +18,7 @@
 from scriptworker.constants import DEFAULT_CONFIG
 from scriptworker.context import Context
 from scriptworker.log import update_logging_config
+from scriptworker.utils import load_json
 
 log = logging.getLogger(__name__)
 
@@ -64,12 +64,9 @@ def read_worker_creds(key="credentials"):
     for path in CREDS_FILES:
         if not os.path.exists(path):
             continue
-        with open(path, "r") as fh:
-            try:
-                contents = json.load(fh)
-                return contents[key]
-            except (json.decoder.JSONDecodeError, KeyError):
-                pass
+        contents = load_json(path, is_path=True, exception=None)
+        if contents.get(key):
+            return contents[key]
     else:
         if key == "credentials" and os.environ.get("TASKCLUSTER_ACCESS_TOKEN") and \
                 os.environ.get("TASKCLUSTER_CLIENT_ID"):