Run taskgraph generation with Python 3

.taskcluster.yml

@@ -10,7 +10,7 @@ tasks:
     - $let:
           taskgraph:
               branch: taskgraph
-              revision: 0c74426138739b1cad3bb4ff169af67eef6003fc
+              revision: 644b0b15ef3ac3888701b3e6c68e26137210dcbc
           trustDomain: app-services
       in:
           $let:
@@ -231,7 +231,7 @@ tasks:
                               # Note: This task is built server side without the context or tooling that
                               # exist in tree so we must hard code the hash
                               image:
-                                  mozillareleases/taskgraph:decision-mobile-6607973bc60e32323a541861cc5856cd6a0f51ea9fd664ef7d43bca8df53db47@sha256:8c471aacc469ea8e7bb4846c16efe086f7350a5cc1df570cc6c86b22895a2456
+                                  mozillareleases/taskgraph:decision-mobile-0e1cefb4ec46060d53bcd0eb2c0dd296210fa969998af468d693d1c6922efd1d@sha256:7cfb913bee636f1ab1477a09f8ad746e1f8c68fab0e2a3e1408c985d2ddc27f7
 
                               maxRunTime: 1800
 
@@ -244,8 +244,8 @@ tasks:
                                   - bash
                                   - -cx
                                   - >
-                                    PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
-                                    PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml &&
+                                    PIP_IGNORE_INSTALLED=0 pip3 install --user /builds/worker/checkouts/taskgraph &&
+                                    PIP_IGNORE_INSTALLED=0 pip3 install --user arrow taskcluster pyyaml &&
                                     ln -s /builds/worker/artifacts artifacts &&
                                     ~/.local/bin/taskgraph decision
                                     --pushlog-id='0'

taskcluster/app_services_taskgraph/__init__.py

@@ -2,13 +2,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 from importlib import import_module
 import os
 
-from six import text_type
-from voluptuous import Required, Any
 
 from .build_config import get_version
 
@@ -27,12 +24,12 @@ def register(graph_config):
 
 def _import_modules(modules):
     for module in modules:
-        import_module(".{}".format(module), package=__name__)
+        import_module(f".{module}", package=__name__)
 
 
 def get_decision_parameters(graph_config, parameters):
     if parameters["tasks_for"] == "github-release":
-        head_tag = parameters["head_tag"].decode("utf-8")
+        head_tag = parameters["head_tag"]
         if not head_tag:
             raise ValueError(
                 "Cannot run github-release if `head_tag` is not defined. Got {}".format(
@@ -47,7 +44,7 @@ def get_decision_parameters(graph_config, parameters):
                 "{version} from buildconfig.yml".format(head_tag[1:], version)
             )
     elif parameters["tasks_for"] == "github-pull-request":
-        pr_title = os.environ.get("APPSERVICES_PULL_REQUEST_TITLE", "").decode("UTF-8")
+        pr_title = os.environ.get("APPSERVICES_PULL_REQUEST_TITLE", "")
         if "[ci full]" in pr_title:
             parameters["target_tasks_method"] = "pr-full"
         elif "[ci skip]" in pr_title:

taskcluster/app_services_taskgraph/build_config.py

@@ -2,7 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 import yaml

taskcluster/app_services_taskgraph/job.py

@@ -2,46 +2,43 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
-from taskgraph.util import path
 from taskgraph.util.schema import Schema, taskref_or_string
 from voluptuous import Required, Optional
-from six import text_type
 
 from pipes import quote as shell_quote
 
 secret_schema = {
-    Required("name"): text_type,
-    Required("path"): text_type,
-    Required("key"): text_type,
+    Required("name"): str,
+    Required("path"): str,
+    Required("key"): str,
     Optional("json"): bool,
 }
 
 dummy_secret_schema = {
-    Required("content"): text_type,
-    Required("path"): text_type,
+    Required("content"): str,
+    Required("path"): str,
     Optional("json"): bool,
 }
 
 gradlew_schema = Schema({
     Required("using"): "gradlew",
-    Optional("pre-gradlew"): [[text_type]],
-    Required("gradlew"): [text_type],
-    Optional("post-gradlew"): [[text_type]],
+    Optional("pre-gradlew"): [[str]],
+    Required("gradlew"): [str],
+    Optional("post-gradlew"): [[str]],
     # Base work directory used to set up the task.
-    Required("workdir"): text_type,
+    Required("workdir"): str,
     Optional("use-caches"): bool,
     Optional("secrets"): [secret_schema],
     Optional("dummy-secrets"): [dummy_secret_schema],
 })
 
 run_commands_schema = Schema({
     Required("using"): "run-commands",
-    Optional("pre-commands"): [[text_type]],
+    Optional("pre-commands"): [[str]],
     Required("commands"): [[taskref_or_string]],
-    Required("workdir"): text_type,
+    Required("workdir"): str,
     Optional("use-caches"): bool,
     Optional("secrets"): [secret_schema],
     Optional("dummy-secrets"): [dummy_secret_schema],
@@ -145,7 +142,7 @@ def _convert_commands_to_string(commands):
                     part_string = part["task-reference"]
                     should_task_reference = True
                 else:
-                    raise ValueError('Unsupported dict: {}'.format(part))
+                    raise ValueError(f'Unsupported dict: {part}')
             else:
                 part_string = part
 

taskcluster/app_services_taskgraph/loader/__init__.py

@@ -6,7 +6,6 @@
 # charge of taking every task in the kind, applying job-defaults and
 # finding what are the right upstream dependencies
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 
@@ -27,7 +26,7 @@ def group_tasks(config, tasks):
 
     groups = group_by_fn(config, tasks)
 
-    for combinations in groups.itervalues():
+    for combinations in groups.values():
         dependencies = [copy.deepcopy(t) for t in combinations]
         yield dependencies
 
@@ -50,7 +49,7 @@ def component_grouping(config, tasks):
         task for task in tasks
         if task.attributes.get("buildconfig", {}).get("name", "") == "all"
     ]
-    for _, tasks in groups.iteritems():
+    for _, tasks in groups.items():
         tasks.extend(copy.deepcopy(tasks_for_all_components))
 
     return groups

taskcluster/app_services_taskgraph/loader/build_config.py

@@ -5,11 +5,8 @@
 # XXX: This loader generates a new build task for every component defined in
 # `.buildconfig-android.yml`
 
-from __future__ import print_function, unicode_literals
 
-import os
 
-from copy import deepcopy
 from taskgraph.loader.transform import loader as base_loader
 
 from ..build_config import get_components

taskcluster/app_services_taskgraph/loader/multi_dep.py

@@ -2,14 +2,12 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 
 from voluptuous import Required
 
 from taskgraph.task import Task
-from taskgraph.util.attributes import sorted_unique_list
 from taskgraph.util.schema import Schema
 
 from . import group_tasks
@@ -20,7 +18,7 @@
     Required(
         'dependent-tasks',
         'dictionary of dependent tasks, keyed by kind',
-    ): {basestring: Task},
+    ): {str: Task},
 })
 
 
@@ -67,7 +65,7 @@ def get_primary_dep(config, dep_tasks):
     is the primary dependency. If it's undefined, return the first dep.
     """
     primary_dependencies = config.get('primary-dependency')
-    if isinstance(primary_dependencies, basestring):
+    if isinstance(primary_dependencies, str):
         primary_dependencies = [primary_dependencies]
     if not primary_dependencies:
         assert len(dep_tasks) == 1, "Must define a primary-dependency!"

taskcluster/app_services_taskgraph/target_tasks.py

@@ -2,7 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.target_tasks import _target_task, filter_for_tasks_for
 
@@ -20,7 +19,7 @@ def filter(task):
         return filter_for_tasks_for(task, parameters) \
             and task.attributes.get("run-on-pr-type", "all") in ("full-ci", "all")
 
-    return [l for l, task in full_task_graph.tasks.iteritems() if filter(task)]
+    return [l for l, task in full_task_graph.tasks.items() if filter(task)]
 
 
 @_target_task('pr-normal')
@@ -31,4 +30,4 @@ def filter(task):
         return filter_for_tasks_for(task, parameters) \
                 and task.attributes.get("run-on-pr-type", "all") in ("normal-ci", "all")
 
-    return [l for l, task in full_task_graph.tasks.iteritems() if filter(task)]
+    return [l for l, task in full_task_graph.tasks.items() if filter(task)]

taskcluster/app_services_taskgraph/transforms/__init__.py

@@ -2,7 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 from ..build_config import EXTENSIONS
 
@@ -12,15 +11,15 @@ def _extensions(type, secondary_extensions):
 
 
 def _artifact_filename(name, version, extension):
-    return "{}-{}{}".format(name, version, extension)
+    return f"{name}-{version}{extension}"
 
 
 def publications_to_artifact_paths(name, version, publications, secondary_extensions=("",)):
     paths = []
     for publication in publications:
         for extension in _extensions(publication["type"], secondary_extensions):
             artifact_filename = _artifact_filename(publication['name'], version, extension)
-            paths.append("public/build/{}".format(artifact_filename))
+            paths.append(f"public/build/{artifact_filename}")
 
     return paths
 
@@ -30,7 +29,7 @@ def publications_to_artifact_map_paths(name, version, publications, secondary_ex
     for publication in publications:
         for extension in _extensions(publication["type"], secondary_extensions):
             artifact_filename = _artifact_filename(publication['name'], version, extension)
-            build_map_paths["public/build/{}".format(artifact_filename)] = {
+            build_map_paths[f"public/build/{artifact_filename}"] = {
                 "checksums_path": "",  # XXX beetmover marks this as required, but it's not needed
                 "destinations": ["maven2/org/mozilla/appservices/{}/{}/{}".format(publication['name'], version, artifact_filename)]
             }

taskcluster/app_services_taskgraph/transforms/beetmover.py

@@ -2,7 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import resolve_keyed_by

taskcluster/app_services_taskgraph/transforms/module_build.py

@@ -2,7 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 
@@ -19,22 +18,22 @@ def rustup_setup(config, tasks):
             [
                 "source",
                 "taskcluster/scripts/toolchain/rustup-setup.sh",
-                unicode(config.params["tasks_for"])
+                config.params["tasks_for"]
             ]
         )
         yield task
 
 @transforms.add
 def release_upload_symbols(config, tasks):
     for task in tasks:
-        if (config.params["tasks_for"] == u"github-release" and
+        if (config.params["tasks_for"] == "github-release" and
             task["attributes"]["buildconfig"]["uploadSymbols"]):
                 task["run"].setdefault("post-gradlew", [])
                 task["run"]["post-gradlew"].append(
                     [
                         "source",
                         "automation/upload_android_symbols.sh",
-                        unicode(task["attributes"]["buildconfig"]["path"])
+                        task["attributes"]["buildconfig"]["path"]
                     ]
                 )
 
@@ -56,10 +55,10 @@ def build_task(config, tasks):
         all_extensions = get_extensions(name)
         for publication_name, extensions in all_extensions.items():
             for extension in extensions:
-                artifact_filename = "{}-{}{}".format(publication_name, version, extension)
+                artifact_filename = f"{publication_name}-{version}{extension}"
                 artifacts.append({
-                    "name": "public/build/{}".format(artifact_filename),
-                    "path": "/builds/worker/checkouts/src/build/maven/org/mozilla/appservices/{}/{}/{}".format(publication_name, version, artifact_filename),
+                    "name": f"public/build/{artifact_filename}",
+                    "path": f"/builds/worker/checkouts/src/build/maven/org/mozilla/appservices/{publication_name}/{version}/{artifact_filename}",
                     "type": "file",
                 })
 

taskcluster/app_services_taskgraph/transforms/multi_dep.py

@@ -2,7 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 

taskcluster/app_services_taskgraph/transforms/secrets.py

@@ -5,7 +5,6 @@
 Resolve secrets and dummy secrets
 """
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import resolve_keyed_by

taskcluster/app_services_taskgraph/transforms/signing.py

@@ -2,7 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import resolve_keyed_by
@@ -22,7 +21,7 @@ def build_upstream_artifacts(config, tasks):
         version = get_version()
 
         worker_definition = {"upstream-artifacts": [{
-            "taskId": {"task-reference": "<{}>".format(dep.kind)},
+            "taskId": {"task-reference": f"<{dep.kind}>"},
             "taskType": "build",
             "paths": publications_to_artifact_paths(name, version, module_info["publications"]),
             "formats": ["autograph_gpg"],

taskcluster/app_services_taskgraph/transforms/toolchain.py

@@ -2,7 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
 
 import subprocess
 
@@ -24,7 +23,7 @@
 
 @memoize
 def git_sha_for_directory(directory):
-    output = subprocess.check_output(["git", "rev-parse", "HEAD:{}".format(directory)])
+    output = subprocess.check_output(["git", "rev-parse", f"HEAD:{directory}"])
     sha = output.decode("utf8").strip()
     return sha