[python3] Reland recipes -> python3 changes

This was enabled by moving the iPhones off of the old RPI2 hosts.

This reverts commit 04cd6fb.
This reverts commit a726978.

Bug: chromium:1256037
Change-Id: I35069089aa39baf62a18235c8d0514923f327c53
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/477987
Reviewed-by: Ravi Mistry <rmistry@google.com>
Commit-Queue: Eric Boren <borenet@google.com>

.vpython

@@ -0,0 +1 @@
+python_version: "3.8"

gn/codesign_ios.py

@@ -23,7 +23,8 @@
 
 # Find the signing identity.
 identity = None
-for line in subprocess.check_output(['security', 'find-identity']).split('\n'):
+for line in subprocess.check_output([
+    'security', 'find-identity']).decode('utf-8').split('\n'):
   m = re.match(r'''.*\) (.*) "''' + identstr + '"', line)
   if m:
     identity = m.group(1)

gn/compile_ib_files.py

@@ -40,9 +40,9 @@ def main():
   ibtool_section_re = re.compile(r'/\*.*\*/')
   ibtool_re = re.compile(r'.*note:.*is clipping its content')
   try:
-    stdout = subprocess.check_output(ibtool_args)
+    stdout = subprocess.check_output(ibtool_args).decode('utf-8')
   except subprocess.CalledProcessError as e:
-    print(e.output)
+    print(e.output.decode('utf-8'))
     raise
   current_section_header = None
   for line in stdout.splitlines():

gn/compile_sksl_tests.py

@@ -28,11 +28,12 @@ def executeWorklist(input, worklist):
     # Invoke skslc, passing in the worklist.
     worklist.close()
     try:
-        output = subprocess.check_output([skslc, worklist.name], stderr=subprocess.STDOUT)
+        output = subprocess.check_output([
+            skslc, worklist.name], stderr=subprocess.STDOUT).decode('utf-8')
     except subprocess.CalledProcessError as err:
         if err.returncode != 1:
             print("### " + input + " skslc error:\n")
-            print("\n".join(err.output.splitlines()))
+            print("\n".join(err.output.decode('utf-8').splitlines()))
             sys.exit(err.returncode)
         pass  # Compile errors (exit code 1) are expected and normal in test code
 

gn/copy_git_directory.py

@@ -21,7 +21,8 @@ def copy_git_directory(src, dst, out=None):
     raise Exception('Directory "%s" does not exist.' % src)
   if not os.path.isdir(dst):
     os.makedirs(dst)
-  ls_files = subprocess.check_output(['git', 'ls-files', '-z', '.'], cwd=src)
+  ls_files = subprocess.check_output([
+      'git', 'ls-files', '-z', '.'], cwd=src).decode('utf-8')
   src_files = set(p for p in ls_files.split('\0') if p)
   abs_src = os.path.abspath(src)
   cwd = os.getcwd()

gn/dehydrate_sksl.py

@@ -20,7 +20,8 @@
         if not os.path.isdir(targetDir):
             os.mkdir(targetDir)
         target = os.path.join(targetDir, tail)
-        subprocess.check_output([skslc, inc, target + ".dehydrated.sksl"])
+        subprocess.check_output([
+            skslc, inc, target + ".dehydrated.sksl"]).decode('utf-8')
     except subprocess.CalledProcessError as err:
         print("### Error compiling " + inc + ":")
         print(err.output)

gn/find_headers.py

@@ -34,9 +34,9 @@
 
 desc_json_txt = ''
 try:
-  desc_json_txt = subprocess.check_output(gn_desc_cmd)
+  desc_json_txt = subprocess.check_output(gn_desc_cmd).decode('utf-8')
 except subprocess.CalledProcessError as e:
-  print(e.output)
+  print(e.output.decode('utf-8'))
   raise
 
 desc_json = {}

gn/find_xcode_sysroot.py

@@ -12,4 +12,5 @@
 
 (sdk,) = sys.argv[1:]
 
-print(subprocess.check_output(['xcrun', '--sdk', sdk, '--show-sdk-path']))
+print(subprocess.check_output([
+    'xcrun', '--sdk', sdk, '--show-sdk-path']).decode('utf-8'))

infra/bots/assets/clang_linux/create.py

@@ -42,8 +42,8 @@ def create_asset(target_dir):
   subprocess.check_call(["cp", "bin/llvm-symbolizer", target_dir + "/bin"])
   subprocess.check_call(["cp", "bin/llvm-profdata", target_dir + "/bin"])
   subprocess.check_call(["cp", "bin/llvm-cov", target_dir + "/bin"])
-  libstdcpp = subprocess.check_output(["c++",
-                                       "-print-file-name=libstdc++.so.6"])
+  libstdcpp = subprocess.check_output([
+      "c++", "-print-file-name=libstdc++.so.6"]).decode('utf-8')
   subprocess.check_call(["cp", libstdcpp.strip(), target_dir + "/lib"])
 
   # Finally, build libc++ for TSAN and MSAN bots using the Clang we just built.

infra/bots/bundle_recipes.sh

@@ -6,8 +6,10 @@
 
 set -x -e
 
+which vpython3
+vpython3 --version
 cd skia
 git init
 git add .
 git commit -m "Commit Recipes"
-python infra/bots/recipes.py bundle --destination ${1}/recipe_bundle
+vpython3 infra/bots/recipes.py bundle --destination ${1}/recipe_bundle

infra/bots/gen_tasks_logic/compile_cas.go

@@ -45,6 +45,7 @@ var (
 	explicitPaths = []string{
 		".clang-format",
 		".clang-tidy",
+		".vpython",
 		"bin/fetch-clang-format",
 		"bin/fetch-gn",
 		"buildtools",

infra/bots/gen_tasks_logic/gen_tasks_logic.go

@@ -140,6 +140,7 @@ var (
 	CAS_SPEC_LOTTIE_CI = &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/infra/bots/run_recipe.py",
 			"skia/infra/lottiecap",
 			"skia/tools/lottie-web-perf",
@@ -403,6 +404,7 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_CANVASKIT, &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/infra/bots/run_recipe.py",
 			"skia/infra/canvaskit",
 			"skia/modules/canvaskit",
@@ -416,6 +418,7 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_LOTTIE_WEB, &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/infra/bots/run_recipe.py",
 			"skia/tools/lottie-web-perf",
 		},
@@ -424,6 +427,7 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_PATHKIT, &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/infra/bots/run_recipe.py",
 			"skia/infra/pathkit",
 			"skia/modules/pathkit",
@@ -433,6 +437,7 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_PERF, &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/infra/bots/assets",
 			"skia/infra/bots/run_recipe.py",
 			"skia/platform_tools/ios/bin",
@@ -444,6 +449,7 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_PUPPETEER, &specs.CasSpec{
 		Root: "../skia", // Needed for other repos.
 		Paths: []string{
+			".vpython",
 			"tools/perf-canvaskit-puppeteer",
 		},
 		Excludes: []string{rbe.ExcludeGitDir},
@@ -463,13 +469,15 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_RUN_RECIPE, &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/infra/bots/run_recipe.py",
 		},
 		Excludes: []string{rbe.ExcludeGitDir},
 	})
 	b.MustAddCasSpec(CAS_SKOTTIE_WASM, &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/infra/bots/run_recipe.py",
 			"skia/tools/skottie-wasm-perf",
 		},
@@ -478,6 +486,7 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_SKPBENCH, &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/infra/bots/assets",
 			"skia/infra/bots/run_recipe.py",
 			"skia/tools/skpbench",
@@ -488,6 +497,7 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_TASK_DRIVERS, &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/go.mod",
 			"skia/go.sum",
 			"skia/infra/bots/build_task_drivers.sh",
@@ -499,6 +509,7 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_TEST, &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/infra/bots/assets",
 			"skia/infra/bots/run_recipe.py",
 			"skia/platform_tools/ios/bin",
@@ -510,6 +521,7 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_WASM_GM, &specs.CasSpec{
 		Root: "../skia", // Needed for other repos.
 		Paths: []string{
+			".vpython",
 			"resources",
 			"tools/run-wasm-gm-tests",
 		},
@@ -519,6 +531,7 @@ func GenTasks(cfg *Config) {
 	b.MustAddCasSpec(CAS_RECREATE_SKPS, &specs.CasSpec{
 		Root: "..",
 		Paths: []string{
+			"skia/.vpython",
 			"skia/DEPS",
 			"skia/bin/fetch-sk",
 			"skia/infra/bots/assets/skp",
@@ -578,6 +591,8 @@ func marshalJson(data interface{}) string {
 func (b *taskBuilder) kitchenTaskNoBundle(recipe string, outputDir string) {
 	b.cipd(CIPD_PKG_LUCI_AUTH)
 	b.cipd(cipd.MustGetPackage("infra/tools/luci/kitchen/${platform}"))
+	b.env("RECIPES_USE_PY3", "true")
+	b.envPrefixes("VPYTHON_DEFAULT_SPEC", "skia/.vpython")
 	b.usesPython()
 	b.recipeProp("swarm_out_dir", outputDir)
 	if outputDir != OUTPUT_NONE {
@@ -976,12 +991,12 @@ func (b *taskBuilder) defaultSwarmDimensions() {
 func (b *jobBuilder) bundleRecipes() string {
 	b.addTask(BUNDLE_RECIPES_NAME, func(b *taskBuilder) {
 		b.cipd(specs.CIPD_PKGS_GIT_LINUX_AMD64...)
-		b.cipd(specs.CIPD_PKGS_PYTHON_LINUX_AMD64...)
 		b.cmd("/bin/bash", "skia/infra/bots/bundle_recipes.sh", specs.PLACEHOLDER_ISOLATED_OUTDIR)
 		b.linuxGceDimensions(MACHINE_TYPE_SMALL)
-		b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin")
 		b.idempotent()
 		b.cas(CAS_RECIPES)
+		b.usesPython()
+		b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin")
 	})
 	return BUNDLE_RECIPES_NAME
 }
@@ -1630,14 +1645,14 @@ func (b *jobBuilder) fm() {
 			// Point sanitizer builds at our prebuilt libc++ for this sanitizer.
 			if b.extraConfig("MSAN") {
 				// We'd see false positives in std::basic_string<char> if this weren't set.
-				b.env("LD_LIBRARY_PATH", "clang_linux/msan")
+				b.envPrefixes("LD_LIBRARY_PATH", "clang_linux/msan")
 			} else if b.extraConfig("TSAN") {
 				// Occasional false positives may crop up in the standard library without this.
-				b.env("LD_LIBRARY_PATH", "clang_linux/tsan")
+				b.envPrefixes("LD_LIBRARY_PATH", "clang_linux/tsan")
 			} else {
 				// This isn't strictly required, but we usually get better sanitizer
 				// diagnostics from libc++ than the default OS-provided libstdc++.
-				b.env("LD_LIBRARY_PATH", "clang_linux/lib")
+				b.envPrefixes("LD_LIBRARY_PATH", "clang_linux/lib")
 			}
 		}
 	})
@@ -1900,7 +1915,7 @@ func (b *jobBuilder) presubmit() {
 		b.cipd(&specs.CipdPackage{
 			Name:    "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
 			Path:    "recipe_bundle",
-			Version: "git_revision:a8bcedad6768e206c4d2bd1718caa849f29cd42d",
+			Version: "git_revision:1a28cb094add070f4beefd052725223930d8c27a",
 		})
 	})
 }

infra/bots/gen_tasks_logic/task_builder.go

@@ -90,8 +90,17 @@ func (b *taskBuilder) cas(casSpec string) {
 	b.Spec.CasSpec = casSpec
 }
 
-// env appends the given values to the given environment variable for the task.
-func (b *taskBuilder) env(key string, values ...string) {
+// env sets the value for the given environment variable for the task.
+func (b *taskBuilder) env(key, value string) {
+	if b.Spec.Environment == nil {
+		b.Spec.Environment = map[string]string{}
+	}
+	b.Spec.Environment[key] = value
+}
+
+// envPrefixes appends the given values to the given environment variable for
+// the task.
+func (b *taskBuilder) envPrefixes(key string, values ...string) {
 	if b.Spec.EnvPrefixes == nil {
 		b.Spec.EnvPrefixes = map[string][]string{}
 	}
@@ -104,7 +113,7 @@ func (b *taskBuilder) env(key string, values ...string) {
 
 // addToPATH adds the given locations to PATH for the task.
 func (b *taskBuilder) addToPATH(loc ...string) {
-	b.env("PATH", loc...)
+	b.envPrefixes("PATH", loc...)
 }
 
 // output adds the given paths as outputs to the task, which results in their
@@ -218,7 +227,7 @@ func (b *taskBuilder) usesGo() {
 	}
 	b.cipd(pkg)
 	b.addToPATH(pkg.Path + "/go/bin")
-	b.env("GOROOT", pkg.Path+"/go")
+	b.envPrefixes("GOROOT", pkg.Path+"/go")
 }
 
 // usesDocker adds attributes to tasks which use docker.
@@ -303,7 +312,7 @@ func (b *taskBuilder) usesPython() {
 		Name: "vpython",
 		Path: "cache/vpython",
 	})
-	b.env("VPYTHON_VIRTUALENV_ROOT", "cache/vpython")
+	b.envPrefixes("VPYTHON_VIRTUALENV_ROOT", "cache/vpython")
 	b.env("VPYTHON_LOG_TRACE", "1")
 }
 

infra/bots/git_utils.py

@@ -23,7 +23,8 @@ def __init__(self, config_dict):
   def __enter__(self):
     for k, v in self._config_dict.items():
       try:
-        prev = subprocess.check_output(['git', 'config', '--local', k]).rstrip()
+        prev = subprocess.check_output([
+            'git', 'config', '--local', k]).decode('utf-8').rstrip()
         if prev:
           self._previous_values[k] = prev
       except subprocess.CalledProcessError:
@@ -60,7 +61,8 @@ def __init__(self, branch_name, commit_msg, upload=True, commit_queue=False,
   def __enter__(self):
     subprocess.check_call(['git', 'reset', '--hard', 'HEAD'])
     subprocess.check_call(['git', 'checkout', 'main'])
-    if self._branch_name in subprocess.check_output(['git', 'branch']).split():
+    if self._branch_name in subprocess.check_output([
+        'git', 'branch']).decode('utf-8').split():
       subprocess.check_call(['git', 'branch', '-D', self._branch_name])
     subprocess.check_call(['git', 'checkout', '-b', self._branch_name,
                            '-t', 'origin/main'])
@@ -81,7 +83,8 @@ def commit_and_upload(self, use_commit_queue=False):
     if self._cc_list:
       upload_cmd.extend(['--cc=%s' % ','.join(self._cc_list)])
     subprocess.check_call(upload_cmd)
-    output = subprocess.check_output(['git', 'cl', 'issue']).rstrip()
+    output = subprocess.check_output([
+        'git', 'cl', 'issue']).decode('utf-8').rstrip()
     return re.match('^Issue number: (?P<issue>\d+) \((?P<issue_url>.+)\)$',
                     output).group('issue_url')
 
@@ -145,7 +148,7 @@ def __enter__(self):
     remote = self._repository
     if self._local:
       remote = self._local
-    subprocess.check_output(args=['git', 'clone', remote])
+    subprocess.check_call(['git', 'clone', remote])
     repo_name = remote.split('/')[-1]
     if repo_name.endswith('.git'):
       repo_name = repo_name[:-len('.git')]

infra/bots/recipe_modules/build/__init__.py

@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-PYTHON_VERSION_COMPATIBILITY = "PY2+3"
+PYTHON_VERSION_COMPATIBILITY = "PY3"
 
 DEPS = [
   'depot_tools/gclient',

infra/bots/recipe_modules/build/examples/full.py

@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-PYTHON_VERSION_COMPATIBILITY = "PY2+3"
+PYTHON_VERSION_COMPATIBILITY = "PY3"
 
 DEPS = [
   'build',

infra/bots/recipe_modules/builder_name_schema/__init__.py

@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-PYTHON_VERSION_COMPATIBILITY = "PY2+3"
+PYTHON_VERSION_COMPATIBILITY = "PY3"
 
 DEPS = [
 ]

infra/bots/recipe_modules/builder_name_schema/examples/full.py

@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-PYTHON_VERSION_COMPATIBILITY = "PY2+3"
+PYTHON_VERSION_COMPATIBILITY = "PY3"
 
 DEPS = [
   'builder_name_schema',

infra/bots/recipe_modules/checkout/__init__.py

@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-PYTHON_VERSION_COMPATIBILITY = "PY2+3"
+PYTHON_VERSION_COMPATIBILITY = "PY3"
 
 DEPS = [
   'depot_tools/bot_update',