Issue 23607: first pass of changes for compatibility with Python3

etc/ci/clean_build_artifacts.sh

@@ -9,3 +9,4 @@ set -o nounset
 set -o pipefail
 
 rm -rf target/
+rm -rf python/_virtualenv/

python/requirements.txt

@@ -32,4 +32,7 @@ boto3 == 1.4.4
 # https://github.com/servo/servo/pull/18942
 certifi
 
+# For Python3 compatibility
+six
+
 -e python/tidy

python/servo/bootstrap.py

@@ -11,7 +11,7 @@
 import platform
 import shutil
 import subprocess
-import urllib
+import six.moves.urllib as urllib
 from subprocess import PIPE
 from zipfile import BadZipfile
 
@@ -293,7 +293,7 @@ def check_cmake(version):
 
     def prepare_file(zip_path, full_spec):
         if not os.path.isfile(zip_path):
-            zip_url = "{}{}.zip".format(deps_url, urllib.quote(full_spec))
+            zip_url = "{}{}.zip".format(deps_url, urllib.parse.quote(full_spec))
             download_file(full_spec, zip_url, zip_path)
 
         print("Extracting {}...".format(full_spec), end='')

python/servo/bootstrap_commands.py

@@ -18,7 +18,7 @@
 import subprocess
 import sys
 import traceback
-import urllib2
+import six.moves.urllib as urllib
 import glob
 
 from mach.decorators import (
@@ -220,7 +220,7 @@ def bootstrap_hsts_preload(self, force=False):
 
         try:
             content_base64 = download_bytes("Chromium HSTS preload list", chromium_hsts_url)
-        except urllib2.URLError:
+        except urllib.error.URLError:
             print("Unable to download chromium HSTS preload list; are you connected to the internet?")
             sys.exit(1)
 
@@ -244,7 +244,7 @@ def bootstrap_hsts_preload(self, force=False):
 
             with open(path.join(preload_path, preload_filename), 'w') as fd:
                 json.dump(entries, fd, indent=4)
-        except ValueError, e:
+        except ValueError as e:
             print("Unable to parse chromium HSTS preload list, has the format changed?")
             sys.exit(1)
 
@@ -258,7 +258,7 @@ def bootstrap_pub_suffix(self, force=False):
 
         try:
             content = download_bytes("Public suffix list", list_url)
-        except urllib2.URLError:
+        except urllib.error.URLError:
             print("Unable to download the public suffix list; are you connected to the internet?")
             sys.exit(1)
 

python/servo/build_commands.py

@@ -16,7 +16,7 @@
 import shutil
 import subprocess
 import sys
-import urllib
+import six.moves.urllib as urllib
 import zipfile
 import stat
 
@@ -484,7 +484,7 @@ def build(self, release=False, dev=False, jobs=None, params=None,
                 print("Downloading GStreamer dependencies")
                 gst_url = "https://servo-deps.s3.amazonaws.com/gstreamer/%s" % gst_lib_zip
                 print(gst_url)
-                urllib.urlretrieve(gst_url, gst_lib_zip)
+                urllib.request.urlretrieve(gst_url, gst_lib_zip)
                 zip_ref = zipfile.ZipFile(gst_lib_zip, "r")
                 zip_ref.extractall(gst_dir)
                 os.remove(gst_lib_zip)

python/servo/command_base.py

@@ -7,6 +7,8 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
+from __future__ import print_function
+
 from errno import ENOENT as NO_SUCH_FILE_OR_DIRECTORY
 from glob import glob
 import shutil
@@ -25,7 +27,7 @@
 import zipfile
 from xml.etree.ElementTree import XML
 from servo.util import download_file
-import urllib2
+import six.moves.urllib as urllib
 from bootstrap import check_gstreamer_lib
 
 from mach.decorators import CommandArgument
@@ -105,7 +107,7 @@ def reset(tarinfo):
         # packaging (in case of exceptional situations like running out of disk space).
         # TODO do this in a temporary folder after #11983 is fixed
         temp_file = '{}.temp~'.format(dest_archive)
-        with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0644), 'w') as out_file:
+        with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0o644), 'w') as out_file:
             if dest_archive.endswith('.zip'):
                 with zipfile.ZipFile(temp_file, 'w', zipfile.ZIP_DEFLATED) as zip_file:
                     for entry in file_list:
@@ -350,15 +352,15 @@ def call_rustup_run(self, args, **kwargs):
                 version_line = subprocess.check_output(["rustup" + BIN_SUFFIX, "--version"])
             except OSError as e:
                 if e.errno == NO_SUCH_FILE_OR_DIRECTORY:
-                    print "It looks like rustup is not installed. See instructions at " \
-                          "https://github.com/servo/servo/#setting-up-your-environment"
-                    print
+                    print("It looks like rustup is not installed. See instructions at "
+                          "https://github.com/servo/servo/#setting-up-your-environment")
+                    print()
                     return 1
                 raise
             version = tuple(map(int, re.match("rustup (\d+)\.(\d+)\.(\d+)", version_line).groups()))
             if version < (1, 11, 0):
-                print "rustup is at version %s.%s.%s, Servo requires 1.11.0 or more recent." % version
-                print "Try running 'rustup self update'."
+                print("rustup is at version %s.%s.%s, Servo requires 1.11.0 or more recent." % version)
+                print("Try running 'rustup self update'.")
                 return 1
             toolchain = self.toolchain()
             if platform.system() == "Windows":
@@ -504,15 +506,15 @@ def get_nightly_binary_path(self, nightly_date):
         nightly_date = nightly_date.strip()
         # Fetch the filename to download from the build list
         repository_index = NIGHTLY_REPOSITORY_URL + "?list-type=2&prefix=nightly"
-        req = urllib2.Request(
+        req = urllib.request.Request(
             "{}/{}/{}".format(repository_index, os_prefix, nightly_date))
         try:
-            response = urllib2.urlopen(req).read()
+            response = urllib.request.urlopen(req).read()
             tree = XML(response)
             namespaces = {'ns': tree.tag[1:tree.tag.index('}')]}
             file_to_download = tree.find('ns:Contents', namespaces).find(
                 'ns:Key', namespaces).text
-        except urllib2.URLError as e:
+        except urllib.error.URLError as e:
             print("Could not fetch the available nightly versions from the repository : {}".format(
                 e.reason))
             sys.exit(1)

python/servo/devenv_commands.py

@@ -14,7 +14,7 @@
 import signal
 import sys
 import tempfile
-import urllib2
+import six.moves.urllib as urllib
 import json
 import subprocess
 
@@ -201,7 +201,7 @@ def grep(self, params):
              category='devenv')
     def rustup(self):
         url = get_static_rust_lang_org_dist() + "/channel-rust-nightly-date.txt"
-        nightly_date = urllib2.urlopen(url, **get_urlopen_kwargs()).read()
+        nightly_date = urllib.request.urlopen(url, **get_urlopen_kwargs()).read()
         toolchain = "nightly-" + nightly_date
         filename = path.join(self.context.topdir, "rust-toolchain")
         with open(filename, "w") as f:

python/servo/lints/wpt_lint.py

@@ -7,6 +7,8 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
+from __future__ import print_function
+
 import os
 import sys
 
@@ -20,7 +22,7 @@ class Lint(LintRunner):
     def _get_wpt_files(self, suite):
         working_dir = os.path.join(WPT_PATH, suite, '')
         file_iter = self.get_files(working_dir, exclude_dirs=[])
-        print '\nRunning the WPT lint on %s...' % working_dir
+        print('\nRunning the WPT lint on %s...' % working_dir)
         for f in file_iter:
             if filter_file(f):
                 yield f[len(working_dir):]

python/servo/mutation/mutator.py

@@ -7,6 +7,8 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
+from __future__ import print_function
+
 import fileinput
 import re
 import random
@@ -28,7 +30,7 @@ def init_variables(if_blocks):
 def deleteStatements(file_name, line_numbers):
     for line in fileinput.input(file_name, inplace=True):
         if fileinput.lineno() not in line_numbers:
-            print line.rstrip()
+            print(line.rstrip())
 
 
 class Strategy:
@@ -48,7 +50,7 @@ def mutate(self, file_name):
             for line in fileinput.input(file_name, inplace=True):
                 if fileinput.lineno() == mutation_line_number:
                     line = re.sub(self._replace_strategy['regex'], self._replace_strategy['replaceString'], line)
-                print line.rstrip()
+                print(line.rstrip())
             return mutation_line_number
 
 

python/servo/package_commands.py

@@ -19,7 +19,7 @@
 import subprocess
 import sys
 import tempfile
-import urllib
+import six.moves.urllib as urllib
 
 from mach.decorators import (
     CommandArgument,
@@ -594,7 +594,7 @@ def get_taskcluster_secret(name):
                 "/secrets/v1/secret/project/servo/" +
                 name
             )
-            return json.load(urllib.urlopen(url))["secret"]
+            return json.load(urllib.request.urlopen(url))["secret"]
 
         def get_s3_secret():
             aws_access_key = None

python/servo/testing_commands.py

@@ -18,10 +18,11 @@
 from collections import OrderedDict
 import time
 import json
-import urllib2
+import six.moves.urllib as urllib
 import base64
 import shutil
 import subprocess
+from six import iteritems
 
 from mach.registrar import Registrar
 from mach.decorators import (
@@ -59,7 +60,7 @@
               "include_arg": "test_name"}),
 ])
 
-TEST_SUITES_BY_PREFIX = {path: k for k, v in TEST_SUITES.iteritems() if "paths" in v for path in v["paths"]}
+TEST_SUITES_BY_PREFIX = {path: k for k, v in iteritems(TEST_SUITES) if "paths" in v for path in v["paths"]}
 
 
 def create_parser_wpt():
@@ -158,7 +159,7 @@ def test(self, params, render_mode=DEFAULT_RENDER_MODE, release=False, tidy_all=
                 return 1
 
         test_start = time.time()
-        for suite, tests in selected_suites.iteritems():
+        for suite, tests in iteritems(selected_suites):
             props = suites[suite]
             kwargs = props.get("kwargs", {})
             if tests:
@@ -174,7 +175,7 @@ def test(self, params, render_mode=DEFAULT_RENDER_MODE, release=False, tidy_all=
     def suite_for_path(self, path_arg):
         if os.path.exists(path.abspath(path_arg)):
             abs_path = path.abspath(path_arg)
-            for prefix, suite in TEST_SUITES_BY_PREFIX.iteritems():
+            for prefix, suite in iteritems(TEST_SUITES_BY_PREFIX):
                 if abs_path.startswith(prefix):
                     return suite
         return None
@@ -510,9 +511,9 @@ def filter_intermittents(self, summary, log_filteredsummary, log_intermittents,
                 elif tracker_api.endswith('/'):
                     tracker_api = tracker_api[0:-1]
 
-                query = urllib2.quote(failure['test'], safe='')
-                request = urllib2.Request("%s/query.py?name=%s" % (tracker_api, query))
-                search = urllib2.urlopen(request)
+                query = urllib.parse.quote(failure['test'], safe='')
+                request = urllib.request.Request("%s/query.py?name=%s" % (tracker_api, query))
+                search = urllib.request.urlopen(request)
                 data = json.load(search)
                 if len(data) == 0:
                     actual_failures += [failure]
@@ -521,11 +522,11 @@ def filter_intermittents(self, summary, log_filteredsummary, log_intermittents,
             else:
                 qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test']
                 # we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+`
-                query = urllib2.quote(qstr, safe='+')
-                request = urllib2.Request("https://api.github.com/search/issues?q=%s" % query)
+                query = urllib.parse.quote(qstr, safe='+')
+                request = urllib.request.Request("https://api.github.com/search/issues?q=%s" % query)
                 if encoded_auth:
                     request.add_header("Authorization", "Basic %s" % encoded_auth)
-                search = urllib2.urlopen(request)
+                search = urllib.request.urlopen(request)
                 data = json.load(search)
                 if data['total_count'] == 0:
                     actual_failures += [failure]

python/servo/util.py

@@ -16,11 +16,11 @@
 import shutil
 from socket import error as socket_error
 import stat
-import StringIO
+from io import BytesIO
 import sys
 import time
 import zipfile
-import urllib2
+import six.moves.urllib as urllib
 
 
 try:
@@ -101,10 +101,10 @@ def download(desc, src, writer, start_byte=0):
     dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())
 
     try:
-        req = urllib2.Request(src)
+        req = urllib.request.Request(src)
         if start_byte:
-            req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
-        resp = urllib2.urlopen(req, **get_urlopen_kwargs())
+            req = urllib.request.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
+        resp = urllib.request.urlopen(req, **get_urlopen_kwargs())
 
         fsize = None
         if resp.info().getheader('Content-Length'):
@@ -136,16 +136,16 @@ def download(desc, src, writer, start_byte=0):
 
         if not dumb:
             print()
-    except urllib2.HTTPError, e:
+    except urllib.error.HTTPError as e:
         print("Download failed ({}): {} - {}".format(e.code, e.reason, src))
         if e.code == 403:
             print("No Rust compiler binary available for this platform. "
                   "Please see https://github.com/servo/servo/#prerequisites")
         sys.exit(1)
-    except urllib2.URLError, e:
+    except urllib.error.URLError as e:
         print("Error downloading {}: {}. The failing URL was: {}".format(desc, e.reason, src))
         sys.exit(1)
-    except socket_error, e:
+    except socket_error as e:
         print("Looks like there's a connectivity issue, check your Internet connection. {}".format(e))
         sys.exit(1)
     except KeyboardInterrupt:
@@ -154,7 +154,7 @@ def download(desc, src, writer, start_byte=0):
 
 
 def download_bytes(desc, src):
-    content_writer = StringIO.StringIO()
+    content_writer = BytesIO()
     download(desc, src, content_writer)
     return content_writer.getvalue()