Bug 1632916 - Run JS/web-platform/ipdl build machinery in Python 3 r=jgraham,nika,glandium

Differential Revision: https://phabricator.services.mozilla.com/D72478

Author: Ricky Stewart

config/check_vanilla_allocations.py

@@ -38,8 +38,7 @@
 # mismatched alloc/free checking.
 # ----------------------------------------------------------------------------
 
-from __future__ import absolute_import
-from __future__ import print_function
+from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import re
@@ -112,7 +111,7 @@ def main():
         ]
 
     # This is like alloc_fns, but regexp chars are not escaped.
-    alloc_fns_unescaped = [fn.translate(None, r'\\') for fn in alloc_fns]
+    alloc_fns_unescaped = [fn.replace('\\', '') for fn in alloc_fns]
 
     # This regexp matches the relevant lines in the output of |nm|, which look
     # like the following.

config/config.mk

@@ -112,7 +112,7 @@ MKDIR ?= mkdir
 SLEEP ?= sleep
 TOUCH ?= touch
 
-PYTHON_PATH = $(PYTHON) $(topsrcdir)/config/pythonpath.py
+PYTHON_PATH = $(PYTHON3) $(topsrcdir)/config/pythonpath.py
 
 #
 # Build using PIC by default

config/pythonpath.py

@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import
-from __future__ import print_function
+from __future__ import absolute_import, print_function, unicode_literals
+
+
 """
 Run a python script, adding extra directories to the python path.
 """
 
 
 def main(args):
     def usage():
-        print >>sys.stderr, "pythonpath.py -I directory script.py [args...]"
+        print("pythonpath.py -I directory script.py [args...]", file=sys.stderr)
         sys.exit(150)
 
     paths = []
@@ -47,14 +48,14 @@ def usage():
     frozenglobals['__name__'] = '__main__'
     frozenglobals['__file__'] = script
 
-    execfile(script, frozenglobals)
+    exec(open(script, encoding='utf-8').read(), frozenglobals)
 
 
 # Freeze scope here ... why this makes things work I have no idea ...
 frozenglobals = globals()
 
-import sys
 import os
+import sys
 
 if __name__ == '__main__':
     main(sys.argv[1:])

ipc/ipdl/Makefile.in

@@ -22,7 +22,7 @@ ipdl_py_deps := \
 # NB: the IPDL compiler manages .ipdl-->.h/.cpp dependencies itself,
 # which is why we don't have explicit .h/.cpp targets here
 ipdl.track: $(ALL_IPDLSRCS) $(srcdir)/sync-messages.ini $(srcdir)/message-metadata.ini $(ipdl_py_deps)
-	$(PYTHON) $(topsrcdir)/config/pythonpath.py \
+	$(PYTHON3) $(topsrcdir)/config/pythonpath.py \
 	  $(PLY_INCLUDE) \
 	  $(srcdir)/ipdl.py \
 	  --sync-msg-list=$(srcdir)/sync-messages.ini \

ipc/ipdl/ipdl.py

@@ -3,11 +3,11 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 from __future__ import print_function
 
+from io import StringIO
 import optparse
 import os
 import sys
-from cStringIO import StringIO
-from ConfigParser import RawConfigParser
+from configparser import RawConfigParser
 
 import ipdl
 

ipc/ipdl/ipdl/__init__.py

@@ -9,7 +9,7 @@
 
 import os
 import sys
-from cStringIO import StringIO
+from io import StringIO
 
 from ipdl.cgen import IPDLCodeGen
 from ipdl.lower import LowerToCxx, msgenums
@@ -75,6 +75,7 @@ def genmsgenum(ast):
 
 
 def writeifmodified(contents, file):
+    contents = contents.encode('utf-8')
     dir = os.path.dirname(file)
     os.path.exists(dir) or os.makedirs(dir)
 

ipc/ipdl/ipdl/cxx/ast.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import copy
+import functools
 
 
 class Visitor:
@@ -442,6 +443,7 @@ def __init__(self, params=[], ret=Type('void')):
         self.ret = ret
 
 
+@functools.total_ordering
 class Typedef(Node):
     def __init__(self, fromtype, totypename, templateargs=[]):
         assert isinstance(totypename, str)
@@ -451,12 +453,12 @@ def __init__(self, fromtype, totypename, templateargs=[]):
         self.totypename = totypename
         self.templateargs = templateargs
 
-    def __cmp__(self, o):
-        return cmp(self.totypename, o.totypename)
+    def __lt__(self, other):
+        return self.totypename < other.totypename
 
-    def __eq__(self, o):
-        return (self.__class__ == o.__class__
-                and 0 == cmp(self, o))
+    def __eq__(self, other):
+        return (self.__class__ == other.__class__
+                and self.totypename == other.totypename)
 
     def __hash__(self):
         return hash(self.totypename)

ipc/ipdl/ipdl/cxx/code.py

@@ -11,7 +11,6 @@
 # API are often easier to read than users of the AST APIs in these cases.
 
 import re
-import sys
 import math
 import textwrap
 
@@ -140,7 +139,7 @@ def _line(raw, skip_indent, lineno, context):
             values = eval(expr, context, {})
         except Exception as e:
             msg = "%s in substitution on line %d" % (repr(e), lineno)
-            raise ValueError(msg), None, sys.exc_traceback
+            raise ValueError(msg) from e
 
         # If we aren't dealing with lists, wrap the result into a
         # single-element list.

ipc/ipdl/ipdl/lower.py

@@ -1450,7 +1450,7 @@ def visitStructDecl(self, sd):
 
             # Compute a permutation of the fields for in-memory storage such
             # that the memory layout of the structure will be well-packed.
-            permutation = range(len(newfields))
+            permutation = list(range(len(newfields)))
 
             # Note that the results of `pod_size` ensure that non-POD fields
             # sort before POD ones.

ipc/ipdl/ipdl/type.py

@@ -125,10 +125,10 @@ def typename(self):
         return self.__class__.__name__
 
     def name(self):
-        raise NotImplementedError
+        raise NotImplementedError()
 
     def fullname(self):
-        raise NotImplementedError
+        raise NotImplementedError()
 
     def accept(self, visitor, *args):
         visit = getattr(visitor, 'visit' + self.__class__.__name__, None)
@@ -229,8 +229,18 @@ def hasBaseType(self): return False
 
     @classmethod
     def convertsTo(cls, lesser, greater):
-        if (lesser.nestedRange[0] < greater.nestedRange[0] or
-                lesser.nestedRange[1] > greater.nestedRange[1]):
+        def _unwrap(nr):
+            if isinstance(nr, dict):
+                return _unwrap(nr['nested'])
+            elif isinstance(nr, int):
+                return nr
+            else:
+                raise ValueError('Got unexpected nestedRange value: %s' % nr)
+
+        lnr0, gnr0, lnr1, gnr1 = (
+            _unwrap(lesser.nestedRange[0]), _unwrap(greater.nestedRange[0]),
+            _unwrap(lesser.nestedRange[1]), _unwrap(greater.nestedRange[1]))
+        if (lnr0 < gnr0 or lnr1 > gnr1):
             return False
 
         # Protocols that use intr semantics are not allowed to use
@@ -593,7 +603,7 @@ def iteractortypes(t, visited=None):
 
 def hasshmem(type):
     """Return true iff |type| is shmem or has it buried within."""
-    class found:
+    class found(BaseException):
         pass
 
     class findShmem(TypeVisitor):

ipc/ipdl/test/cxx/Makefile.in

@@ -20,7 +20,7 @@ include $(topsrcdir)/config/rules.mk
 IPDLUNITTEST_BIN = $(DEPTH)/dist/bin/ipdlunittest$(BIN_SUFFIX)
 
 IPDLUnitTests.cpp : Makefile.in moz.build $(GENTESTER) $(TESTER_TEMPLATE) $(IPDLTESTHDRS)
-	$(PYTHON) $(GENTESTER) $(TESTER_TEMPLATE) -t $(IPDLTESTS) -e $(EXTRA_PROTOCOLS) > $@
+	$(PYTHON3) $(GENTESTER) $(TESTER_TEMPLATE) -t $(IPDLTESTS) -e $(EXTRA_PROTOCOLS) > $@
 
 check-proc::
 	@$(EXIT_ON_ERROR)  \

ipc/ipdl/test/cxx/genIPDLUnitTests.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 import string
 import sys
@@ -124,7 +124,7 @@ def main(argv):
     }
 ''' % (t, t, t, t) for t in unittests+extras])
 
-    templatefile = open(template, 'r')
+    templatefile = open(template, 'r', encoding='utf-8')
     sys.stdout.write(
         string.Template(templatefile.read()).substitute(
             INCLUDES=includes,

ipc/ipdl/test/ipdl/IPDLCompile.py

@@ -28,9 +28,9 @@ def run(self):
                 self.specfilename
             ])
 
-            proc = subprocess.Popen(args=self.argv,
-                                    stdout=subprocess.PIPE,
-                                    stderr=subprocess.PIPE)
+            proc = subprocess.Popen(
+                args=self.argv, stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE, universal_newlines=True)
             self.stdout, self.stderr = proc.communicate()
 
             self.returncode = proc.returncode

ipc/ipdl/test/ipdl/Makefile.in

@@ -8,9 +8,9 @@ OKTESTS := $(wildcard $(srcdir)/ok/*.ipdl) $(wildcard $(srcdir)/ok/*.ipdlh)
 ERRORTESTS := $(wildcard $(srcdir)/error/*.ipdl) $(wildcard $(srcdir)/error/*.ipdlh)
 
 check::
-	@$(PYTHON) $(srcdir)/runtests.py  \
+	@$(PYTHON3) $(srcdir)/runtests.py  \
 		$(srcdir)/ok $(srcdir)/error  \
-		$(PYTHON) $(topsrcdir)/config/pythonpath.py  \
+		$(PYTHON3) $(topsrcdir)/config/pythonpath.py  \
 		$(PLY_INCLUDE)  \
 		$(topsrcdir)/ipc/ipdl/ipdl.py  \
 		--sync-msg-list=$(srcdir)/sync-messages.ini \

ipc/ipdl/test/ipdl/error/BadNestedManagee.ipdl

@@ -0,0 +1,9 @@
+//error: protocol `PBadNestedManagee' requires more powerful send semantics than its manager `PBadNestedManager' provides
+
+include protocol PBadNestedManager;
+
+nested(upto inside_sync) async protocol PBadNestedManagee {
+    manager PBadNestedManager;
+child:
+    async __delete__();
+};

ipc/ipdl/test/ipdl/error/BadNestedManager.ipdl

@@ -0,0 +1,9 @@
+//error: protocol `PBadNestedManagee' requires more powerful send semantics than its manager `PBadNestedManager' provides
+
+include protocol PBadNestedManagee;
+
+nested(upto not) async protocol PBadNestedManager {
+    manages PBadNestedManagee;
+parent:
+    async PBadNestedManagee();
+};

ipc/ipdl/test/ipdl/error/PBadNestedManagee.ipdl

@@ -40,10 +40,10 @@ endif
 endif
 
 check-js-msg::
-	(cd $(topsrcdir) && $(PYTHON) $(topsrcdir)/config/check_js_msg_encoding.py);
+	(cd $(topsrcdir) && $(PYTHON3) $(topsrcdir)/config/check_js_msg_encoding.py);
 
 check-jit-test::
-	$(JITTEST_SANITIZER_ENV) $(wildcard $(RUN_TEST_PROGRAM)) $(PYTHON) -u $(srcdir)/jit-test/jit_test.py \
+	$(JITTEST_SANITIZER_ENV) $(wildcard $(RUN_TEST_PROGRAM)) $(PYTHON3) -u $(srcdir)/jit-test/jit_test.py \
 	        --no-slow --no-progress --format=automation --jitflags=all \
 			$(JITTEST_VALGRIND_FLAG) \
 			$(JITTEST_EXTRA_ARGS) \
@@ -52,7 +52,7 @@ check-jit-test::
 check:: check-js-msg
 
 check-jstests:
-	$(wildcard $(RUN_TEST_PROGRAM)) $(PYTHON) -u $(srcdir)/tests/jstests.py \
+	$(wildcard $(RUN_TEST_PROGRAM)) $(PYTHON3) -u $(srcdir)/tests/jstests.py \
 		--no-progress --format=automation --timeout 300 \
 		$(JSTESTS_EXTRA_ARGS) \
 		$(DIST)/bin/$(JS_SHELL_NAME)$(BIN_SUFFIX)

ipc/ipdl/test/ipdl/error/PBadNestedManager.ipdl

@@ -9,12 +9,12 @@ include $(topsrcdir)/config/rules.mk
 # check_vanilla_allocations.py is tailored to Linux, so only run it there.
 # That should be enough to catch any problems.
 check-vanilla-allocations:
-	$(PYTHON) $(topsrcdir)/config/check_vanilla_allocations.py $(REAL_LIBRARY)
+	$(PYTHON3) $(topsrcdir)/config/check_vanilla_allocations.py $(REAL_LIBRARY)
 
 # The "aggressive" variant will likely fail on some compiler/platform
 # combinations, but is worth running by hand every once in a while.
 check-vanilla-allocations-aggressive:
-	$(PYTHON) $(topsrcdir)/config/check_vanilla_allocations.py --aggressive $(REAL_LIBRARY)
+	$(PYTHON3) $(topsrcdir)/config/check_vanilla_allocations.py --aggressive $(REAL_LIBRARY)
 
 ifeq ($(OS_ARCH),Linux)
 ifeq (,$(filter -flto,$(COMPUTED_CFLAGS) $(COMPUTED_CXXFLAGS) $(COMPUTED_LDFLAGS)))

js/src/Makefile.in

@@ -24,4 +24,4 @@ PKG_STAGE = $(DIST)/test-stage
 stage-package:
 	$(NSINSTALL) -D $(PKG_STAGE)/jsreftest/tests/js/src/tests
 	(cd $(srcdir) && tar $(TAR_CREATE_FLAGS) - $(TEST_FILES)) | (cd $(PKG_STAGE)/jsreftest/tests/js/src/tests && tar -xf -)
-	$(PYTHON) $(srcdir)/jstests.py --make-manifests $(PKG_STAGE)/jsreftest/tests/js/src/tests
+	$(PYTHON3) $(srcdir)/jstests.py --make-manifests $(PKG_STAGE)/jsreftest/tests/js/src/tests

js/src/build/Makefile.in

@@ -4,6 +4,7 @@
 
 from __future__ import print_function
 
+import io
 import os
 import posixpath
 import re
@@ -71,7 +72,7 @@ def create(cls, jsdir):
         # Read the values.
         val_re = re.compile(r'(TARGET_XPCOM_ABI|OS_TARGET|MOZ_DEBUG)\s*=\s*(.*)')
         kw = {'isdebug': False}
-        for line in open(path):
+        for line in io.open(path, encoding='utf-8'):
             m = val_re.match(line)
             if m:
                 key, val = m.groups()
@@ -338,7 +339,7 @@ def _emit_manifest_at(location, relative, test_gen, depth):
         manifest = ["url-prefix {}jsreftest.html?test={}/".format(
             '../' * depth, relative)] + manifest
 
-    fp = open(filename, 'w')
+    fp = io.open(filename, 'w', encoding='utf-8', newline='\n')
     try:
         fp.write('\n'.join(manifest) + '\n')
     finally:
@@ -378,7 +379,7 @@ def _parse_test_header(fullpath, testcase, xul_tester):
     This looks a bit weird.  The reason is that it needs to be efficient, since
     it has to be done on every test
     """
-    fp = open(fullpath, 'r')
+    fp = io.open(fullpath, 'r', encoding='utf-8')
     try:
         buf = fp.read(512)
     finally:
@@ -415,7 +416,7 @@ def _parse_external_manifest(filename, relpath):
 
     entries = []
 
-    with open(filename, 'r') as fp:
+    with io.open(filename, 'r', encoding='utf-8') as fp:
         manifest_re = re.compile(r'^\s*(?P<terms>.*)\s+(?P<type>include|script)\s+(?P<path>\S+)$')
         include_re = re.compile(r'^\s*include\s+(?P<path>\S+)$')
         for line in fp: