Python 2-3 compatible utility scripts

test/language/ignore_carriage_returns.wren

@@ -1,6 +1,6 @@
-// Sprinkle some carriage returns to ensure they are ignored anywhere:
-System.print("one")
-System.print("two")
-
-// Generate a compile error so we can test that the line number is correct.
-+ * // expect error
+// Sprinkle some carriage returns to ensure they are ignored anywhere:
+System.print("one")
+System.print("two")
+
+// Generate a compile error so we can test that the line number is correct.
++ * // expect error

util/generate_docs.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import codecs
-import glob
 import fnmatch
 import os
 import posixpath

util/metrics.py

@@ -2,9 +2,9 @@
 
 from __future__ import print_function
 
+import codecs
 import glob
 import fnmatch
-import itertools
 import os
 import re
 
@@ -86,7 +86,8 @@ def wren_metrics(label, directories):
       for file_name in fnmatch.filter(file_names, "*.wren"):
         num_files += 1
 
-        with open(os.path.join(dir_path, file_name), "r") as input:
+        # NOTE: In the old Python 2 version of the script, files were opened as ISO-8859-1
+        with codecs.open(os.path.join(dir_path, file_name), "r", encoding="utf-8", errors="ignore") as input:
           for line in input:
             if line.strip() == "":
               num_empty += 1

util/test.py

@@ -2,10 +2,11 @@
 
 from __future__ import print_function
 
+from io import open
 from argparse import ArgumentParser
 from collections import defaultdict
 from os import listdir
-from os.path import abspath, basename, dirname, isdir, isfile, join, realpath, relpath, splitext
+from os.path import abspath, dirname, isdir, join, realpath, relpath, splitext
 import re
 from subprocess import Popen, PIPE
 import sys
@@ -63,8 +64,16 @@ def parse(self):
 
     input_lines = []
     line_num = 1
-    with open(self.path, 'r') as file:
-      for line in file:
+    with open(self.path, 'rb') as file:
+
+      # NOTE: Using Unix-newlines only. Python 3 will otherwise interpret \r as a
+      # newline, causing test/language/ignore_carriage_returns.wren to fail.
+      for line in file.read().split(b'\n'):
+        try:
+          line = line.decode("utf-8")
+        except:
+          line = line.decode("latin-1")
+
         match = EXPECT_PATTERN.search(line)
         if match:
           self.output.append((match.group(1), line_num))
@@ -112,7 +121,6 @@ def parse(self):
 
         line_num += 1
 
-
     # If any input is fed to the test in stdin, concatenate it into one string.
     if input_lines:
       self.input_bytes = "\n".join(input_lines).encode("utf-8")
@@ -249,8 +257,6 @@ def validate_output(self, out):
 
     index = 0
     for line in out_lines:
-      if sys.version_info < (3, 0):
-        line = line.encode('utf-8')
 
       if index >= len(self.output):
         self.fail('Got output "{0}" when none was expected.', line)
@@ -267,7 +273,10 @@ def validate_output(self, out):
 
   def fail(self, message, *args):
     if args:
-      message = message.format(*args)
+      try:
+        message = (message).format(*args)
+      except:
+        print("Message:", message, "Args:", args)
     self.failures.append(message)
 
 

util/util.py

@@ -1,7 +1,5 @@
 # Utility functions used by other Python files in this directory.
 
-from __future__ import print_function
-
 import os
 import os.path
 import platform

util/wren_to_c_string.py

@@ -2,9 +2,7 @@
 # coding: utf-8
 
 import argparse
-import glob
 import os.path
-import re
 
 # The source for the Wren modules that are built into the VM or CLI are turned
 # include C string literals. This way they can be compiled directly into the