Reland "py2-3: more stage2 fixes (#406). (#472)" (#476)" (#494)

* Reland "py2-3: more stage2 fixes (#406). (#472)" (#476)"

This reverts commit 80dac9a.

docker/oss-fuzz/host/start_host.py

@@ -13,6 +13,10 @@
 # limitations under the License.
 """Start host."""
 from __future__ import print_function
+
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
 import os
 import shutil
 import socket

src/appengine/handlers/cron/oss_fuzz_setup.py

@@ -15,6 +15,7 @@
 from __future__ import absolute_import
 
 from builtins import object
+from past.builtins import basestring
 import base64
 import copy
 import json

src/appengine/handlers/fuzzers.py

@@ -13,8 +13,10 @@
 # limitations under the License.
 """Manage fuzzers types."""
 
+from future import standard_library
+standard_library.install_aliases()
 import datetime
-import StringIO
+import io
 
 from base import utils
 from datastore import data_handler
@@ -73,13 +75,13 @@ def get(self):
 class BaseEditHandler(base_handler.GcsUploadHandler):
   """Base edit handler."""
 
-  def _read_to_stringio(self, gcs_path):
-    """Return a StringIO representing a GCS object."""
+  def _read_to_bytesio(self, gcs_path):
+    """Return a bytesio representing a GCS object."""
     data = storage.read_data(gcs_path)
     if not data:
       raise helpers.EarlyExitException('Failed to read uploaded archive.', 500)
 
-    return StringIO.StringIO(data)
+    return io.BytesIO(data)
 
   def _get_executable_path(self, upload_info):
     """Get executable path."""
@@ -93,7 +95,7 @@ def _get_executable_path(self, upload_info):
     if not executable_path:
       executable_path = 'run'  # Check for default.
 
-    reader = self._read_to_stringio(upload_info.gcs_path)
+    reader = self._read_to_bytesio(upload_info.gcs_path)
     return archive.get_first_file_matching(executable_path, reader,
                                            upload_info.filename)
 
@@ -109,7 +111,7 @@ def _get_launcher_script(self, upload_info):
     if upload_info.size > ARCHIVE_READ_SIZE_LIMIT:
       return launcher_script
 
-    reader = self._read_to_stringio(upload_info.gcs_path)
+    reader = self._read_to_bytesio(upload_info.gcs_path)
     launcher_script = archive.get_first_file_matching(launcher_script, reader,
                                                       upload_info.filename)
     if not launcher_script:

src/appengine/handlers/upload_testcase.py

@@ -13,12 +13,14 @@
 # limitations under the License.
 """Handler that uploads a testcase"""
 
+from future import standard_library
+standard_library.install_aliases()
 from builtins import object
 import ast
 import cgi
 import datetime
+import io
 import json
-import StringIO
 
 from base import external_users
 from base import tasks
@@ -105,19 +107,19 @@ def get_result(this):
   return result, params
 
 
-def _read_to_stringio(gcs_path):
-  """Return a StringIO representing a GCS object."""
+def _read_to_bytesio(gcs_path):
+  """Return a bytesio representing a GCS object."""
   data = storage.read_data(gcs_path)
   if not data:
     raise helpers.EarlyExitException('Failed to read uploaded archive.', 500)
 
-  return StringIO.StringIO(data)
+  return io.BytesIO(data)
 
 
 def guess_input_file(uploaded_file, filename):
   """Guess the main test case file from an archive."""
   for file_pattern in RUN_FILE_PATTERNS:
-    blob_reader = _read_to_stringio(uploaded_file.gcs_path)
+    blob_reader = _read_to_bytesio(uploaded_file.gcs_path)
     file_path_input = archive.get_first_file_matching(file_pattern, blob_reader,
                                                       filename)
     if file_path_input:
@@ -525,12 +527,12 @@ def post(self):
     self.do_post()
 
 
-class NamedStringIO(StringIO.StringIO):
-  """Named StringIO."""
+class NamedBytesIO(io.BytesIO):
+  """Named bytesio."""
 
   def __init__(self, name, value):
     self.name = name
-    StringIO.StringIO.__init__(self, value)
+    io.BytesIO.__init__(self, value)
 
 
 class UploadHandlerOAuth(base_handler.Handler, UploadHandlerCommon):
@@ -547,8 +549,8 @@ def get_upload(self):
     if not isinstance(uploaded_file, cgi.FieldStorage):
       raise helpers.EarlyExitException('File upload not found.', 400)
 
-    string_io = NamedStringIO(uploaded_file.filename,
-                              uploaded_file.file.getvalue())
+    string_io = NamedBytesIO(uploaded_file.filename,
+                             uploaded_file.file.getvalue())
     key = blobs.write_blob(string_io)
     return blobs.get_blob_info(key)
 

src/local/butler/common.py

@@ -19,11 +19,12 @@
 from builtins import object
 from future import standard_library
 standard_library.install_aliases()
+from past.builtins import basestring
 import datetime
+import io
 import os
 import platform
 import shutil
-import StringIO
 import subprocess
 import sys
 import tempfile
@@ -212,7 +213,7 @@ def _install_chromedriver():
   archive_request = urllib.request.urlopen(
       constants.CHROMEDRIVER_DOWNLOAD_PATTERN.format(
           version=version, archive_name=archive_name))
-  archive_io = StringIO.StringIO(archive_request.read())
+  archive_io = io.BytesIO(archive_request.read())
   chromedriver_archive = zipfile.ZipFile(archive_io)
 
   chromedriver_path = get_chromedriver_path()

src/local/butler/generate_datastore_models.py

@@ -16,6 +16,7 @@
 """Script to convert python Datastore types to Go and protobufs."""
 
 from builtins import object
+from past.builtins import basestring
 import ast
 import inspect
 import os
@@ -187,7 +188,7 @@ def capitalize_acronyms(name):
 
 def py_to_go_type(py_type):
   """Return the Go equivalent for a python type."""
-  if py_type in [str, unicode, basestring]:
+  if isinstance(py_type, basestring):
     return 'string'
 
   if py_type in six.integer_types:

src/local/butler/py_unittest.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """py_unittest.py runs tests under src/appengine and butler/tests"""
 from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
 from builtins import object
 from builtins import range
 import coverage
@@ -21,13 +23,13 @@
 COV = coverage.Coverage(config_file='.coveragerc')
 COV.start()
 
+import io
 import itertools
 import logging
 import multiprocessing
 import os
 import platform
 import signal
-import StringIO
 import sys
 import time
 import unittest
@@ -102,7 +104,7 @@ def __exit__(self, exc_type, value, traceback):
     COV.html_report(directory='coverage')
 
     print('The tests cover %0.2f%% of the source code.' %
-          COV.report(file=StringIO.StringIO()))
+          COV.report(file=io.BytesIO()))
     print('The test coverage by lines can be seen at ./coverage/index.html')
 
 
@@ -131,7 +133,7 @@ def run_one_test_parallel(args):
   test_modules, suppress_output = args
   suite = unittest.loader.TestLoader().loadTestsFromNames(test_modules)
 
-  stream = StringIO.StringIO()
+  stream = io.BytesIO()
 
   # Verbosity=0 since we cannot see real-time test execution order when tests
   # are executed in parallel.

src/python/base/utils.py

@@ -13,9 +13,10 @@
 # limitations under the License.
 """Common utility functions."""
 
+from builtins import range
 from future import standard_library
 standard_library.install_aliases()
-from builtins import range
+from past.builtins import basestring
 import ast
 import datetime
 import functools
@@ -27,8 +28,8 @@
 import requests
 import sys
 import time
+import urllib.parse
 import urllib.request
-import urlparse
 import weakref
 
 from base import errors
@@ -151,7 +152,7 @@ def file_path_to_file_url(path):
     return ''
 
   path = path.lstrip(WINDOWS_PREFIX_PATH)
-  return urlparse.urljoin('file:', urllib.request.pathname2url(path))
+  return urllib.parse.urljoin(u'file:', urllib.request.pathname2url(path))
 
 
 def filter_file_list(file_list):

src/python/bot/fuzzers/options.py

@@ -13,8 +13,10 @@
 # limitations under the License.
 """Fuzzer options."""
 
+from future import standard_library
+standard_library.install_aliases()
 from builtins import object
-import ConfigParser
+import configparser
 import os
 import random
 import re
@@ -88,11 +90,12 @@ def __init__(self, options_file_path, cwd=None):
     else:
       self._cwd = os.path.dirname(options_file_path)
 
-    self._config = ConfigParser.ConfigParser()
-    try:
-      self._config.read(options_file_path)
-    except ConfigParser.Error:
-      raise FuzzerOptionsException('Failed to parse fuzzer options file.')
+    self._config = configparser.ConfigParser()
+    with open(options_file_path, 'r') as f:
+      try:
+        self._config.read_file(f)
+      except configparser.Error:
+        raise FuzzerOptionsException('Failed to parse fuzzer options file.')
 
   def _get_dict_path(self, relative_dict_path):
     """Return a full path to the dictionary."""

src/python/bot/minimizer/minimizer.py

@@ -14,6 +14,8 @@
 """Base classes for other minimizers."""
 from __future__ import absolute_import
 
+from future import standard_library
+standard_library.install_aliases()
 from builtins import object
 from builtins import range
 import copy

src/python/bot/startup/run_bot.py

@@ -32,6 +32,8 @@
 from python.base import modules
 modules.fix_module_search_paths()
 
+from future import standard_library
+standard_library.install_aliases()
 from builtins import object
 import os
 import time

src/python/bot/tasks/analyze_task.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Analyze task for handling user uploads."""
 
+from future import standard_library
+standard_library.install_aliases()
 import datetime
 import os
 

src/python/bot/tasks/fuzz_task.py

@@ -14,6 +14,8 @@
 """Fuzz task for handling fuzzing."""
 from __future__ import division
 
+from future import standard_library
+standard_library.install_aliases()
 from builtins import object
 import collections
 import datetime

src/python/bot/webserver/http_server.py

@@ -13,7 +13,9 @@
 # limitations under the License.
 """Runs http(s) server in the background."""
 
-import BaseHTTPServer
+from future import standard_library
+standard_library.install_aliases()
+import http.server
 import mimetypes
 import os
 import socket
@@ -68,11 +70,11 @@ def guess_mime_type(filename):
   return mimetypes.guess_type(filename)[0]
 
 
-class BotHTTPServer(BaseHTTPServer.HTTPServer):
+class BotHTTPServer(http.server.HTTPServer):
   """Host the bot's test case directories over HTTP."""
 
   def __init__(self, server_address, handler_class):
-    BaseHTTPServer.HTTPServer.__init__(self, server_address, handler_class)
+    http.server.HTTPServer.__init__(self, server_address, handler_class)
 
   def _handle_request_noblock(self):
     """Process a single http request."""
@@ -87,7 +89,7 @@ def _handle_request_noblock(self):
         self.close_request(request)
 
 
-class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+class RequestHandler(http.server.BaseHTTPRequestHandler):
   """Handler for get requests to test cases."""
 
   def do_GET(self):  # pylint: disable=invalid-name

src/python/build_management/revisions.py

@@ -16,6 +16,7 @@
 from future import standard_library
 standard_library.install_aliases()
 from builtins import range
+from past.builtins import basestring
 import base64
 import bisect
 import json
@@ -25,7 +26,6 @@
 import six
 import time
 import urllib.parse
-import urlparse
 
 from base import memoize
 from base import utils
@@ -202,7 +202,7 @@ def _is_clank(url):
 
 def _is_deps(url):
   """Return bool on whether this is a DEPS url or not."""
-  return urlparse.urlparse(url).path.endswith('/DEPS')
+  return urllib.parse.urlparse(url).path.endswith('/DEPS')
 
 
 def _src_map_to_revisions_dict(src_map, default_project_name):

src/python/chrome/crash_uploader.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Crash minidump and symbols uploader."""
 
+from future import standard_library
+standard_library.install_aliases()
 from builtins import object
 from builtins import range
 import email

src/python/google_cloud_utils/blobs.py

@@ -14,6 +14,7 @@
 """Blobs handling."""
 from __future__ import absolute_import
 
+from past.builtins import basestring
 import os
 import re
 import uuid

src/python/google_cloud_utils/compute_engine_projects.py

@@ -14,6 +14,7 @@
 """Load project data."""
 
 from builtins import object
+from past.builtins import basestring
 import os
 import six