Convert the codebase to Python 3

Python 2 will be deprecated on January 1, 2020.  Most Linux
distributions have either already switched to Python 3 as default, or
are planning to soon.

Most changes here are just renamings or basic syntax, thankfully most
op-test code is not particularly problematic here, outside of pexpect.

The only manual changes made other than running 2to3:

  - removed the "async" arg from expect()
      it was unused and async is a keyword in Python 3
  - no need to set the default encoding to UTF-8 in ./op-test
  - ignore "__pycache__" directory in addons/
  - convert some strings to byte types
  - rename __del__() to cleanup()
      python3 prevents you from calling open() in __del__ handlers,
      which happens if you try to log.  we already register these with
      atexit, which works fine
  - call pexpect.spawn with encoding='utf-8'
  - open logfile with text=True
  - petitboot-i18n converted to *actually* unicode

There WILL be breakages, please report them or fix them.  This is a
necessary change and the sooner we do it, the better.

Signed-off-by: Russell Currey <ruscur@russell.cc>

OpTestConfiguration.py

@@ -26,7 +26,7 @@
 from datetime import datetime
 import subprocess
 import sys
-import ConfigParser
+import configparser
 import errno
 import OpTestLogger
 import logging
@@ -433,17 +433,19 @@ def __init__(self):
 
         self.util_server = None  # Hostlocker or AES
         self.util_bmc_server = None  # OpenBMC REST Server
-        atexit.register(self.__del__)  # allows cleanup handler to run (OpExit)
+        atexit.register(self.cleanup)  # allows cleanup handler to run (OpExit)
         self.firmware_versions = None
         self.nvram_debug_opts = None
 
-        for dir in (os.walk(os.path.join(self.basedir, 'addons')).next()[1]):
-            optAddons[dir] = importlib.import_module(
-                "addons." + dir + ".OpTest" + dir + "Setup")
+        for dirname in (next(os.walk(os.path.join(self.basedir, 'addons')))[1]):
+            if dirname == "__pycache__":
+                continue
+            optAddons[dirname] = importlib.import_module(
+                "addons." + dirname + ".OpTest" + dirname + "Setup")
 
         return
 
-    def __del__(self):
+    def cleanup(self):
         if self.atexit_ready:
             # calling cleanup before args initialized pointless
             # attribute errors thrown in cleanup, e.g. ./op-test -h
@@ -458,7 +460,7 @@ def parse_args(self, argv=None):
 
         args, remaining_args = conf_parser.parse_known_args(argv)
         defaults = {}
-        config = ConfigParser.SafeConfigParser()
+        config = configparser.SafeConfigParser()
         config.read([os.path.expanduser("~/.op-test-framework.conf")])
         if args.config_file:
             if os.access(args.config_file, os.R_OK):
@@ -468,7 +470,7 @@ def parse_args(self, argv=None):
                     errno.ENOENT), args.config_file)
         try:
             defaults = dict(config.items('op-test'))
-        except ConfigParser.NoSectionError:
+        except configparser.NoSectionError:
             pass
 
         parser = get_parser()
@@ -547,7 +549,7 @@ def parse_args(self, argv=None):
         if (not os.path.exists(self.logdir)):
             os.makedirs(self.logdir)
 
-        print("Logs in: {}".format(self.logdir))
+        print(("Logs in: {}".format(self.logdir)))
 
         OpTestLogger.optest_logger_glob.logdir = self.logdir
 
@@ -584,7 +586,8 @@ def parse_args(self, argv=None):
                                              stdin=subprocess.PIPE,
                                              stderr=sys.stderr,
                                              stdout=sys.stdout,
-                                             shell=True)
+                                             shell=True,
+                                             text=True)
         self.logfile = self.logfile_proc.stdin
 
         # we have enough setup to allow
@@ -817,7 +820,7 @@ def objs(self):
                 )
                 bmc.set_system(self.op_system)
             elif self.args.bmc_type in ['qemu']:
-                print(repr(self.args))
+                print((repr(self.args)))
                 bmc = OpTestQemu(conf=self,
                                  qemu_binary=self.args.qemu_binary,
                                  pnor=self.args.host_pnor,

OpTestLogger.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # OpenPOWER Automated Test Project
 #
 # Contributors Listed Below - COPYRIGHT 2018
@@ -34,7 +34,7 @@ def __init__(self, l):
     def write(self, data):
         lines = data.splitlines()
         for line in lines:
-            self.log.debug(line.rstrip("\n"))
+            self.log.debug(line.rstrip(b"\n"))
 
     def flush(self):
         pass

README.md

@@ -28,12 +28,12 @@ Run something (see below Running the tests).
 
 This framework runs on most Linux based systems.
 
-You need python 2.7 or greater and also needs below modules to be installed
+You need python 3.4 or greater and also needs below modules to be installed
 
     pip install pexpect importlib ptyprocess requests pysocks
 
     Optionally:  pip install unittest-xml-reporting unittest2
-    For XML output: sudo apt-get install python-xmlrunner
+    For XML output: sudo apt-get install python3-xmlrunner
 
     For qemu: sudo apt-get install qemu-utils
 

addons/__init__.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # IBM_PROLOG_BEGIN_TAG
 # This is an automatically generated prolog.
 #

common/Exceptions.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # OpenPOWER Automated Test Project
 #
 # Contributors Listed Below - COPYRIGHT 2017
@@ -290,7 +290,7 @@ def __init__(self, **kwargs):
         default_vals = {'state': None, 'message': None}
         self.kwargs = {}
         for key in default_vals:
-            if key not in kwargs.keys():
+            if key not in list(kwargs.keys()):
                 self.kwargs[key] = default_vals[key]
             else:
                 self.kwargs[key] = kwargs[key]
@@ -313,7 +313,7 @@ def __init__(self, **kwargs):
         default_vals = {'expect_dict': None, 'reconnect_count': 0}
         self.kwargs = {}
         for key in default_vals:
-            if key not in kwargs.keys():
+            if key not in list(kwargs.keys()):
                 self.kwargs[key] = default_vals[key]
             else:
                 self.kwargs[key] = kwargs[key]
@@ -334,7 +334,7 @@ def __init__(self, **kwargs):
         default_vals = {'before': None, 'after': None, 'msg': None}
         self.kwargs = {}
         for key in default_vals:
-            if key not in kwargs.keys():
+            if key not in list(kwargs.keys()):
                 self.kwargs[key] = default_vals[key]
             else:
                 self.kwargs[key] = kwargs[key]
@@ -357,7 +357,7 @@ def __init__(self, **kwargs):
         default_vals = {'state': None, 'message': None}
         self.kwargs = {}
         for key in default_vals:
-            if key not in kwargs.keys():
+            if key not in list(kwargs.keys()):
                 self.kwargs[key] = default_vals[key]
             else:
                 self.kwargs[key] = kwargs[key]
@@ -381,7 +381,7 @@ def __init__(self, **kwargs):
         default_vals = {'message': None}
         self.kwargs = {}
         for key in default_vals:
-            if key not in kwargs.keys():
+            if key not in list(kwargs.keys()):
                 self.kwargs[key] = default_vals[key]
             else:
                 self.kwargs[key] = kwargs[key]
@@ -403,7 +403,7 @@ def __init__(self, **kwargs):
         default_vals = {'message': None}
         self.kwargs = {}
         for key in default_vals:
-            if key not in kwargs.keys():
+            if key not in list(kwargs.keys()):
                 self.kwargs[key] = default_vals[key]
             else:
                 self.kwargs[key] = kwargs[key]
@@ -430,7 +430,7 @@ def __init__(self, **kwargs):
         default_vals = {'message': None, 'code': 0}
         self.kwargs = {}
         for key in default_vals:
-            if key not in kwargs.keys():
+            if key not in list(kwargs.keys()):
                 self.kwargs[key] = default_vals[key]
             else:
                 self.kwargs[key] = kwargs[key]
@@ -449,7 +449,7 @@ def __init__(self, **kwargs):
         default_vals = {'message': None}
         self.kwargs = {}
         for key in default_vals:
-            if key not in kwargs.keys():
+            if key not in list(kwargs.keys()):
                 self.kwargs[key] = default_vals[key]
             else:
                 self.kwargs[key] = kwargs[key]
@@ -472,7 +472,7 @@ def __init__(self, **kwargs):
         default_vals = {'message': None}
         self.kwargs = {}
         for key in default_vals:
-            if key not in kwargs.keys():
+            if key not in list(kwargs.keys()):
                 self.kwargs[key] = default_vals[key]
             else:
                 self.kwargs[key] = kwargs[key]
@@ -507,7 +507,7 @@ def __init__(self, **kwargs):
         default_vals = {'before': None, 'after': None, 'msg': None}
         self.kwargs = {}
         for key in default_vals:
-            if key not in kwargs.keys():
+            if key not in list(kwargs.keys()):
                 self.kwargs[key] = default_vals[key]
             else:
                 self.kwargs[key] = kwargs[key]

common/OPexpect.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #
 # OpenPOWER Automated Test Project
 #
@@ -35,15 +35,15 @@
 """
 
 import pexpect
-from Exceptions import *
+from .Exceptions import *
 
 
 class spawn(pexpect.spawn):
 
     def __init__(self, command, args=[], maxread=8000,
                  searchwindowsize=None, logfile=None, cwd=None, env=None,
                  ignore_sighup=False, echo=True, preexec_fn=None,
-                 encoding=None, codec_errors='strict', dimensions=None,
+                 encoding='utf-8', codec_errors='strict', dimensions=None,
                  failure_callback=None, failure_callback_data=None):
         self.command = command
         self.failure_callback = failure_callback
@@ -53,13 +53,14 @@ def __init__(self, command, args=[], maxread=8000,
                                     searchwindowsize=searchwindowsize,
                                     logfile=logfile,
                                     cwd=cwd, env=env,
-                                    ignore_sighup=ignore_sighup)
+                                    ignore_sighup=ignore_sighup,
+                                    encoding=encoding)
 
     def set_system(self, system):
         self.op_test_system = system
         return
 
-    def expect(self, pattern, timeout=-1, searchwindowsize=-1, async=False):
+    def expect(self, pattern, timeout=-1, searchwindowsize=-1):
         op_patterns = ["qemu: could find kernel",
                        "INFO: rcu_sched self-detected stall on CPU",
                        "kernel BUG at",

common/OpTestASM.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # encoding=utf8
 # IBM_PROLOG_BEGIN_TAG
 # This is an automatically generated prolog.
@@ -39,14 +39,18 @@
 import os
 import pexpect
 import sys
-import commands
+import subprocess
 
-from OpTestConstants import OpTestConstants as BMC_CONST
-from OpTestError import OpTestError
+from .OpTestConstants import OpTestConstants as BMC_CONST
+from .OpTestError import OpTestError
 
-import cookielib
-import urllib
-import urllib2
+import http.cookiejar
+import urllib.request
+import urllib.parse
+import urllib.error
+import urllib.request
+import urllib.error
+import urllib.parse
 import re
 import ssl
 # Work around issues with python < 2.7.9
@@ -62,10 +66,11 @@ def __init__(self, i_fspIP, i_fspUser, i_fspPasswd):
         self.user_name = i_fspUser
         self.password = i_fspPasswd
         self.url = "https://%s/cgi-bin/cgi?" % self.host_name
-        self.cj = cookielib.CookieJar()
-        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
+        self.cj = http.cookiejar.CookieJar()
+        opener = urllib.request.build_opener(
+            urllib.request.HTTPCookieProcessor(self.cj))
         opener.addheaders = [('User-agent', 'LTCTest')]
-        urllib2.install_opener(opener)
+        urllib.request.install_opener(opener)
         self.setforms()
 
     def setforms(self):
@@ -83,8 +88,8 @@ def setforms(self):
     def getcsrf(self, form):
         while True:
             try:
-                myurl = urllib2.urlopen(self.url+form, timeout=10)
-            except urllib2.URLError:
+                myurl = urllib.request.urlopen(self.url+form, timeout=10)
+            except urllib.error.URLError:
                 time.sleep(2)
                 continue
             break
@@ -97,19 +102,19 @@ def getcsrf(self, form):
     def getpage(self, form):
         while True:
             try:
-                myurl = urllib2.urlopen(self.url+form, timeout=60)
-            except (urllib2.URLError, ssl.SSLError):
+                myurl = urllib.request.urlopen(self.url+form, timeout=60)
+            except (urllib.error.URLError, ssl.SSLError):
                 time.sleep(2)
                 continue
             break
         return myurl.read()
 
     def submit(self, form, param):
         param['CSRF_TOKEN'] = self.getcsrf(form)
-        data = urllib.urlencode(param)
-        req = urllib2.Request(self.url+form, data)
+        data = urllib.parse.urlencode(param)
+        req = urllib.request.Request(self.url+form, data)
 
-        return urllib2.urlopen(req)
+        return urllib.request.urlopen(req)
 
     def login(self):
         if not len(self.cj) == 0:

common/OpTestBMC.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # IBM_PROLOG_BEGIN_TAG
 # This is an automatically generated prolog.
 #
@@ -39,15 +39,15 @@
 import pexpect
 import os.path
 import subprocess
-import OPexpect
-
-from OpTestIPMI import OpTestIPMI
-from OpTestSSH import OpTestSSH
-from OpTestUtil import OpTestUtil
-from OpTestConstants import OpTestConstants as BMC_CONST
-from OpTestError import OpTestError
-from OpTestWeb import OpTestWeb
-from Exceptions import CommandFailed, SSHSessionDisconnected
+from . import OPexpect
+
+from .OpTestIPMI import OpTestIPMI
+from .OpTestSSH import OpTestSSH
+from .OpTestUtil import OpTestUtil
+from .OpTestConstants import OpTestConstants as BMC_CONST
+from .OpTestError import OpTestError
+from .OpTestWeb import OpTestWeb
+from .Exceptions import CommandFailed, SSHSessionDisconnected
 
 import logging
 import OpTestLogger

common/OpTestConstants.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # IBM_PROLOG_BEGIN_TAG
 # This is an automatically generated prolog.
 #

common/OpTestCronus.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # IBM_PROLOG_BEGIN_TAG
 # This is an automatically generated prolog.
 #
@@ -31,8 +31,8 @@
 import traceback
 import socket
 
-from Exceptions import ParameterCheck, UnexpectedCase
-from OpTestSystem import OpSystemState
+from .Exceptions import ParameterCheck, UnexpectedCase
+from .OpTestSystem import OpSystemState
 
 import logging
 import OpTestLogger

common/OpTestError.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # IBM_PROLOG_BEGIN_TAG
 # This is an automatically generated prolog.
 #