More python 3 compatibility fixes

Our biggest problem for Python3 compatibility was the _CommandOutput
class in rdopkg/utils/cmd.py, since it derives from str, which is
Unicode in Python3.

Solving this by making all strings unicode, and adjusting when needed.

Also reimplemented functions from rpmUtils.miscutils which are python2-only,
and fixed warnings about non-closed file handles.

Change-Id: I2cf5321edaddf5d3e22764c7c9e4a8c5ff84e7c6

Author: javierpena

rdopkg.spec

@@ -15,6 +15,7 @@ BuildRequires:    python2-devel
 BuildRequires:    PyYAML
 
 Requires:         python-bunch
+Requires:         python-future
 Requires:         python-paramiko
 Requires:         python-pymod2pkg >= 0.2.1
 Requires:         python-requests

rdopkg/guess.py

@@ -174,7 +174,7 @@ def user():
     if not user:
         raise exception.CantGuess(what="user name",
                                   why='git config user.name not set')
-    return user.decode('utf-8')
+    return user
 
 
 def email():

rdopkg/utils/cmd.py

@@ -1,3 +1,6 @@
+from __future__ import unicode_literals
+from builtins import str
+
 import json
 import re
 import subprocess
@@ -53,6 +56,7 @@ def run(cmd, *params, **kwargs):
 
     if input:
         stdin = subprocess.PIPE
+        input = input.encode()
     else:
         stdin = None
 
@@ -75,16 +79,16 @@ def run(cmd, *params, **kwargs):
         if print_stdout:
             log.info(out)
     else:
-        out = ''
+        out = b''
 
     if err:
         err = err.rstrip()
         if print_stderr:
             log.info(err)
     else:
-        err = ''
+        err = b''
 
-    cout = _CommandOutput(out)
+    cout = _CommandOutput(out.decode('utf-8'))
     cout.stderr = err
     cout.return_code = prc.returncode
     cout.cmd = cmd_str
@@ -262,8 +266,9 @@ def get_commit_bzs(self, from_revision, to_revision=None):
                        log_cmd=False, fatal=False)
         if not log_out:
             return []
+
         log = log_out.strip('\n\x1e').split("\x1e")
-        log = [row.strip().split("\x1f") for row in log]
+        log = [row.strip('\n\t ').split("\x1f") for row in log]
         log = [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]
         BZ_REGEX = r'rhbz#(\d+)'
         result = []

rdopkg/utils/specfile.py

@@ -9,12 +9,64 @@
 RPM_AVAILABLE = False
 try:
     import rpm
-    import rpmUtils.miscutils
     RPM_AVAILABLE = True
 except ImportError:
     pass
 
 
+def split_filename(filename):
+    """
+    Received a standard style rpm fullname and returns
+    name, version, release, epoch, arch
+    Example: foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386
+             1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64
+
+    This function replaces rpmUtils.miscutils.splitFilename, see
+    https://bugzilla.redhat.com/1452801
+    """
+
+    # Remove .rpm suffix
+    if filename.endswith('.rpm'):
+        filename = filename.split('.rpm')[0]
+
+    # is there an epoch?
+    components = filename.split(':')
+    if len(components) > 1:
+        epoch = components[0]
+    else:
+        epoch = ''
+
+    # Arch is the last item after .
+    arch = filename.rsplit('.')[-1]
+    remaining = filename.rsplit('.%s' % arch)[0]
+    release = remaining.rsplit('-')[-1]
+    version = remaining.rsplit('-')[-2]
+    name = '-'.join(remaining.rsplit('-')[:-2])
+
+    return name, version, release, epoch, arch
+
+
+def string_to_version(verstring):
+    """
+    Return a tuple of (epoch, version, release) from a version string
+
+    This function replaces rpmUtils.miscutils.stringToVersion, see
+    https://bugzilla.redhat.com/1364504
+    """
+    # is there an epoch?
+    components = verstring.split(':')
+    if len(components) > 1:
+        epoch = components[0]
+    else:
+        epoch = 0
+
+    remaining = components[:2][0].split('-')
+    version = remaining[0]
+    release = remaining[1]
+
+    return (epoch, version, release)
+
+
 def spec_fn(spec_dir='.'):
     specs = [f for f in os.listdir(spec_dir)
              if os.path.isfile(f) and f.endswith('.spec')]
@@ -32,7 +84,8 @@ def get_patches_from_files(patches_dir='.'):
         return []
     patches = []
     for pfn in patches_fns:
-        txt = codecs.open(pfn, 'r', encoding='utf-8').read()
+        with codecs.open(pfn, 'r', encoding='utf-8') as fp:
+            txt = fp.read()
         hash = None
         m = re.search(r'^From ([a-z0-9]+)', txt, flags=re.M)
         if m:
@@ -84,8 +137,8 @@ def has_macros(s):
 def nvrcmp(nvr1, nvr2):
     if not RPM_AVAILABLE:
         raise exception.RpmModuleNotAvailable()
-    t1 = rpmUtils.miscutils.stringToVersion(nvr1)
-    t2 = rpmUtils.miscutils.stringToVersion(nvr2)
+    t1 = string_to_version(nvr1)
+    t2 = string_to_version(nvr2)
     return rpm.labelCompare(t1, t2)
 
 
@@ -98,9 +151,7 @@ def vcmp(v1, v2):
 
 
 def nvr2version(nvr):
-    if not RPM_AVAILABLE:
-        raise exception.RpmModuleNotAvailable()
-    _, v, _, _, _ = rpmUtils.miscutils.splitFilename(nvr)
+    _, v, _, _, _ = split_filename(nvr)
     return v
 
 
@@ -130,7 +181,8 @@ def fn(self):
     @property
     def txt(self):
         if not self._txt:
-            self._txt = codecs.open(self.fn, 'r', encoding='utf-8').read()
+            with codecs.open(self.fn, 'r', encoding='utf-8') as fp:
+                self._txt = fp.read()
         return self._txt
 
     @property
@@ -548,7 +600,7 @@ def get_last_changelog_entry(self, strip=False):
         entry = entries[0]
         lines = entry.split("\n")
         if strip:
-            lines = map(lambda x: x.lstrip(" -*\t"), lines)
+            lines = list(map(lambda x: x.lstrip(" -*\t"), lines))
         return lines[0], lines[1:]
 
     def get_requires(self, versions_as_string=False, remove_epoch=True):

requirements.txt

@@ -1,5 +1,6 @@
 blessings
 bunch
+future
 paramiko
 PyYAML
 requests

tests/test_common.py

@@ -106,7 +106,8 @@ def assert_distgit(dg_path, img_dg):
     exp_tree = cfind(img_path)
     assert current_tree == exp_tree
     current_txt = dg_path.join('foo.spec').read()
-    exp_txt = open(os.path.join(img_path, 'foo.spec')).read()
+    with open(os.path.join(img_path, 'foo.spec')) as fp:
+        exp_txt = fp.read()
     assert current_txt == exp_txt
 
 

tests/test_update_patches.py

@@ -178,15 +178,18 @@ def test_wipe(tmpdir):
         spec = specfile.Spec()
         spec.wipe_patches()
         spec.save()
-        spec_after = open('foo.spec', 'r').read()
-    spec_exp = open(common.spec_path('empty'), 'r').read()
+        with open('foo.spec', 'r') as fp:
+            spec_after = fp.read()
+    with open(common.spec_path('empty'), 'r') as fp:
+        spec_exp = fp.read()
     assert spec_after == spec_exp
     # second wipe must be a no-op
     with tmpdir.as_cwd():
         spec = specfile.Spec()
         spec.wipe_patches()
         spec.save()
-        spec_after = open('foo.spec', 'r').read()
+        with open('foo.spec', 'r') as fp:
+            spec_after = fp.read()
     assert spec_after == spec_exp