Fixes for PyPy. Make the Py3 tests mostly run, but the hashseed issue…

… stops them from fully working in some cases.

src/zc/recipe/cmmi/README.rst

@@ -227,9 +227,9 @@ When downloading a source archive or a patch, we can optionally make sure of
 its authenticity by supplying an MD5 checksum that must be matched. If it
 matches, we'll not be bothered with the check by buildout's output:
 
-    >>> try: from hashlib import md5
-    ... except ImportError: from md5 import new as md5
-    >>> foo_md5sum = md5(open(join(distros, 'foo.tgz')).read()).hexdigest()
+    >>> from hashlib import md5
+    >>> with open(join(distros, 'foo.tgz'), 'rb') as f:
+    ...     foo_md5sum = md5(f.read()).hexdigest()
 
     >>> write('buildout.cfg',
     ... """
@@ -344,8 +344,9 @@ After a successful build, such temporary directories are removed.
     >>> import glob
     >>> import tempfile
 
-    >>> tempdir = tempfile.gettempdir()
-    >>> dirs = len(glob.glob(os.path.join(tempdir, '*buildout-foo')))
+    >>> old_tempdir = tempfile.gettempdir()
+    >>> tempdir = tempfile.tempdir = tempfile.mkdtemp(suffix='.buildout.build')
+    >>> dirs = glob.glob(os.path.join(tempdir, '*buildout-foo'))
 
     >>> write('buildout.cfg',
     ... """
@@ -368,6 +369,7 @@ After a successful build, such temporary directories are removed.
     installing foo
     <BLANKLINE>
 
-    >>> new_dirs = len(glob.glob(os.path.join(tempdir, '*buildout-foo')))
-    >>> dirs == new_dirs
+    >>> new_dirs = glob.glob(os.path.join(tempdir, '*buildout-foo'))
+    >>> len(dirs) == len(new_dirs) == 0
     True
+    >>> tempfile.tempdir = old_tempdir

src/zc/recipe/cmmi/__init__.py

@@ -110,7 +110,10 @@ def _state_hash(self):
                        in sorted(self.environ.items())])
         state = [self.url, self.extra_options, self.autogen,
                  self.patch, self.patch_options, env]
-        return sha1(''.join(state)).hexdigest()
+        data = ''.join(state)
+        if not isinstance(data, bytes):
+            data = data.encode('utf-8')
+        return sha1(data).hexdigest()
 
     def install(self):
         self.build()

src/zc/recipe/cmmi/downloadcache.rst

@@ -42,7 +42,7 @@ We used the url option to specify the location of the archive.
 
 It creates a make file which is also run:
 
-    >>> print system('bin/buildout')
+    >>> print(system('bin/buildout'))
     In...
     ...
     Installing foo.
@@ -82,7 +82,7 @@ files are not downloaded afresh:
     ...             remove("parts", f)
     >>> remove_parts()
 
-    >>> print system(buildout)
+    >>> print(system(buildout))
     In...
     ...
     Uninstalling foo.
@@ -107,7 +107,7 @@ are also removed, then it is downloaded afresh:
 
     >>> remove_parts()
 
-    >>> print system('bin/buildout')
+    >>> print(system('bin/buildout'))
     In...
     ...
     Installing foo.
@@ -139,7 +139,7 @@ the new cache is created and repopulated:
     ... url = %s/foo.tgz
     ... """ % (cache2, distros))
 
-    >>> print system('bin/buildout')
+    >>> print(system('bin/buildout'))
     In...
     ...
     Installing foo.
@@ -188,7 +188,7 @@ been removed, the files from the cache are used:
 
     >>> remove_parts()
 
-    >>> print system(buildout)
+    >>> print(system(buildout))
     In...
     ...
     Uninstalling foo.
@@ -211,7 +211,7 @@ the cache, an error is raised because the file is not in the cache:
 
     >>> remove_parts()
 
-    >>> print system(buildout)
+    >>> print(system(buildout))
     In...
     ...
     Uninstalling foo.
@@ -222,4 +222,3 @@ the cache, an error is raised because the file is not in the cache:
       Installing foo.
     Error: Couldn't download 'http://localhost//foo.tgz' in offline mode.
     <BLANKLINE>
-

src/zc/recipe/cmmi/misc.rst

@@ -9,7 +9,7 @@ Creating the location folder
 When the recipe is subclassed, the `location` folder might be created
 before `zc.recipe.cmmi` has a chance to create it, so we need to make
 sure it checks that the folder does not exists before it is created.
-   
+
 In the test below, the `foo` folder is created before the recipe
 is launched::
 
@@ -27,7 +27,7 @@ is launched::
     ... url = file://%s/foo.tgz
     ... """ % (distros))
 
-    >>> print system('bin/buildout')
+    >>> print(system('bin/buildout'))
     Installing...
     ...
     installing foo
@@ -50,7 +50,7 @@ The part's directory is created when the part is installed:
     >>> remove('.installed.cfg')
     >>> rmdir('parts', 'foo')
 
-    >>> print system('bin/buildout')
+    >>> print(system('bin/buildout'))
     Installing...
     ...
     installing foo
@@ -64,10 +64,10 @@ part now uses a shared build or because the part is gone altogether):
     >>> write('buildout.cfg',
     ... """
     ... [buildout]
-    ... parts = 
+    ... parts =
     ... """)
 
-    >>> print system('bin/buildout')
+    >>> print(system('bin/buildout'))
     Uninstalling foo.
 
     >>> os.path.isdir(join(sample_buildout, "parts", "foo"))
@@ -95,7 +95,7 @@ aren't of the form NAME=.....
     ...   CFLAGS=-I/yyy -I/xxx --x=y 2=1+1 a=b
     ... """ % distros_url)
 
-    >>> print system('bin/buildout'),
+    >>> print(system('bin/buildout'))
     Installing foo.
     foo: Downloading http://localhost/foo.tgz
     foo: Unpacking and configuring

src/zc/recipe/cmmi/patching.rst

@@ -66,6 +66,7 @@ Now let's create a buildout.cfg file.
     foo: Cache miss; will cache http://localhost//config.patch as /cache/cmmi/...
     foo: Downloading http://localhost//config.patch
     patching file configure
+    ...
     configuring foo patched /sample-buildout/parts/foo
     echo building foo patched
     building foo patched

src/zc/recipe/cmmi/shared.rst

@@ -53,8 +53,8 @@ For example, if the download url changes, the build is executed again:
 
     >>> import os
     >>> import shutil
-    >>> shutil.copy(os.path.join(distros, 'foo.tgz'),
-    ...             os.path.join(distros, 'qux.tgz'))
+    >>> _ = shutil.copy(os.path.join(distros, 'foo.tgz'),
+    ...                 os.path.join(distros, 'qux.tgz'))
 
     >>> remove('.installed.cfg')
     >>> write('buildout.cfg',
@@ -149,21 +149,21 @@ mistaking some half-baked build directory as a good cached shared build.
 
 Let's simulate a build error. First, we backup a working build.
 
-    >>> shutil.copy(os.path.join(distros, 'foo.tgz'),
-    ...             os.path.join(distros, 'foo.tgz.bak'))
+    >>> _ = shutil.copy(os.path.join(distros, 'foo.tgz'),
+    ...                 os.path.join(distros, 'foo.tgz.bak'))
 
 Then we create a broken tarball:
 
     >>> import tarfile
-    >>> import StringIO
+    >>> from zc.recipe.cmmi.tests import BytesIO
     >>> import sys
     >>> tarpath = os.path.join(distros, 'foo.tgz')
     >>> with tarfile.open(tarpath, 'w:gz') as tar:
     ...    configure = 'invalid'
     ...    info = tarfile.TarInfo('configure.off')
     ...    info.size = len(configure)
     ...    info.mode = 0o755
-    ...    tar.addfile(info, StringIO.StringIO(configure))
+    ...    tar.addfile(info, BytesIO(configure))
 
 Now we reset the cache to force our broken tarball to be used:
 
@@ -183,7 +183,7 @@ Now we reset the cache to force our broken tarball to be used:
 
     >>> remove('.installed.cfg')
     >>> res = system('bin/buildout')
-    >>> print res
+    >>> print(res)
     Installing foo.
     ...
     ValueError: Couldn't find configure
@@ -198,8 +198,8 @@ When we now fix the error (by copying back the working version and resetting the
 cache), the build will be run again, and we don't use a half-baked shared
 directory:
 
-    >>> shutil.copy(os.path.join(distros, 'foo.tgz.bak'),
-    ...             os.path.join(distros, 'foo.tgz'))
+    >>> _ = shutil.copy(os.path.join(distros, 'foo.tgz.bak'),
+    ...                 os.path.join(distros, 'foo.tgz'))
     >>> shutil.rmtree(cache)
     >>> cache = tmpdir('cache')
     >>> write('buildout.cfg',

src/zc/recipe/cmmi/tests.py

@@ -14,7 +14,7 @@
 
 import os
 import re
-import StringIO
+from io import BytesIO as _BytesIO
 import sys
 import tarfile
 import zc.buildout.testing
@@ -25,6 +25,12 @@
 from zc.buildout.tests import easy_install_SetUp
 from zc.buildout.tests import normalize_bang
 
+def _as_bytes(s):
+    return s.encode('utf-8') if not isinstance(s, bytes) else s
+
+def BytesIO(s):
+    return _BytesIO(_as_bytes(s))
+
 def setUp(test):
     zc.buildout.testing.buildoutSetUp(test)
     zc.buildout.testing.install_develop('zc.recipe.cmmi', test)
@@ -36,35 +42,35 @@ def setUp(test):
         info = tarfile.TarInfo('configure')
         info.size = len(configure)
         info.mode = 0o755
-        tar.addfile(info, StringIO.StringIO(configure))
+        tar.addfile(info, BytesIO(configure))
 
     tarpath = os.path.join(distros, 'bar.tgz')
     with tarfile.open(tarpath, 'w:gz') as tar:
         configure = configure_template % sys.executable
         info = tarfile.TarInfo('configure.in')
         info.size = len(configure)
         info.mode = 0o755
-        tar.addfile(info, StringIO.StringIO(configure))
+        tar.addfile(info, BytesIO(configure))
 
         autogen = autogen_template
         info = tarfile.TarInfo('autogen.sh')
         info.size = len(autogen)
         info.mode = 0o755
-        tar.addfile(info, StringIO.StringIO(autogen))
+        tar.addfile(info, BytesIO(autogen))
 
     tarpath = os.path.join(distros, 'baz.tgz')
     with tarfile.open(tarpath, 'w:gz') as tar:
         configure = configure_template % sys.executable
         info = tarfile.TarInfo('configure.py')
         info.size = len(configure)
         info.mode = 0o755
-        tar.addfile(info, StringIO.StringIO(configure))
+        tar.addfile(info, BytesIO(configure))
 
 def add(tar, name, src, mode=None):
     info.size = len(src)
     if mode is not None:
         info.mode = mode
-    tar.addfile(info, StringIO.StringIO(src))
+    tar.addfile(info, BytesIO(src))
 
 configure_template = """#!%s
 import sys
@@ -78,7 +84,7 @@ def add(tar, name, src, mode=None):
 \techo installing foo
 '''
 
-open('Makefile', 'w').write(Makefile_template)
+with open('Makefile', 'w') as f: f.write(Makefile_template)
 
 """
 
@@ -100,9 +106,19 @@ def test_suite():
                 (re.compile('http://localhost:[0-9]{4,5}/'),
                  'http://localhost/'),
                 (re.compile('occured'), 'occurred'),
+                # Buildout or setuptools has a bug not closing .egg-link files,
+                # leading to issues being reported by PyPy, which naturally mess up
+                # doctests.
+                (re.compile('Exception IOError: IOError.*finalizer of closed file.*'),
+                ''),
+                # IGNORE_EXCEPTION_MODULE_IN_PYTHON2 fails because the output doesn't
+                # always look like a traceback.
+                (re.compile('subprocess.CalledProcessError'), 'CalledProcessError'),
                ]),
-            optionflags=(doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE)
-           ),
+            optionflags=(doctest.ELLIPSIS
+                         | doctest.NORMALIZE_WHITESPACE
+                         | renormalizing.IGNORE_EXCEPTION_MODULE_IN_PYTHON2)
+        ),
         doctest.DocFileSuite(
             'downloadcache.rst',
             'patching.rst',
@@ -121,7 +137,7 @@ def test_suite():
                 (re.compile('[0-9a-f]{40}'), '<BUILDID>'),
                 ]),
             optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE
-            ),
+        ),
         doctest.DocFileSuite(
             'misc.rst',
             setUp=setUp,
@@ -134,5 +150,5 @@ def test_suite():
                  'http://localhost/'),
                 ]),
             optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE
-            ),
-        ))
+        ),
+    ))