Merge pull request #8 from zopefoundation/issue7

Don't assume headers are unicode in FakeResponse.

.gitignore

@@ -16,3 +16,4 @@ develop-eggs
 dist
 parts
 htmlcov/
+.coverage.py*

.travis.yml

@@ -1,17 +1,19 @@
 # manual (marker for update script)
 language: python
+sudo: false
 python:
     - 2.7
     - pypy-5.4.1
     - 3.3
     - 3.4
     - 3.5
+    - 3.6
 install:
     - pip install -U pip setuptools
     - pip install -U coverage coveralls
     - pip install -U -e .[test]
 script:
-    - coverage run `which zope-testrunner` --test-path=src
+    - coverage run -m zope.testrunner --test-path=src --all
 after_success:
     - coveralls
 notifications:

CHANGES.rst

@@ -5,10 +5,18 @@ CHANGES
 4.1.0 (unreleased)
 ------------------
 
-- Use `base64.b64encode` to avoid deprecation warning with Python 3.
+- Use ``base64.b64encode`` to avoid deprecation warning with Python 3.
 
 - Add support for PyPy.
 
+- Add support for Python 3.6.
+
+- Fix the testlayer's ``FakeResponse`` assuming that headers were in
+  unicode on Python 2, where they should usually be encoded bytes
+  already. This could lead to UnicodeDecodeError if the headers
+  contained non-ascii characters. Also make it implement
+  ``__unicode__`` on Python 2 and ``__bytes__`` on Python 3 to ease
+  cross version testing. See `issue 7 <https://github.com/zopefoundation/zope.app.wsgi/issues/7>`_.
 
 4.0.0 (2016-08-08)
 ------------------

setup.py

@@ -62,6 +62,7 @@ def read(*rnames):
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: Implementation',
         'Programming Language :: Python :: Implementation :: CPython',
         'Natural Language :: English',

src/zope/app/wsgi/testlayer.py

@@ -140,6 +140,16 @@ class NotInBrowserLayer(Exception):
 class FakeResponse(object):
     """This behave like a Response object returned by HTTPCaller of
     zope.app.testing.functional.
+
+    .. versionchanged:: 4.1.0
+       Implement support for unicode() on Python 2 to be equivalent to
+       str() on Python 3, and implement support for bytes() on Python 3
+       to be equivalent to str() on Python 2. This should help in cross version
+       testing.
+
+       On Python 2, ``getOutput`` and ``__str__`` should no longer produce
+       UnicodeErrors.
+
     """
 
     def __init__(self, response):
@@ -161,8 +171,15 @@ def getBody(self):
         return self.response.body
 
     def getOutput(self):
-        parts = [b'HTTP/1.0 ' + self.response.status.encode('latin1')]
-        parts += [('%s: %s' % h).encode('latin1') for h in self.getHeaders()]
+        status = self.response.status
+        status = status.encode('latin1') if not isinstance(status, bytes) else status
+        parts = [b'HTTP/1.0 ' + status]
+
+        headers = [(k.encode('latin1') if not isinstance(k, bytes) else k,
+                    v.encode('latin1') if not isinstance(v, bytes) else v)
+                   for k, v in self.getHeaders()]
+
+        parts += [k + b': ' + v for k, v in headers]
 
         body = self.response.body
         if body:
@@ -171,9 +188,21 @@ def getOutput(self):
             parts += [b'', body]
         return b'\n'.join(parts)
 
-    def __str__(self):
-        out = self.getOutput()
-        return out.decode('latin1')
+    if str is bytes: # Py2
+
+        # Forcing __str__ through latin1, as Py3 does, will return
+        # unicode which will then be decoded as ascii, which could
+        # cause an UnicodeError.
+        __str__ = getOutput
+
+        def __unicode__(self):
+            return self.getOutput().decode('latin-1')
+    else:
+        __bytes__ = getOutput
+
+        def __str__(self):
+            return self.getOutput().decode('latin-1')
+
 
 def http(wsgi_app, string, handle_errors=True):
     request = TestRequest.from_file(BytesIO(string))

src/zope/app/wsgi/tests.py

@@ -144,6 +144,51 @@ def test_auth_non_encoded_colon(self):
         self.assertEquals(auth_header(header), expected)
 
 
+class TestFakeResponse(unittest.TestCase):
+
+    def test_doesnt_assume_encoding_of_headers(self):
+        # https://github.com/zopefoundation/zope.app.wsgi/issues/7
+        # headers on Py2 should already be bytes or at least be allowed
+        # to be bytes. For BWC, we allow them to be bytes or unicode either
+        # platform.
+        # The body/__str__ can be decoded correctly too when this happens
+        from zope.app.wsgi.testlayer import FakeResponse
+
+        try:
+            text_type = unicode
+        except NameError:
+            text_type = str
+        class MockResponse(object):
+
+            status = '200 OK'
+            body = ''
+
+            def __init__(self):
+                self.headerlist = []
+
+        response = MockResponse()
+        # A latin-1 byte.
+        response.headerlist.append(("X-Header".encode('latin-1'),
+                                    u"voill\xe0".encode('latin-1')))
+
+        fake = FakeResponse(response)
+        self.assertEqual(fake.getOutput(), b'HTTP/1.0 200 OK\nX-Header: voill\xe0')
+        # No matter the platform, str/bytes should not raise
+        self.assertIn('HTTP', str(fake))
+        self.assertIn(b'HTTP', bytes(fake))
+        self.assertEqual(text_type(fake),
+                         u'HTTP/1.0 200 OK\nX-Header: voill\xe0')
+
+        # A utf-8 byte, smuggled inside latin-1, as discussed in PEP3333
+        response.headerlist[0] = (b'X-Header',
+                                  u'p-o-p \U0001F4A9'.encode('utf-8').decode('latin-1'))
+        self.assertEqual(fake.getOutput(),
+                         b'HTTP/1.0 200 OK\nX-Header: p-o-p \xf0\x9f\x92\xa9')
+        self.assertIn('HTTP', str(fake))
+        self.assertIn(b'HTTP', bytes(fake))
+        self.assertEqual(text_type(fake),
+                         u'HTTP/1.0 200 OK\nX-Header: p-o-p \xf0\x9f\x92\xa9')
+
 def test_suite():
     suites = []
     checker = renormalizing.RENormalizing([
@@ -161,9 +206,6 @@ def test_suite():
     dt_suite.layer = wsgiapp_layer
     suites.append(dt_suite)
 
-    suites.append(unittest.makeSuite(WSGIPublisherApplicationTests))
-    suites.append(unittest.makeSuite(AuthHeaderTestCase))
-
     readme_test = doctest.DocFileSuite(
         'README.txt',
         checker=checker,
@@ -186,4 +228,6 @@ def test_suite():
     testlayer_suite.layer = wsgiapp_layer
     suites.append(testlayer_suite)
 
+    suites.append(unittest.defaultTestLoader.loadTestsFromName(__name__))
+
     return unittest.TestSuite(suites)

tox.ini

@@ -1,9 +1,9 @@
 [tox]
-envlist = coverage-clean,py27,pypy,py33,py34,py35,coverage-report
+envlist = coverage-clean,py27,pypy,py33,py34,py35,py36,coverage-report
 
 [testenv]
 commands =
-    coverage run {envbindir}/zope-testrunner --test-path=src
+    coverage run -m zope.testrunner --test-path=src --all []
 setenv =
   COVERAGE_FILE=.coverage.{envname}
 deps =