Merge branch 'master' into fix_propertysheet_get

CHANGES.rst

@@ -16,9 +16,17 @@ Features
 
 - Optionally control the use of Zope's built-in XML-RPC support for
   POST requests with Content-Type ``text/xml`` via the
-  registration of a ``ZPublisher.interfaces.IXmlrpcChecker` utility
+  registration of a ``ZPublisher.interfaces.IXmlrpcChecker`` utility
   (`#620 <https://github.com/zopefoundation/Zope/issues/620>`_).
 
+Fixes
++++++
+
+- Prevent encoding issues in existing DTML Method and Document objects
+
+- Fixed logic error in exceptions handling during publishing. This error would
+  prevent correct Unauthorized handling when exceptions debug mode was set.
+
 
 4.0 (2019-05-10)
 ----------------

constraints.txt

@@ -4,7 +4,7 @@ AuthEncoding==4.1
 BTrees==4.5.1
 Chameleon==3.6.1
 DateTime==4.3
-DocumentTemplate==3.0
+DocumentTemplate==3.1b1
 ExtensionClass==4.4
 Missing==4.1
 MultiMapping==4.1

requirements-full.txt

@@ -5,7 +5,7 @@ AuthEncoding==4.1
 BTrees==4.5.1
 Chameleon==3.6.1
 DateTime==4.3
-DocumentTemplate==3.0
+DocumentTemplate==3.1b1
 ExtensionClass==4.4
 Missing==4.1
 MultiMapping==4.1

src/OFS/DTMLDocument.py

@@ -13,7 +13,7 @@
 """DTML Document objects.
 """
 
-from six import binary_type
+import six
 from six.moves.urllib.parse import quote
 
 from AccessControl import getSecurityManager
@@ -28,6 +28,7 @@
 from OFS.PropertyManager import PropertyManager
 from zExceptions.TracebackSupplement import PathTracebackSupplement
 from zope.contenttype import guess_content_type
+from ZPublisher.HTTPRequest import default_encoding
 
 
 done = 'done'
@@ -88,7 +89,7 @@ def __call__(self, client=None, REQUEST={}, RESPONSE=None, **kw):
 
             r = HTML.__call__(self, (client, bself), REQUEST, **kw)
 
-            if RESPONSE is None or not isinstance(r, binary_type):
+            if RESPONSE is None or not isinstance(r, str):
                 if not self._cache_namespace_keys:
                     self.ZCacheable_set(r)
                 return r
@@ -101,10 +102,11 @@ def __call__(self, client=None, REQUEST={}, RESPONSE=None, **kw):
             if 'content_type' in self.__dict__:
                 c = self.content_type
             else:
-                if isinstance(r, binary_type):
-                    c, e = guess_content_type(self.__name__, r)
-                else:
-                    c, e = guess_content_type(self.__name__, r.encode('utf-8'))
+                encoding = getattr(self, 'encoding', default_encoding)
+                if six.PY2 and not isinstance(r, six.text_type):
+                    # Prevent double-encoding edge cases under Python 2
+                    r = r.decode(encoding)
+                c, e = guess_content_type(self.getId(), r.encode(encoding))
             RESPONSE.setHeader('Content-Type', c)
         result = decapitate(r, RESPONSE)
         if not self._cache_namespace_keys:

src/OFS/DTMLMethod.py

@@ -44,6 +44,7 @@
 from zExceptions import ResourceLockedError
 from zExceptions.TracebackSupplement import PathTracebackSupplement
 from zope.contenttype import guess_content_type
+from ZPublisher.HTTPRequest import default_encoding
 from ZPublisher.Iterators import IStreamIterator
 
 
@@ -162,6 +163,7 @@ def __call__(self, client=None, REQUEST={}, RESPONSE=None, **kw):
                 return result
 
             r = HTML.__call__(self, client, REQUEST, **kw)
+
             if RESPONSE is None or not isinstance(r, str):
                 if not self._cache_namespace_keys:
                     self.ZCacheable_set(r)
@@ -177,11 +179,11 @@ def __call__(self, client=None, REQUEST={}, RESPONSE=None, **kw):
             if 'content_type' in self.__dict__:
                 c = self.content_type
             else:
+                encoding = getattr(self, 'encoding', default_encoding)
                 if PY2 and not isinstance(r, text_type):
                     # Prevent double-encoding edge cases under Python 2
-                    r = r.decode(self.encoding)
-                c, e = guess_content_type(self.getId(),
-                                          r.encode(self.encoding))
+                    r = r.decode(encoding)
+                c, e = guess_content_type(self.getId(), r.encode(encoding))
             RESPONSE.setHeader('Content-Type', c)
         result = decapitate(r, RESPONSE)
         if not self._cache_namespace_keys:

src/OFS/tests/test_DTMLDocument.py

@@ -2,6 +2,8 @@
 import io
 import unittest
 
+from Testing.makerequest import makerequest
+
 
 class DTMLDocumentTests(unittest.TestCase):
 
@@ -50,6 +52,19 @@ def test_manage_upload__BytesIO(self):
         self.assertEqual(doc.read(), 'bÿtës')
         self.assertIsInstance(doc.read(), str)
 
+    def test__call__missing_encoding_old_instances(self):
+        """ Existing DTML documents have no "encoding" attribute """
+        from OFS.Folder import Folder
+        client = makerequest(Folder('client'))
+        response = client.REQUEST['RESPONSE']
+        doc = self._makeOne(source_string='foo')
+
+        # In order to test the issue I need to delete the "encoding" attribute
+        # that existing instances did not have.
+        del doc.encoding
+
+        self.assertEqual(doc(client=client, RESPONSE=response), u'foo')
+
 
 class FactoryTests(unittest.TestCase):
 

src/OFS/tests/test_DTMLMethod.py

@@ -6,6 +6,7 @@
 import Testing.ZopeTestCase
 import zExceptions
 import Zope2.App.zcml
+from Testing.makerequest import makerequest
 
 
 def _lock_item(item):
@@ -96,6 +97,19 @@ def test_manage_edit__invalid_code(self):
             'unexpected end tag, for tag </dtml-let>, on line 1 of <string>',
             str(err.exception))
 
+    def test__call__missing_encoding_old_instances(self):
+        """ Existing DTML methods have no "encoding" attribute """
+        from OFS.Folder import Folder
+        client = makerequest(Folder('client'))
+        response = client.REQUEST['RESPONSE']
+        doc = self._makeOne(source_string='foo')
+
+        # In order to test the issue I need to delete the "encoding" attribute
+        # that existing instances did not have.
+        del doc.encoding
+
+        self.assertEqual(doc(client=client, RESPONSE=response), u'foo')
+
 
 class DTMLMethodBrowserTests(Testing.ZopeTestCase.FunctionalTestCase):
     """Browser testing ..OFS.DTMLMethod"""

src/ZPublisher/WSGIPublisher.py

@@ -176,10 +176,13 @@ def transaction_pubevents(request, response, tm=transaction.manager):
             if request.environ.get('x-wsgiorg.throw_errors', False):
                 reraise(*exc_info)
 
+            retry = False
+            unauth = False
+            debug_exc = getattr(response, 'debug_exceptions', False)
+
             # If the exception is transient and the request can be retried,
             # shortcut further processing. It makes no sense to have an
             # exception view registered for this type of exception.
-            retry = False
             if isinstance(exc, TransientError) and request.supports_retry():
                 retry = True
             else:
@@ -195,6 +198,7 @@ def transaction_pubevents(request, response, tm=transaction.manager):
                 # is used, an exception view for Unauthorized has to merge
                 # the state of the response and the exception instance.
                 if isinstance(exc, Unauthorized):
+                    unauth = True
                     exc.setRealm(response.realm)
                     response._unauthorized()
                     response.setStatus(exc.getStatus())
@@ -205,8 +209,7 @@ def transaction_pubevents(request, response, tm=transaction.manager):
             notify(pubevents.PubFailure(request, exc_info, retry))
 
             if retry or \
-               not (exc_view_created or isinstance(exc, Unauthorized)) or \
-               getattr(response, 'debug_exceptions', False):
+               (not unauth and (debug_exc or not exc_view_created)):
                 reraise(*exc_info)
 
         finally:

src/ZPublisher/interfaces.py

@@ -89,12 +89,12 @@ def __call__(request):
 
         Only called for a non-SOAP POST request whose `Content-Type`
         contains `text/xml` (any other request automatically does not
-        use Zope's buildin XML-RPC).
+        use Zope's built-in XML-RPC).
 
         Note: this is called very early during request handling when most
         typical attributes of *request* are not yet set up -- e.g. it
         cannot rely on information in `form` or `other`.
         Usually, it will look up information in `request.environ`
-        which at this time is garanteed (only) to contain the
+        which at this time is guaranteed (only) to contain the
         typical CGI information, such as `PATH_INFO` and `QUERY_STRING`.
         """

versions-prod.cfg

@@ -10,7 +10,7 @@ AuthEncoding = 4.1
 BTrees = 4.5.1
 Chameleon = 3.6.1
 DateTime = 4.3
-DocumentTemplate = 3.0
+DocumentTemplate = 3.1b1
 ExtensionClass = 4.4
 Missing = 4.1
 MultiMapping = 4.1