- Be much more careful when setting encoding if it is not passed in

CHANGES.rst

@@ -4,6 +4,8 @@ Changelog
 3.0b9 (unreleased)
 ------------------
 
+- Be much more careful when setting encoding if it is not passed in
+
 - Only decode input in ``html_quote`` when needed under Python 3
   (`Products.PythonScripts#28 <https://github.com/zopefoundation/Products.PythonScripts/issues/28`_>)
 

src/DocumentTemplate/DT_String.py

@@ -17,6 +17,7 @@
 
 import six
 
+import DocumentTemplate as _dt
 from DocumentTemplate.DT_Util import ParseError, InstanceDict
 from DocumentTemplate.DT_Util import TemplateDict, render_blocks
 from DocumentTemplate.DT_Var import Var, Call, Comment
@@ -219,6 +220,9 @@ def parse_block(self, text, start, result, tagre,
 
         start = self.skip_eol(text, start)
 
+        # If this is an older object without encoding set we use the old
+        # pre-Zope 4 default.
+        encoding = getattr(self, 'encoding', _dt.OLD_DEFAULT_ENCODING)
         blocks = []
         tname = scommand.name
         sname = stag
@@ -259,7 +263,7 @@ def parse_block(self, text, start, result, tagre,
                     sstart = start
                 else:
                     try:
-                        r = scommand(blocks, encoding=self.encoding)
+                        r = scommand(blocks, encoding=encoding)
                         if hasattr(r, 'simple_form'):
                             r = r.simple_form
                         result.append(r)
@@ -302,7 +306,9 @@ def __init__(self, source_string='', mapping=None, __name__='<string>',
         mapping object containing defaults for values to be inserted.
         """
         self.raw = source_string
-        self.encoding = encoding
+        # For newly created objects, if no encoding is passed in we use the
+        # ZPublisher default
+        self.encoding = encoding or _dt.NEW_DEFAULT_ENCODING
         self.initvars(mapping, vars)
         self.setName(__name__)
 
@@ -437,6 +443,7 @@ def __call__(self, client=None, mapping={}, **kw):
         # print(client)
         # print(mapping)
         # print('============================================================')
+        encoding = getattr(self, 'encoding', None)
 
         if mapping is None:
             mapping = {}
@@ -518,7 +525,7 @@ def __call__(self, client=None, mapping={}, **kw):
             if value is _marker:
                 try:
                     result = render_blocks(self._v_blocks, md,
-                                           encoding=self.encoding)
+                                           encoding=encoding)
                 except DTReturn as v:
                     result = v.v
                 self.ZDocumentTemplate_afterRender(md, result)

src/DocumentTemplate/_DocumentTemplate.py

@@ -130,7 +130,7 @@ def join_unicode(rendered, encoding=None):
         # Fix up the list, treating normal strings as encoded in the
         # passed-in ``encoding``, or Latin-1 as fallback.
         if encoding is None:
-            encoding = _dt.DEFAULT_ENCODING
+            encoding = _dt.OLD_DEFAULT_ENCODING
         rendered = list(rendered)
         for i in range(len(rendered)):
             if isinstance(rendered[i], six.binary_type):

src/DocumentTemplate/__init__.py

@@ -32,4 +32,7 @@
 del security
 
 # This encoding is used for backwards compatibility
-DEFAULT_ENCODING = "latin-1"
+OLD_DEFAULT_ENCODING = 'Latin-1'
+
+# This is the new ZPublisher default and will be used on new objects
+NEW_DEFAULT_ENCODING = 'UTF-8'

src/DocumentTemplate/tests/testDTMLUnicode.py

@@ -35,7 +35,12 @@ def _get_doc_class(self):
         from DocumentTemplate.DT_HTML import HTML
 
         def make(*args):
-            return HTML(*args, encoding=self.encoding_arg)
+            doc = HTML(*args, encoding=self.encoding_arg)
+            # Need to cheat here because the mewning of passing
+            # None as encoding has changed, it will be UTF-8 then
+            if self.encoding_arg != doc.encoding:
+                doc.encoding = self.encoding_arg
+            return doc
 
         return make
     doc_class = property(_get_doc_class,)
@@ -117,9 +122,9 @@ class DTMLUnicodeTestsModuleGlobal(DTMLUnicodeTests):
 
     def setUp(self):
         import DocumentTemplate as dt
-        self._default_encoding = dt.DEFAULT_ENCODING
-        dt.DEFAULT_ENCODING = self.encoding
+        self._default_encoding = dt.OLD_DEFAULT_ENCODING
+        dt.OLD_DEFAULT_ENCODING = self.encoding
 
     def tearDown(self):
         import DocumentTemplate as dt
-        dt.DEFAULT_ENCODING = self._default_encoding
+        dt.OLD_DEFAULT_ENCODING = self._default_encoding