Add optional keyword argument 'encoding'

CHANGES.rst

@@ -4,6 +4,9 @@ Changelog
 3.0b7 (unreleased)
 ------------------
 
+- Make the rendering encoding configurable to fix rendering on Zope 4
+  (`#43 <https://github.com/zopefoundation/DocumentTemplate/issues/43>`_)
+
 - Add unit tests for ``dtml-if``, ``dtml-unless`` and ``dtml-in`` variables
   (`#7 <https://github.com/zopefoundation/DocumentTemplate/issues/7>`_)
 

src/DocumentTemplate/DT_If.py

@@ -89,12 +89,13 @@ class If(object):
     elses = None
     expr = ''
 
-    def __init__(self, blocks):
+    def __init__(self, blocks, encoding=None):
 
         tname, args, section = blocks[0]
         args = parse_params(args, name='', expr='')
         name, expr = name_param(args, 'if', 1)
         self.__name__ = name
+        self.encoding = encoding
         if expr is None:
             cond = name
         else:

src/DocumentTemplate/DT_In.py

@@ -357,8 +357,8 @@ class InFactory(object):
     blockContinuations = ('else', )
     name = 'in'
 
-    def __call__(self, blocks):
-        i = InClass(blocks)
+    def __call__(self, blocks, encoding=None):
+        i = InClass(blocks, encoding)
         if i.batch:
             return i.renderwb
         else:
@@ -375,7 +375,7 @@ class InClass(object):
     reverse = None
     sort_expr = reverse_expr = None
 
-    def __init__(self, blocks):
+    def __init__(self, blocks, encoding=None):
         tname, args, section = blocks[0]
         args = parse_params(args, name='', start='1', end='-1', size='10',
                             orphan='0', overlap='1', mapping=1,
@@ -385,6 +385,7 @@ def __init__(self, blocks):
                             reverse=1, sort_expr='', reverse_expr='',
                             prefix='')
         self.args = args
+        self.encoding = encoding
 
         if 'sort' in args:
             self.sort = sort = args['sort']
@@ -461,7 +462,7 @@ def renderwb(self, md):
 
         if not sequence:
             if self.elses:
-                return render_blocks(self.elses, md)
+                return render_blocks(self.elses, md, self.encoding)
             return ''
 
         if isinstance(sequence, str):
@@ -647,7 +648,7 @@ def renderwb(self, md):
                     if index == first:
                         pkw['sequence-start'] = 0
 
-                result = join_unicode(result)
+                result = join_unicode(result, self.encoding)
 
         finally:
             if cache:
@@ -669,7 +670,7 @@ def renderwob(self, md):
 
         if not sequence:
             if self.elses:
-                return render_blocks(self.elses, md)
+                return render_blocks(self.elses, md, self.encoding)
             return ''
 
         if isinstance(sequence, str):
@@ -759,7 +760,7 @@ def renderwob(self, md):
                 if index == 0:
                     pkw['sequence-start'] = 0
 
-            result = join_unicode(result)
+            result = join_unicode(result, self.encoding)
 
         finally:
             if cache:

src/DocumentTemplate/DT_Let.py

@@ -52,9 +52,10 @@ class Let(object):
     blockContinuations = ()
     name = 'let'
 
-    def __init__(self, blocks):
+    def __init__(self, blocks, encoding=None):
         tname, args, section = blocks[0]
         self.__name__ = args
+        self.encoding = encoding
         self.section = section.blocks
         self.args = args = parse_let_params(args)
 
@@ -81,7 +82,7 @@ def render(self, md):
                     d[name] = md[expr]
                 else:
                     d[name] = expr(md)
-            return render_blocks(self.section, md)
+            return render_blocks(self.section, md, self.encoding)
         finally:
             md._pop(1)
 

src/DocumentTemplate/DT_Raise.py

@@ -41,9 +41,10 @@ class Raise(object):
     name = 'raise'
     expr = ''
 
-    def __init__(self, blocks):
+    def __init__(self, blocks, encoding=None):
 
         tname, args, section = blocks[0]
+        self.encoding = encoding
         self.section = section.blocks
         args = parse_params(args, type='', expr='')
         self.__name__, self.expr = name_param(args, 'raise', 1, attr='type')
@@ -63,7 +64,7 @@ def render(self, md):
                     t = InvalidErrorTypeExpression
 
         try:
-            v = render_blocks(self.section, md)
+            v = render_blocks(self.section, md, self.encoding)
         except Exception:
             v = 'Invalid Error Value'
 

src/DocumentTemplate/DT_String.py

@@ -259,7 +259,7 @@ def parse_block(self, text, start, result, tagre,
                     sstart = start
                 else:
                     try:
-                        r = scommand(blocks)
+                        r = scommand(blocks, encoding=self.encoding)
                         if hasattr(r, 'simple_form'):
                             r = r.simple_form
                         result.append(r)
@@ -294,14 +294,15 @@ def parse_close(self, text, start, tagre, stag, sloc, scommand, sa):
     shared_globals = {}
 
     def __init__(self, source_string='', mapping=None, __name__='<string>',
-                 **vars):
+                 encoding=None, **vars):
         """\
         Create a document template from a string.
 
         The optional parameter, 'mapping', may be used to provide a
         mapping object containing defaults for values to be inserted.
         """
         self.raw = source_string
+        self.encoding = encoding
         self.initvars(mapping, vars)
         self.setName(__name__)
 
@@ -516,7 +517,7 @@ def __call__(self, client=None, mapping={}, **kw):
             value = self.ZDocumentTemplate_beforeRender(md, _marker)
             if value is _marker:
                 try:
-                    result = render_blocks(self._v_blocks, md)
+                    result = render_blocks(self._v_blocks, md, self.encoding)
                 except DTReturn as v:
                     result = v.v
                 self.ZDocumentTemplate_afterRender(md, result)

src/DocumentTemplate/DT_Try.py

@@ -148,7 +148,7 @@ def render_try_except(self, md):
 
         # first we try to render the first block
         try:
-            result = render_blocks(self.section, md)
+            result = render_blocks(self.section, md, self.encoding)
         except DTReturn:
             raise
         except Exception:
@@ -170,7 +170,7 @@ def render_try_except(self, md):
                 ns = namespace(md, error_type=errname, error_value=v,
                                error_tb=error_tb)[0]
                 md._push(InstanceDict(ns, md))
-                return render_blocks(handler, md)
+                return render_blocks(handler, md, self.encoding)
             finally:
                 md._pop(1)
 
@@ -179,16 +179,16 @@ def render_try_except(self, md):
             if (self.elseBlock is None):
                 return result
             else:
-                return result + render_blocks(self.elseBlock, md)
+                return result + render_blocks(self.elseBlock, md, self.encoding)
 
     def render_try_finally(self, md):
         result = ''
         # first try to render the first block
         try:
-            result = render_blocks(self.section, md)
+            result = render_blocks(self.section, md, self.encoding)
         # Then handle finally block
         finally:
-            result = result + render_blocks(self.finallyBlock, md)
+            result = result + render_blocks(self.finallyBlock, md, self.encoding)
         return result
 
     def find_handler(self, exception):

src/DocumentTemplate/DT_With.py

@@ -44,7 +44,7 @@ class With(object):
     mapping = None
     only = 0
 
-    def __init__(self, blocks):
+    def __init__(self, blocks, encoding=None):
         tname, args, section = blocks[0]
         args = parse_params(args, name='', expr='', mapping=1, only=1)
         name, expr = name_param(args, 'with', 1)
@@ -53,6 +53,7 @@ def __init__(self, blocks):
         else:
             expr = expr.eval
         self.__name__, self.expr = name, expr
+        self.encoding = encoding
         self.section = section.blocks
         if 'mapping' in args and args['mapping']:
             self.mapping = 1
@@ -81,7 +82,7 @@ def render(self, md):
 
         md._push(v)
         try:
-            return render_blocks(self.section, md)
+            return render_blocks(self.section, md, self.encoding)
         finally:
             md._pop(1)
 

src/DocumentTemplate/_DocumentTemplate.py

@@ -125,7 +125,7 @@
 _marker = object()
 
 
-def join_unicode(rendered):
+def join_unicode(rendered, encoding=None):
     """join a list of plain strings into a single plain string,
     a list of unicode strings into a single unicode strings,
     or a list containing a mix into a single unicode string with
@@ -136,27 +136,29 @@ def join_unicode(rendered):
     except UnicodeError:
         # A mix of unicode string and non-ascii plain strings.
         # Fix up the list, treating normal strings as UTF-8
+        if encoding is None:
+            encoding = _dt.DEFAULT_ENCODING
         rendered = list(rendered)
         for i in range(len(rendered)):
             if isinstance(rendered[i], str):
-                rendered[i] = unicode(rendered[i], _dt.DEFAULT_ENCODING)
+                rendered[i] = unicode(rendered[i], encoding)
         return u''.join(rendered)
 
 
-def render_blocks(blocks, md):
+def render_blocks(blocks, md, encoding=None):
     rendered = []
 
-    render_blocks_(blocks, rendered, md)
+    render_blocks_(blocks, rendered, md, encoding)
 
     l_ = len(rendered)
     if l_ == 0:
         return ''
     elif l_ == 1:
         return rendered[0]
-    return join_unicode(rendered)
+    return join_unicode(rendered, encoding)
 
 
-def render_blocks_(blocks, rendered, md):
+def render_blocks_(blocks, rendered, md, encoding):
     for block in blocks:
         append = True
 
@@ -229,7 +231,7 @@ def render_blocks_(blocks, rendered, md):
                         if cond:
                             block = block[icond + 2]
                             if block:
-                                render_blocks_(block, rendered, md)
+                                render_blocks_(block, rendered, md, encoding)
                             m = -1
                             break
 
@@ -238,7 +240,7 @@ def render_blocks_(blocks, rendered, md):
                     if icond == m:
                         block = block[icond + 1]
                         if block:
-                            render_blocks_(block, rendered, md)
+                            render_blocks_(block, rendered, md, encoding)
                 finally:
                     md._pop()
 

src/DocumentTemplate/tests/testDTMLUnicode.py

@@ -31,20 +31,19 @@ def __str__(self):
 
 
 class DTMLUnicodeTests(unittest.TestCase):
-    def setUp(self):
-        import DocumentTemplate as dt
-        self._encoding = dt.DEFAULT_ENCODING
-        dt.DEFAULT_ENCODING = "utf-8"
-
-    def tearDown(self):
-        import DocumentTemplate as dt
-        dt.DEFAULT_ENCODING = self._encoding
+    recode_to = "latin-1"
+    encoding_arg = None
 
     def _get_doc_class(self):
         from DocumentTemplate.DT_HTML import HTML
-        return HTML
+        def make(*args):
+            return HTML(*args, encoding=self.encoding_arg)
+        return make
     doc_class = property(_get_doc_class,)
 
+    def _recode(self, string):
+        return string.decode("utf-8").encode(self.recode_to)
+
     def testAA(self):
         html = self.doc_class('<dtml-var a><dtml-var b>')
         expected = 'helloworld'
@@ -65,32 +64,32 @@ def testAU(self):
 
     def testAB(self):
         html = self.doc_class('<dtml-var a><dtml-var b>')
-        expected = 'hello\xc8'
-        res = html(a='hello', b='\xc8')
+        expected = self._recode('hello\xc3\x88')
+        res = html(a='hello', b=self._recode('\xc3\x88'))
         self.assertEqual(res, expected)
 
     def testUB(self):
         html = self.doc_class('<dtml-var a><dtml-var b>')
         expected = u'hello\xc8'
-        res = html(a=u'hello', b='\xc3\x88')
+        res = html(a=u'hello', b=self._recode('\xc3\x88'))
         self.assertEqual(res, expected)
 
     def testUB2(self):
         html = self.doc_class('<dtml-var a><dtml-var b>')
         expected = u'\u07d0\xc8'
-        res = html(a=unichr(2000), b='\xc3\x88')
+        res = html(a=unichr(2000), b=self._recode('\xc3\x88'))
         self.assertEqual(res, expected)
 
     def testUnicodeStr(self):
         html = self.doc_class('<dtml-var a><dtml-var b>')
         expected = u'\u07d0\xc8'
-        res = html(a=force_str(unichr(2000)), b='\xc3\x88')
+        res = html(a=force_str(unichr(2000)), b=self._recode('\xc3\x88'))
         self.assertEqual(res, expected)
 
     def testUqB(self):
         html = self.doc_class('<dtml-var a html_quote><dtml-var b>')
         expected = u'he&gt;llo\xc8'
-        res = html(a=u'he>llo', b='\xc3\x88')
+        res = html(a=u'he>llo', b=self._recode('\xc3\x88'))
         self.assertEqual(res, expected)
 
     def testSize(self):
@@ -102,3 +101,22 @@ def testSize(self):
             expected = unichr(200) * 2 + '...'
         res = html()
         self.assertEqual(res, expected)
+
+
+class DTMLUnicodeTestsUTF8(DTMLUnicodeTests):
+    recode_to = "utf-8"
+    encoding_arg = "utf-8"
+
+
+class DTMLUnicodeTestsModuleGlobal(DTMLUnicodeTests):
+    encoding = "utf-8"
+    recode_to = "utf-8"
+
+    def setUp(self):
+        import DocumentTemplate as dt
+        self._default_encoding = dt.DEFAULT_ENCODING
+        dt.DEFAULT_ENCODING = self.encoding
+
+    def tearDown(self):
+        import DocumentTemplate as dt
+        dt.DEFAULT_ENCODING = self._default_encoding