Merge pull request #13 from zopefoundation/decode-compatibitlity

Improve the unicode compatibility between Python 2 and 3

CHANGES.rst

@@ -8,6 +8,9 @@ Changes
 
 - Drop support for Python 2.6 and 3.3.
 
+- Use ``UTF-8`` as default encoding when casting bytes to unicode for Python 2
+  *and* 3.
+
 
 4.3.0 (2014-12-24)
 ==================

src/zope/formlib/_compat.py

@@ -35,8 +35,14 @@ def safeBase64Encode(obj):
 
     from StringIO import StringIO
     from itertools import imap
-    unicode = toUnicode = unicode
+    unicode = unicode
     basestring = basestring
 
+    def toUnicode(obj):
+        if isinstance(obj, bytes):
+            return unicode(obj, 'utf-8')
+        else:
+            return unicode(obj)
+
     def safeBase64Encode(obj):
         return base64.b64encode(toUnicode(obj)).strip().replace('=', '_')

src/zope/formlib/tests/test_textwidget.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 ##############################################################################
 #
 # Copyright (c) 2001, 2002, 2004, 2005 Zope Foundation and Contributors.
@@ -132,6 +133,14 @@ def testRender(self):
         self._widget.extra = 'style="color: red"'
         self.verifyResult(self._widget.hidden(), check_list)
 
+    def testRenderUTF8Input(self):
+        value = u"☃".encode('utf-8')  # results in \u2603
+        self._widget.setRenderedValue(value)
+        check_list = ('type="text"', 'id="field.foo"', 'name="field.foo"',
+                      u'value="\u2603"', 'size="20"')
+        self.verifyResult(self._widget(), check_list)
+
+
 class URIDisplayWidgetTest(BrowserWidgetTest):
 
     _WidgetFactory = URIDisplayWidget