Merge acac3e8 into d26a77b

CHANGES.rst

@@ -8,11 +8,16 @@ The change log for the previous version, Zope 4, is at
 https://github.com/zopefoundation/Zope/blob/4.x/CHANGES.rst
 
 
-5.2.2 (unreleased)
-------------------
+5.3 (unreleased)
+----------------
 
 - Update to newest compatible versions of dependencies.
 
+- Deprecate usage of "unicode" converters. Also, the behavior of
+  ``field2lines`` is now aligned to the other converters and returns a list of
+  strings instead of a list of bytes.
+  (`#962 <https://github.com/zopefoundation/Zope/issues/962>`_)
+
 
 5.2.1 (2021-06-08)
 ------------------

docs/zdgbook/ObjectPublishing.rst

@@ -749,6 +749,10 @@ The publisher supports many converters:
 - **ulines**, **utokens**, **utext** -- like **lines**, **tokens**,
   **text**, but always converts into unicode strings.
 
+.. note::
+  Please note that the above listed **unicode converters** are deprecated,
+  and will be removed in Zope 6.
+
 The full list of supported converters can be found
 in ``ZPublisher.Converters.type_converters``.
 
@@ -772,6 +776,22 @@ could create a list of integers like so::
         <input type="checkbox" name="numbers:list:int" value="3">
 
 
+Create and register a custom converter
+++++++++++++++++++++++++++++++++++++++
+
+If you need a custom converter, you can create one on your own and register
+it as follows::
+
+        from ZPublisher.Converters import field2bytes, type_converters
+
+        def field2bytelines(v)
+            if isinstance(v, (list, tuple)):
+                return [field2bytes(item) for item in v]
+            return field2bytes(v).splitlines()
+
+        type_converters['bytelines'] = field2bytelines
+
+
 Aggregators
 ~~~~~~~~~~~
 

setup.py

@@ -29,7 +29,7 @@ def _read_file(filename):
 README = _read_file('README.rst')
 CHANGES = _read_file('CHANGES.rst')
 
-version = '5.2.2.dev0'
+version = '5.3.0.dev0'
 
 
 setup(

src/App/tests/testManagement.py

@@ -1,3 +1,5 @@
+import warnings
+
 import Testing.ZopeTestCase
 
 
@@ -18,21 +20,39 @@ def test_Management__Navigation__manage_page_header__1(self):
 
     def test_Management__Navigation__manage_page_header__2(self):
         """It respects `zmi_additional_css_paths` ustring property."""
-        self.folder.manage_addProperty(
-            'zmi_additional_css_paths', '/foo/bar.css', 'ustring')
+        # unicode converters will go away with Zope 6
+        # ignore deprecation warning for test run
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            self.folder.manage_addProperty(
+                'zmi_additional_css_paths', '/foo/bar.css', 'ustring')
         self.assertIn('href="/foo/bar.css"', self.folder.manage_page_header())
 
     def test_Management__Navigation__manage_page_header__3(self):
         """It respects `zmi_additional_css_paths` lines property."""
-        self.folder.manage_addProperty(
-            'zmi_additional_css_paths', ['/foo/bar.css', '/baz.css'], 'ulines')
+        # unicode converters will go away with Zope 6
+        # ignore deprecation warning for test run
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            self.folder.manage_addProperty(
+                'zmi_additional_css_paths',
+                ['/foo/bar.css', '/baz.css'],
+                'ulines'
+            )
         self.assertIn('href="/foo/bar.css"', self.folder.manage_page_header())
         self.assertIn('href="/baz.css"', self.folder.manage_page_header())
 
     def test_Management__Navigation__manage_page_header__4(self):
         """It respects `zmi_additional_css_paths` ulines property."""
-        self.folder.manage_addProperty(
-            'zmi_additional_css_paths', ['/foo/bar.css', '/baz.css'], 'ulines')
+        # unicode converters will go away with Zope 6
+        # ignore deprecation warning for test run
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            self.folder.manage_addProperty(
+                'zmi_additional_css_paths',
+                ['/foo/bar.css', '/baz.css'],
+                'ulines'
+            )
         self.assertIn('href="/foo/bar.css"', self.folder.manage_page_header())
         self.assertIn('href="/baz.css"', self.folder.manage_page_header())
 

src/OFS/tests/testProperties.py

@@ -90,10 +90,10 @@ def test_updateProperty_transforms(self):
         pm._setProperty('test_lines', [], type='lines')
 
         pm._updateProperty('test_lines', 'foo\nbar')
-        self.assertEqual(pm.getProperty('test_lines'), (b'foo', b'bar'))
+        self.assertEqual(pm.getProperty('test_lines'), ('foo', 'bar'))
 
         pm._updateProperty('test_lines', b'bar\nbaz')
-        self.assertEqual(pm.getProperty('test_lines'), (b'bar', b'baz'))
+        self.assertEqual(pm.getProperty('test_lines'), ('bar', 'baz'))
 
 
 class TestPropertySheets(unittest.TestCase):
@@ -147,7 +147,7 @@ def test_updateProperty_transforms(self):
         ps._setProperty('test_lines', [], type='lines')
 
         ps._updateProperty('test_lines', 'foo\nbar')
-        self.assertEqual(ps.getProperty('test_lines'), (b'foo', b'bar'))
+        self.assertEqual(ps.getProperty('test_lines'), ('foo', 'bar'))
 
         ps._updateProperty('test_lines', b'bar\nbaz')
-        self.assertEqual(ps.getProperty('test_lines'), (b'bar', b'baz'))
+        self.assertEqual(ps.getProperty('test_lines'), ('bar', 'baz'))

src/ZPublisher/Converters.py

@@ -13,6 +13,7 @@
 
 import html
 import re
+import warnings
 
 from DateTime import DateTime
 from DateTime.interfaces import SyntaxError
@@ -22,14 +23,19 @@
 default_encoding = 'utf-8'
 
 
-def field2string(v):
-    """Converts value to string."""
+def convert_single_value_to_string(v):
+    """Converts a single value to string."""
     if isinstance(v, bytes):
         return v.decode(default_encoding)
     else:
         return str(v)
 
 
+def field2string(v):
+    """Converts value to string."""
+    return convert_single_value_to_string(v)
+
+
 def field2bytes(v):
     """Converts value to bytes."""
     if hasattr(v, 'read'):
@@ -41,7 +47,7 @@ def field2bytes(v):
 
 
 def field2text(value, nl=re.compile('\r\n|\n\r').search):
-    value = field2string(value)
+    value = convert_single_value_to_string(value)
     match_object = nl(value)
     if match_object is None:
         return value
@@ -63,7 +69,7 @@ def field2text(value, nl=re.compile('\r\n|\n\r').search):
 
 
 def field2required(v):
-    v = field2string(v)
+    v = convert_single_value_to_string(v)
     if v.strip():
         return v
     raise ValueError('No input for required field<p>')
@@ -72,7 +78,7 @@ def field2required(v):
 def field2int(v):
     if isinstance(v, (list, tuple)):
         return list(map(field2int, v))
-    v = field2string(v)
+    v = convert_single_value_to_string(v)
     if v:
         try:
             return int(v)
@@ -88,7 +94,7 @@ def field2int(v):
 def field2float(v):
     if isinstance(v, (list, tuple)):
         return list(map(field2float, v))
-    v = field2string(v)
+    v = convert_single_value_to_string(v)
     if v:
         try:
             return float(v)
@@ -104,7 +110,7 @@ def field2float(v):
 def field2long(v):
     if isinstance(v, (list, tuple)):
         return list(map(field2long, v))
-    v = field2string(v)
+    v = convert_single_value_to_string(v)
     # handle trailing 'L' if present.
     if v[-1:] in ('L', 'l'):
         v = v[:-1]
@@ -120,21 +126,18 @@ def field2long(v):
 
 
 def field2tokens(v):
-    v = field2string(v)
+    v = convert_single_value_to_string(v)
     return v.split()
 
 
 def field2lines(v):
     if isinstance(v, (list, tuple)):
-        result = []
-        for item in v:
-            result.append(field2bytes(item))
-        return result
-    return field2bytes(v).splitlines()
+        return [convert_single_value_to_string(item) for item in v]
+    return convert_single_value_to_string(v).splitlines()
 
 
 def field2date(v):
-    v = field2string(v)
+    v = convert_single_value_to_string(v)
     try:
         v = DateTime(v)
     except SyntaxError:
@@ -143,7 +146,7 @@ def field2date(v):
 
 
 def field2date_international(v):
-    v = field2string(v)
+    v = convert_single_value_to_string(v)
     try:
         v = DateTime(v, datefmt="international")
     except SyntaxError:
@@ -157,68 +160,47 @@ def field2boolean(v):
     return bool(v)
 
 
-class _unicode_converter:
-
-    def __call__(self, v):
-        # Convert a regular python string. This probably doesn't do
-        # what you want, whatever that might be. If you are getting
-        # exceptions below, you probably missed the encoding tag
-        # from a form field name. Use:
-        #       <input name="description:utf8:ustring" .....
-        # rather than
-        #       <input name="description:ustring" .....
-        v = str(v)
-        return self.convert_unicode(v)
-
-    def convert_unicode(self, v):
-        raise NotImplementedError('convert_unicode')
-
-
-class field2ustring(_unicode_converter):
-    def convert_unicode(self, v):
-        return v
-
-
-field2ustring = field2ustring()
-
-
-class field2utokens(_unicode_converter):
-    def convert_unicode(self, v):
-        return v.split()
-
-
-field2utokens = field2utokens()
-
-
-class field2utext(_unicode_converter):
-    def convert_unicode(self, v):
-        if isinstance(v, bytes):
-            return str(field2text(v.encode('utf8')), 'utf8')
-        return v
-
+def field2ustring(v):
+    warnings.warn(
+        "The converter `(field2)ustring` is deprecated "
+        "and will be removed in Zope 6. "
+        "Please use `(field2)string` instead.",
+        DeprecationWarning)
+    return field2string(v)
 
-field2utext = field2utext()
 
+def field2utokens(v):
+    warnings.warn(
+        "The converter `(field2)utokens` is deprecated "
+        "and will be removed in Zope 6. "
+        "Please use `(field2)tokens` instead.",
+        DeprecationWarning)
+    return field2tokens(v)
 
-class field2ulines:
-    def __call__(self, v):
-        if isinstance(v, (list, tuple)):
-            return [field2ustring(x) for x in v]
-        v = str(v)
-        return self.convert_unicode(v)
 
-    def convert_unicode(self, v):
-        return field2utext.convert_unicode(v).splitlines()
+def field2utext(v):
+    warnings.warn(
+        "The converter `(field2)utext` is deprecated "
+        "and will be removed in Zope 6. "
+        "Please use `(field2)text` instead.",
+        DeprecationWarning)
+    return field2text(v)
 
 
-field2ulines = field2ulines()
+def field2ulines(v):
+    warnings.warn(
+        "The converter `(field2u)lines` is deprecated "
+        "and will be removed in Zope 6. "
+        "Please use `(field2)lines` instead.",
+        DeprecationWarning)
+    return field2lines(v)
 
 
 type_converters = {
     'float': field2float,
     'int': field2int,
     'long': field2long,
-    'string': field2string,  # to native str
+    'string': field2string,
     'bytes': field2bytes,
     'date': field2date,
     'date_international': field2date_international,