Merge branch 'master' into issue_397

CHANGES.rst

@@ -44,6 +44,9 @@ Fixes
   Python 3.
   (`#577 <https://github.com/zopefoundation/Zope/pull/577>`_)
 
+- Prevent ``FindSupport.ZopeFind`` from throwing ``UnicodeDecodeErrors``
+  (`#594 <https://github.com/zopefoundation/Zope/issues/594>`_)
+
 Features
 ++++++++
 
@@ -363,7 +366,7 @@ New features
   `#307 <https://github.com/zopefoundation/Zope/pull/307>`_)
 
 - Add zconsole module for running scripts and interactive mode.
-  See the `document Running Zope 
+  See the `document Running Zope
   <https://zope.readthedocs.io/en/latest/operation.html#debugging-zope>`_.
 
 - Add support for Python 3.7.

src/OFS/FindSupport.py

@@ -135,11 +135,17 @@ def ZopeFindAndApply(self, obj, obj_ids=None, obj_metatypes=None,
                 if hasattr(ob, 'PrincipiaSearchSource'):
                     pss = ob.PrincipiaSearchSource()
                     if six.PY3 and not isinstance(pss, str):
-                        pss = pss.decode(default_encoding)
+                        try:
+                            pss = pss.decode(default_encoding)
+                        except UnicodeDecodeError:
+                            pss = ''
                 if hasattr(ob, 'SearchableText'):
                     st = ob.SearchableText()
                     if six.PY3 and not isinstance(st, str):
-                        st = st.decode(default_encoding)
+                        try:
+                            st = st.decode(default_encoding)
+                        except UnicodeDecodeError:
+                            st = ''
             else:
                 pss = st = ''
 

src/OFS/tests/testFindSupport.py

@@ -77,6 +77,22 @@ def test_find_text_nonascii(self):
         self.assertEqual(len(res), 2)
         self.assertEqual(set([x[0] for x in res]), set(['text', 'bytes']))
 
+    @unittest.skipIf(six.PY2, 'Not applicable under Python 2')
+    def test_find_text_nondecodable(self):
+        # Make sure ZopeFind does not crash searching text in nondecodable data
+        encoded = b'\xf6'
+        self.base['bytes'] = DummyItem('bytes', text=encoded)
+
+        def SearchableText():
+            return encoded
+        st_item = DummyItem('text')
+        st_item.SearchableText = SearchableText
+        self.base['text'] = st_item
+        try:
+            self.base.ZopeFind(self.base, obj_searchterm='anything')
+        except UnicodeDecodeError:  # pragma: no cover
+            self.fail('ZopeFind in undecodable data raises UnicodeDecodeError')
+
     def test_find_text_tainted(self):
         # Make sure ZopeFind can handle "Tainted" text for searches
         # Tainted strings are created when the publisher sees what appears