Fix richtext encoding in deserializer (#913)

* use unescape to avoid encoding errors on save

* add tests

* add changelog

news/913.bugfix

@@ -0,0 +1,2 @@
+call unescape method on received html for richtext before save it in Plone.
+[cekk]

src/plone/restapi/deserializer/dxfields.py

@@ -29,6 +29,12 @@
 import dateutil
 import six
 
+if six.PY2:
+    import HTMLParser
+    html_parser = HTMLParser.HTMLParser()
+else:
+    import html as html_parser
+
 
 @implementer(IFieldDeserializer)
 @adapter(IField, IDexterityContent, IBrowserRequest)
@@ -274,9 +280,8 @@ def __call__(self, value):
                 data = f.read().decode("utf8")
         else:
             data = value
-
         value = RichTextValue(
-            raw=data,
+            raw=html_parser.unescape(data),
             mimeType=content_type,
             outputMimeType=self.field.output_mime_type,
             encoding=encoding,

src/plone/restapi/tests/test_content_patch.py

@@ -17,6 +17,7 @@
 from zope.lifecycleevent.interfaces import IObjectCreatedEvent
 from zope.lifecycleevent.interfaces import IObjectModifiedEvent
 
+import json
 import requests
 import transaction
 import unittest
@@ -182,6 +183,24 @@ def record_event(event):
         sm.unregisterHandler(record_event, (IObjectAddedEvent,))
         sm.unregisterHandler(record_event, (IObjectModifiedEvent,))
 
+    def test_patch_document_with_apostrophe_dont_return_500(self):
+        data = {
+            "text": {
+                "content-type": "text/html",
+                "encoding": "utf8",
+                "data": "<p>example with &#x27;</p>"
+            }
+        }
+        response = requests.patch(
+            self.portal.doc1.absolute_url(),
+            headers={"Accept": "application/json"},
+            auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD),
+            data=json.dumps(data),
+        )
+        self.assertEqual(204, response.status_code)
+        transaction.begin()
+        self.assertEqual("<p>example with '</p>", self.portal.doc1.text.raw)
+
 
 class TestATContentPatch(unittest.TestCase):
 

src/plone/restapi/tests/test_content_post.py

@@ -182,6 +182,27 @@ def record_event(event):
         sm.unregisterHandler(record_event, (IObjectAddedEvent,))
         sm.unregisterHandler(record_event, (IObjectModifiedEvent,))
 
+    def test_post_to_folder_with_apostrophe_dont_return_500(self):
+        response = requests.post(
+            self.portal.folder1.absolute_url(),
+            headers={"Accept": "application/json"},
+            auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD),
+            json={
+                "@type": "Document",
+                "id": "mydocument2",
+                "title": "My Document 2",
+                "text": {
+                    "content-type": "text/html",
+                    "encoding": "utf8",
+                    "data": "<p>example with &#x27;</p>"
+                }
+            },
+        )
+        self.assertEqual(201, response.status_code)
+        transaction.begin()
+        self.assertEqual("<p>example with '</p>", self.portal.folder1.mydocument2.text.raw)
+        self.assertEqual("<p>example with '</p>", response.json()['text']['data'])
+
 
 class TestATFolderCreate(unittest.TestCase):
 

src/plone/restapi/tests/test_dxfield_deserializer.py

@@ -301,6 +301,10 @@ def test_richtext_deserialization_sets_encoding(self):
         )
         self.assertEqual("latin1", value.encoding)
 
+    def test_richtext_deserialization_fix_apostrophe(self):
+        value = self.deserialize("test_richtext_field", u"<p>char with &#x27;</p>")
+        self.assertEqual("<p>char with '</p>", value.raw)
+
     def test_namedfield_deserialization_decodes_value(self):
         value = self.deserialize(
             "test_namedfile_field",