diff --git a/plone/app/blocks/tests/test_transforms.py b/plone/app/blocks/tests/test_transforms.py index a3a56bf9..4102f312 100644 --- a/plone/app/blocks/tests/test_transforms.py +++ b/plone/app/blocks/tests/test_transforms.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from plone.app.blocks.interfaces import IBlocksLayer from plone.app.blocks.interfaces import IBlocksTransformEnabled +from plone.app.blocks.transform import ParseXML from plone.app.blocks.testing import BLOCKS_INTEGRATION_TESTING from plone.transformchain.zpublisher import applyTransform from zope.interface import alsoProvides @@ -10,41 +11,49 @@ @implementer(IBlocksTransformEnabled) -class TestTransformedView(object): +class TransformedView(object): def __init__(self, ret_body): - self.__call__ = lambda b=ret_body: b + self.body = ret_body + def __call__(self): + return self.body -class TestTransforms(unittest.TestCase): +class BaseTestCase(unittest.TestCase): layer = BLOCKS_INTEGRATION_TESTING + def prepare_request(self, body=None): + if body is None: + body = """\ + + +Empty + +""" + request = self.layer['request'] + request.set('PUBLISHED', TransformedView(body)) + request.response.setBase(request.getURL()) + request.response.setHeader('content-type', 'text/html') + request.response.setBody(body) + alsoProvides(request, IBlocksLayer) + return request + + +class TestTransforms(BaseTestCase): + def test_transforms_with_crlf(self): """Test fix for issue where layouts with CR[+LF] line-endings are somehow turned into having line-endings and getting their heads being dropped """ - - @implementer(IBlocksTransformEnabled) - class TransformedView(object): - - def __init__(self, ret_body): - self.__call__ = lambda b=ret_body: b - body = """\ """ - request = self.layer['request'] - request.set('PUBLISHED', TransformedView(body)) - request.response.setBase(request.getURL()) - request.response.setHeader('content-type', 'text/html') - request.response.setBody(body) - - alsoProvides(request, IBlocksLayer) + request = self.prepare_request(body) result = applyTransform(request) self.assertIn('', ''.join(str(result))) @@ -52,25 +61,89 @@ def test_transforms_with_cdata(self): """Test fix for issue where layouts with inline js got rendered with quoted (and therefore broken) block """ - - @implementer(IBlocksTransformEnabled) - class TransformedView(object): - - def __init__(self, ret_body): - self.__call__ = lambda b=ret_body: b - body = """\ """ - request = self.layer['request'] - request.set('PUBLISHED', TransformedView(body)) - request.response.setBase(request.getURL()) - request.response.setHeader('content-type', 'text/html') - request.response.setBody(body) - - alsoProvides(request, IBlocksLayer) + request = self.prepare_request(body) result = applyTransform(request) self.assertIn('', ''.join(str(result))) + + +class TestParseXML(BaseTestCase): + """Test our XMLParser transform with only bytes. + + Things can go wrong when there is more than one item in the iterable, + especially when one item is bytes and the other is text. + Or at least that is a way that I can more or less reproduce some problems. + See https://github.com/plone/plone.app.mosaic/issues/480 + + I test various combinations in this and the next test methods. + """ + + def test_transformBytes_method(self): + one = b"

one

" + request = self.prepare_request() + parser = ParseXML(request.get("PUBLISHED"), request) + result = parser.transformBytes(one, encoding="utf-8") + html = result.serialize() + self.assertIn(one, html) + + def test_transformUnicode_method(self): + one = b"

one

" + request = self.prepare_request() + parser = ParseXML(request.get("PUBLISHED"), request) + result = parser.transformBytes(one.decode("utf-8"), encoding="utf-8") + html = result.serialize() + self.assertIn(one, html) + + # The rest of the tests use the transformIterable method. + # We use a helper method 'transform' to make this easier. + + def transform(self, iterable): + request = self.prepare_request() + parser = ParseXML(request.get("PUBLISHED"), request) + result = parser.transformIterable(iterable, encoding="utf-8") + return result.serialize() + + def test_transform_one_byte(self): + one = b"

one

" + html = self.transform([one]) + self.assertIn(one, html) + + def test_transform_one_unicode(self): + one = b"

one

" + # Note: decoding creates a unicode (string on PY3). + html = self.transform([one.decode("utf-8")]) + # Note: the html result is always bytes, so we must compare with bytes. + self.assertIn(one, html) + + def test_transform_two_bytes(self): + one = b"

one

" + two = b"

two

" + html = self.transform([one, two]) + self.assertIn(one, html) + self.assertIn(two, html) + + def test_transform_two_unicodes(self): + one = b"

one

" + two = b"

two

" + html = self.transform([one.decode("utf-8"), two.decode("utf-8")]) + self.assertIn(one, html) + self.assertIn(two, html) + + def test_transform_byte_unicode(self): + one = b"

one

" + two = b"

two

" + html = self.transform([one, two.decode("utf-8")]) + self.assertIn(one, html) + self.assertIn(two, html) + + def test_transform_unicode_byte(self): + one = b"

one

" + two = b"

two

" + html = self.transform([one.decode("utf-8"), two]) + self.assertIn(one, html) + self.assertIn(two, html) diff --git a/plone/app/blocks/transform.py b/plone/app/blocks/transform.py index e8fb14fe..178e1bed 100644 --- a/plone/app/blocks/transform.py +++ b/plone/app/blocks/transform.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from Products.CMFPlone.utils import safe_unicode from lxml import etree from lxml import html from plone.app.blocks import panel @@ -15,6 +14,13 @@ import re import logging +try: + # Plone 5.2+ + from Products.CMFPlone.utils import safe_bytes +except ImportError: + # BBB for Plone 5.1 and lower + from Products.CMFPlone.utils import safe_encode as safe_bytes + logger = logging.getLogger(__name__) @@ -66,7 +72,6 @@ def __init__(self, published, request): self.request = request def transformBytes(self, result, encoding): - result = safe_unicode(result, encoding) return self.transformIterable([result], encoding) def transformUnicode(self, result, encoding): @@ -89,31 +94,24 @@ def transformIterable(self, result, encoding): try: # Fix layouts with CR[+LF] line endings not to lose their heads # (this has been seen with downloaded themes with CR[+LF] endings) + # The html serializer much prefers only bytes, no unicode/text, + # and it return a serializer that returns bytes. + # So we start with ensuring all items in the iterable are bytes. + iterable = [ + re.sub(b' ', b'\n', re.sub(b' \n', b'\n', safe_bytes(item))) + for item in result if item] + result = getHTMLSerializer( + iterable, pretty_print=self.pretty_print, encoding=encoding) + # Fix XHTML layouts with where etree.tostring breaks