Fix export of files with non-latin-1 compatible names

With Zope 4 unicode characters for ids are allowed.

The `zexp` export was not updated yet.

This has been fixed now.

closes #890

modified:   CHANGES.rst
modified:   src/OFS/ObjectManager.py
modified:   src/Testing/ZopeTestCase/testZODBCompat.py
new file:   src/ZPublisher/http_header_utils.py
new file:   src/ZPublisher/tests/test_http_header_utils.py

CHANGES.rst

@@ -11,6 +11,8 @@ https://github.com/zopefoundation/Zope/blob/4.x/CHANGES.rst
 5.0a3 (unreleased)
 ------------------
 
+- Fix export of files with non-latin-1 compatible names
+  (`#890 <https://github.com/zopefoundation/Zope/issues/890>`_)
 
 - Add ``pyupgrade`` via ``pre-commit``
   (`#859 <https://github.com/zopefoundation/Zope/issues/859>`_)

src/OFS/ObjectManager.py

@@ -61,6 +61,7 @@
 from zope.interface.interfaces import ComponentLookupError
 from zope.lifecycleevent import ObjectAddedEvent
 from zope.lifecycleevent import ObjectRemovedEvent
+from ZPublisher.http_header_utils import make_content_disposition
 
 
 # Constants: __replaceable__ flags:
@@ -611,8 +612,11 @@ def manage_exportObject(
 
             if RESPONSE is not None:
                 RESPONSE.setHeader('Content-type', 'application/data')
-                RESPONSE.setHeader('Content-Disposition',
-                                   f'inline;filename={id}.{suffix}')
+                RESPONSE.setHeader(
+                    'Content-Disposition',
+                    make_content_disposition(
+                        'inline', f'filename={id}.{suffix}')
+                )
             return result
 
         f = os.path.join(CONFIG.clienthome, f'{id}.{suffix}')

src/Testing/ZopeTestCase/testZODBCompat.py

@@ -32,6 +32,24 @@
 cutpaste_permissions = [add_documents_images_and_files, delete_objects]
 
 
+def make_request_response(environ=None):
+    from io import StringIO
+    from ZPublisher.HTTPRequest import HTTPRequest
+    from ZPublisher.HTTPResponse import HTTPResponse
+
+    if environ is None:
+        environ = {}
+
+    stdout = StringIO()
+    stdin = StringIO()
+    resp = HTTPResponse(stdout=stdout)
+    environ.setdefault('SERVER_NAME', 'foo')
+    environ.setdefault('SERVER_PORT', '80')
+    environ.setdefault('REQUEST_METHOD', 'GET')
+    req = HTTPRequest(stdin, environ, resp)
+    return req, resp
+
+
 class DummyObject(SimpleItem):
     id = 'dummy'
     foo = None
@@ -96,6 +114,8 @@ class TestImportExport(ZopeTestCase.ZopeTestCase):
     def afterSetUp(self):
         self.setupLocalEnvironment()
         self.folder.addDTMLMethod('doc', file='foo')
+        # please note the usage of the turkish i
+        self.folder.addDTMLMethod('ıq', file='foo')
         # _p_oids are None until we create a savepoint
         self.assertEqual(self.folder._p_oid, None)
         transaction.savepoint(optimistic=True)
@@ -105,6 +125,23 @@ def testExport(self):
         self.folder.manage_exportObject('doc')
         self.assertTrue(os.path.exists(self.zexp_file))
 
+    def testExportNonLatinFileNames(self):
+        """Test compatibility of the export with unicode characters.
+
+        Since Zope 4 also unicode ids can be used."""
+        _, response = make_request_response()
+        # please note the usage of a turkish `i`
+        self.folder.manage_exportObject(
+            'ıq', download=1, RESPONSE=response)
+
+        found = False
+        for header in response.listHeaders():
+            if header[0] == 'Content-Disposition':
+                # value needs to be latin-1 compatible
+                assert header[1].encode("latin-1")
+                found = True
+        self.assertTrue(found)
+
     def testImport(self):
         self.folder.manage_exportObject('doc')
         self.folder._delObject('doc')

src/ZPublisher/http_header_utils.py

@@ -0,0 +1,61 @@
+"""This module offers helpers to handle HTTP headers."""
+
+# The function `make_content_disposition` was vendored from our
+# friends from `CherryPy` - thank you!
+#
+# Copyright © 2004-2019, CherryPy Team (team@cherrypy.org)
+#
+# All rights reserved.
+#
+# * * *
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of CherryPy nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE  # noqa: E501
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import unicodedata
+import urllib
+
+
+def make_content_disposition(disposition, file_name):
+    """Create HTTP header for downloading a file with a UTF-8 filename.
+    This function implements the recommendations of :rfc:`6266#appendix-D`.
+    See this and related answers: https://stackoverflow.com/a/8996249/2173868.
+    """
+    # As normalization algorithm for `unicodedata` is used composed form (NFC
+    # and NFKC) with compatibility equivalence criteria (NFK), so "NFKC" is the
+    # one. It first applies the compatibility decomposition, followed by the
+    # canonical composition. Should be displayed in the same manner, should be
+    # treated in the same way by applications such as alphabetizing names or
+    # searching, and may be substituted for each other.
+    # See: https://en.wikipedia.org/wiki/Unicode_equivalence.
+    ascii_name = (
+        unicodedata.normalize('NFKC', file_name).
+        encode('ascii', errors='ignore').decode()
+    )
+    header = '{}; filename="{}"'.format(disposition, ascii_name)
+    if ascii_name != file_name:
+        quoted_name = urllib.parse.quote(file_name)
+        header += '; filename*=UTF-8\'\'{}'.format(quoted_name)
+    return header

src/ZPublisher/tests/test_http_header_utils.py

@@ -0,0 +1,37 @@
+##############################################################################
+#
+# Copyright (c) 2017 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import unittest
+
+from ZPublisher.http_header_utils import make_content_disposition
+
+
+class MakeDispositionHeaderTests(unittest.TestCase):
+
+    def test_ascii(self):
+        self.assertEqual(
+            make_content_disposition("inline", "iq.png"),
+            'inline; filename="iq.png"')
+
+    def test_unicode(self):
+        """HTTP headers need to be latin-1 compatible
+
+        In order to offer file downloads which contain unicode file names,
+        the file name has to be treated in a special way, see 
+        https://stackoverflow.com/questions/1361604 .
+        """
+        self.assertEqual(
+            make_content_disposition("inline", "ıq.png"),
+            """inline; filename="q.png"; filename*=UTF-8''%C4%B1q.png"""
+        )