Decode URL data using UTF-8 instead of the default.

Fixes #37

CHANGES.rst

@@ -11,6 +11,9 @@ Change History for ZConfig
 
 - Drop support for Python 3.3.
 
+- Fix configuration loaders to decode byte data using UTF-8 instead of
+  the default encoding (usually ASCII). See `issue 37
+  <https://github.com/zopefoundation/ZConfig/issues/37>`_.
 
 3.2.0 (2017-06-22)
 ------------------

ZConfig/loader.py

@@ -227,9 +227,13 @@ def openResource(self, url):
             # open for an unbounded amount of time and to catch IOErrors here,
             # where they make sense.
             try:
-                data = file.read().decode()
+                data = file.read()
             finally:
                 file.close()
+            if isinstance(data, bytes):
+                # Be sure to specify an (useful) encoding so we don't get
+                # the system default, typically ascii.
+                data = data.decode('utf-8')
             file = StringIO(data)
         return self.createResource(file, url)
 

ZConfig/tests/input/non-ascii.txt

@@ -0,0 +1,2 @@
+# -*-coding: utf-8; mode: conf-*-
+This file contains a snowman, U+2603: ☃

ZConfig/tests/test_loader.py

@@ -34,12 +34,24 @@
 
 class LoaderTestCase(TestHelper, unittest.TestCase):
 
+    def test_open_resource_non_ascii(self):
+        # Files are decoded using utf-8 on open
+        loader = ZConfig.loader.SchemaLoader()
+        url = ZConfig.url.urljoin(CONFIG_BASE, "non-ascii.txt")
+        stream = loader.openResource(url)
+        val = stream.read()
+        self.assertEqual(
+            val,
+            u'# -*-coding: utf-8; mode: conf-*-\n'
+            u'This file contains a snowman, U+2603: \u2603\n'
+        )
+
     def test_schema_caching(self):
         loader = ZConfig.loader.SchemaLoader()
         url = ZConfig.url.urljoin(CONFIG_BASE, "simple.xml")
         schema1 = loader.loadURL(url)
         schema2 = loader.loadURL(url)
-        self.assertTrue(schema1 is schema2)
+        self.assertIs(schema1, schema2)
 
     def test_simple_import_with_cache(self):
         loader = ZConfig.loader.SchemaLoader()