Unicode test for file loads #74

mappyfile.pyproj

@@ -10,7 +10,7 @@
     <OutputPath>.</OutputPath>
     <Name>mappyfile</Name>
     <RootNamespace>mappyfile</RootNamespace>
-    <InterpreterId>MSBuild|mappyfile|$(MSBuildProjectFullPath)</InterpreterId>
+    <InterpreterId>MSBuild|mappyfile3|$(MSBuildProjectFullPath)</InterpreterId>
     <StartupFile>tests\test_comments.py</StartupFile>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
@@ -271,6 +271,7 @@
     <Content Include="tests\samples\pmapper_demo.map">
       <SubType>Code</SubType>
     </Content>
+    <Content Include="tests\samples\unicode.map" />
     <Content Include="tests\sample_maps\256color_overdose.map" />
     <Content Include="tests\sample_maps\256color_overdose_cmt.map" />
     <Content Include="tests\sample_maps\256color_overdose_res.map" />

mappyfile/parser.py

@@ -186,7 +186,7 @@ def parse(self, text, fn=None):
         Parse the Mapfile
         """
 
-        text = str(text)
+        text = str(text) # ensure the input is a unicode string
         if self.expand_includes:
             text = self.load_includes(text, fn=fn)
 

mappyfile/utils.py

@@ -83,7 +83,7 @@ def load(fp, expand_includes=True, include_position=False, include_comments=Fals
     ----------
 
     fp: file
-        A file-like object
+        A file-like object - as with all Mapfiles this should be encoded in "utf-8"
     expand_includes: boolean
         Load any ``INCLUDE`` files in the MapFile
     include_comments: boolean

tests/samples/unicode.map

@@ -0,0 +1,23 @@
+		CLASS
+
+			NAME 'Enw’r stryd ddynodedig'
+
+			EXPRESSION /^Designated street.*$/
+
+			STYLE
+
+				#SYMBOL 'line'
+
+				COLOR 255 0 0#red
+
+				OPACITY 100
+
+				OUTLINECOLOR 255 0 0
+
+				WIDTH 2
+
+			END
+
+
+
+		END

tests/test_sample_maps.py

@@ -114,13 +114,21 @@ def test_non_ascii():
     print(mappyfile.dumps(d))
 
 
+def test_unicode_map():
+
+    with open('./tests/samples/unicode.map', "r", encoding="utf-8") as mf_file:
+        mf = mappyfile.load(mf_file)
+
+    print(mappyfile.dumps(mf))
+
+
 def run_tests():
     pytest.main(["tests/test_sample_maps.py"])
 
 
 if __name__ == '__main__':
     logging.basicConfig(level=logging.DEBUG)
     logging.getLogger('mappyfile').setLevel(logging.INFO)
-    run_tests()
-    # test_includes()
+    # run_tests()
+    test_unicode_map()
     print("Done!")