Merge 01d63dd into f5b02ee

tests/test_binary_vdf.py

@@ -123,3 +123,8 @@ def test_vbkv_loads_invalid_checksum(self):
         with self.assertRaises(ValueError):
             vdf.vbkv_loads(b'VBKV\x01\x02\x03\x04\x00a\x00\x0b\x0b')
 
+    def test_loads_utf8_invalmid(self):
+        self.assertEqual({'aaa': b'bb\xef\xbf\xbdbb'.decode('utf-8')}, vdf.binary_loads(b'\x01aaa\x00bb\xffbb\x00\x08'))
+
+    def test_loads_utf16(self):
+        self.assertEqual({'aaa': b'b\x00b\x00\xff\xffb\x00b\x00'.decode('utf-16le')}, vdf.binary_loads(b'\x05aaa\x00b\x00b\x00\xff\xffb\x00b\x00\x00\x00\x08'))

vdf/__init__.py

@@ -295,19 +295,25 @@ def binary_loads(s, mapper=dict, merge_duplicate_keys=True, alt_format=False):
     float32 = struct.Struct('<f')
 
     def read_string(s, idx, wide=False):
-        end = s.find(b'\x00\x00' if wide else b'\x00', idx)
+        if wide:
+            end = s.find(b'\x00\x00', idx)
+            if (end - idx) % 2 != 0:
+                end += 1
+        else:
+            end = s.find(b'\x00', idx)
+
         if end == -1:
             raise SyntaxError("Unterminated cstring, index: %d" % idx)
         result = s[idx:end]
         if wide:
             result = result.decode('utf-16')
         elif bytes is not str:
-            result = result.decode('utf-8')
+            result = result.decode('utf-8', 'replace')
         else:
             try:
                 result.decode('ascii')
             except:
-                result = result.decode('utf-8')
+                result = result.decode('utf-8', 'replace')
         return result, end + (2 if wide else 1)
 
     stack = [mapper()]