Merge pull request #224 from jmwri/master

Prevent the 'u' prefix on keys for Python 2 and Python 3 unicode conversions

voluptuous/schema_builder.py

@@ -260,12 +260,17 @@ def _compile_mapping(self, schema, invalid_msg=None):
         candidates = list(_iterate_mapping_candidates(_compiled_schema))
 
         def validate_mapping(path, iterable, out):
+            try:
+                from util import to_utf8_py2
+            except ImportError:
+                from .util import to_utf8_py2
             required_keys = all_required_keys.copy()
             # keeps track of all default keys that haven't been filled
             default_keys = all_default_keys.copy()
             error = None
             errors = []
             for key, value in iterable:
+                key = to_utf8_py2(key)
                 key_path = path + [key]
                 remove_key = False
 
@@ -809,6 +814,11 @@ class Marker(object):
     """Mark nodes for special treatment."""
 
     def __init__(self, schema_, msg=None):
+        try:
+            from util import to_utf8_py2
+        except ImportError:
+            from .util import to_utf8_py2
+        schema_ = to_utf8_py2(schema_)
         self.schema = schema_
         self._schema = Schema(schema_)
         self.msg = msg

voluptuous/tests/tests.py

@@ -1,5 +1,5 @@
 import copy
-from nose.tools import assert_equal, assert_raises
+from nose.tools import assert_equal, assert_raises, assert_true
 
 from voluptuous import (
     Schema, Required, Extra, Invalid, In, Remove, Literal,
@@ -8,6 +8,7 @@
     validate, ExactSequence, Equal, Unordered, Number
 )
 from voluptuous.humanize import humanize_error
+from voluptuous.util import to_utf8_py2, u
 
 
 def test_exact_sequence():
@@ -558,6 +559,17 @@ def fn(arg):
     assert_raises(Invalid, fn, 1)
 
 
+def test_unicode_key_is_converted_to_utf8_when_in_marker():
+    """Verify that when using unicode key the 'u' prefix is not thrown in the exception"""
+    schema = Schema({Required(u('q')): 1})
+    # Can't use nose's raises (because we need to access the raised
+    # exception, nor assert_raises which fails with Python 2.6.9.
+    try:
+        schema({})
+    except Invalid as e:
+        assert_equal(str(e), "required key not provided @ data['q']")
+
+
 def test_number_validation_with_string():
     """ test with Number with string"""
     schema = Schema({"number" : Number(precision=6, scale=2)})
@@ -570,6 +582,17 @@ def test_number_validation_with_string():
         assert False, "Did not raise Invalid for String"
 
 
+def test_unicode_key_is_converted_to_utf8_when_plain_text():
+    key = u('q')
+    schema = Schema({key: int})
+    # Can't use nose's raises (because we need to access the raised
+    # exception, nor assert_raises which fails with Python 2.6.9.
+    try:
+        schema({key: 'will fail'})
+    except Invalid as e:
+        assert_equal(str(e), "expected int for dictionary value @ data['q']")
+
+
 def test_number_validation_with_invalid_precision_invalid_scale():
     """ test with Number with invalid precision and scale"""
     schema = Schema({"number" : Number(precision=6, scale=2)})
@@ -610,6 +633,11 @@ def test_number_when_precision_none_n_valid_scale_case2_yield_decimal_true():
     assert_equal(float(out_.get("number")), 123456789012.00)
 
 
+def test_to_utf8():
+    s = u('hello')
+    assert_true(isinstance(to_utf8_py2(s), str))
+
+
 def test_number_when_precision_none_n_invalid_scale_yield_decimal_true():
     """ test with Number with no precision and invalid scale"""
     schema = Schema({"number" : Number(scale=2, yield_decimal=True)})

voluptuous/util.py

@@ -1,3 +1,5 @@
+import sys
+
 try:
     from error import LiteralInvalid, TypeInvalid, Invalid
     from schema_builder import Schema, default_factory, raises
@@ -148,3 +150,16 @@ def __str__(self):
 
     def __repr__(self):
         return repr(self.lit)
+
+
+def to_utf8_py2(data):
+    if sys.version_info < (3,) and isinstance(data, unicode):
+        return data.encode('utf-8')
+    return data
+
+
+def u(x):
+    if sys.version_info < (3,):
+        return unicode(x)
+    else:
+        return x