From 0eecd98840cfbafab415eb6af1b55442235fc946 Mon Sep 17 00:00:00 2001 From: Hans Kirchner Date: Wed, 20 Dec 2017 02:28:15 +0100 Subject: [PATCH 1/8] Fix issue with SetValidator and large valid_set (#50) A new function stringify_set only stringifies n elements of a given set. --- vladiate/validators.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/vladiate/validators.py b/vladiate/validators.py index 72dccdb..2a4e29b 100644 --- a/vladiate/validators.py +++ b/vladiate/validators.py @@ -70,7 +70,8 @@ def validate(self, field, row={}): if field not in self.valid_set: self.invalid_set.add(field) raise ValidationException( - "'{}' is not in {}".format(field, self.valid_set)) + "'{}' is not in {}".format(field, + stringify_set(self.valid_set, 100))) @property def bad(self): @@ -208,3 +209,14 @@ def validate(self, field, row={}): @property def bad(self): pass + + +def stringify_set(a_set, max_len): + ''' Stringify `max_len` elements of `a_set` and count the remainings ''' + # Don't convert `a_set` to a list for performance reasons + text = "[{}]".format(", ".join( + "'{}'".format(value) for _, value in zip(range(max_len), a_set) + )) + if len(a_set) > max_len: + text += " ({} more suppressed)".format(len(a_set) - max_len) + return text From 28561dac90d94c44878275c908097e880ba677bb Mon Sep 17 00:00:00 2001 From: Hans Kirchner Date: Fri, 22 Dec 2017 07:27:43 +0100 Subject: [PATCH 2/8] Use itertools.islice for better python2 performance --- vladiate/validators.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vladiate/validators.py b/vladiate/validators.py index 2a4e29b..6de043f 100644 --- a/vladiate/validators.py +++ b/vladiate/validators.py @@ -1,4 +1,5 @@ import re +from itertools import islice from vladiate.exceptions import ValidationException, BadValidatorException @@ -215,7 +216,7 @@ def stringify_set(a_set, max_len): ''' Stringify `max_len` elements of `a_set` and count the remainings ''' # Don't convert `a_set` to a list for performance reasons text = "[{}]".format(", ".join( - "'{}'".format(value) for _, value in zip(range(max_len), a_set) + "'{}'".format(value) for value in islice(a_set, max_len) )) if len(a_set) > max_len: text += " ({} more suppressed)".format(len(a_set) - max_len) From 4c75caa2e57f8f2f148fc63a94b7b20cb9131083 Mon Sep 17 00:00:00 2001 From: Hans Kirchner Date: Fri, 22 Dec 2017 07:58:59 +0100 Subject: [PATCH 3/8] Add SetValidator test for coverage --- vladiate/test/test_validators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vladiate/test/test_validators.py b/vladiate/test/test_validators.py index 369e9a4..df11030 100644 --- a/vladiate/test/test_validators.py +++ b/vladiate/test/test_validators.py @@ -85,6 +85,7 @@ def test_set_validator_works(field_set, field): ([], 'bar'), (['foo'], 'bar'), (['foo', 'bar'], 'baz'), + ([str(x) for x in range(200)], "notanumber"), ]) def test_set_validator_fails(field_set, field): validator = SetValidator(field_set) From c8e71f37a9c1b9f11de290f2b32745e4661f8186 Mon Sep 17 00:00:00 2001 From: Hans Kirchner Date: Sat, 30 Dec 2017 18:33:47 +0100 Subject: [PATCH 4/8] Fix stringify_set: Return {...} instead of [...] --- vladiate/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vladiate/validators.py b/vladiate/validators.py index 6de043f..5166ec7 100644 --- a/vladiate/validators.py +++ b/vladiate/validators.py @@ -215,7 +215,7 @@ def bad(self): def stringify_set(a_set, max_len): ''' Stringify `max_len` elements of `a_set` and count the remainings ''' # Don't convert `a_set` to a list for performance reasons - text = "[{}]".format(", ".join( + text = "{{{}}}".format(", ".join( "'{}'".format(value) for value in islice(a_set, max_len) )) if len(a_set) > max_len: From 4f600b72c6aca0102bcd84afd5b90aa17e87ab9d Mon Sep 17 00:00:00 2001 From: Hans Kirchner Date: Tue, 2 Jan 2018 11:58:51 +0100 Subject: [PATCH 5/8] Add test cases for stringify_set Fix stringify_set: Sort the elements before displaying for small sets --- vladiate/test/test_validators.py | 13 ++++++++++++- vladiate/validators.py | 13 ++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/vladiate/test/test_validators.py b/vladiate/test/test_validators.py index df11030..c083e27 100644 --- a/vladiate/test/test_validators.py +++ b/vladiate/test/test_validators.py @@ -4,7 +4,7 @@ from ..validators import ( CastValidator, EmptyValidator, FloatValidator, Ignore, IntValidator, NotEmptyValidator, RangeValidator, RegexValidator, SetValidator, - UniqueValidator, Validator + UniqueValidator, Validator, stringify_set ) from ..exceptions import BadValidatorException, ValidationException @@ -217,3 +217,14 @@ def test_base_class_raises(): def test_all_validators_support_empty_ok(validator_class, args): validator = validator_class(*args, empty_ok=True) validator.validate('') + + +@pytest.mark.parametrize('a_set, max_len, stringified', [ + ({'A', 'B', 'C'}, 4, "{'A', 'B', 'C'}"), + ({'A', 'B', 'C'}, 3, "{'A', 'B', 'C'}"), + ({'A', 'B', 'C'}, 2, "{'A', 'B'} (1 more suppressed)"), + ({'A', 'B', 'C'}, 0, "{} (3 more suppressed)"), + ({}, 5, "{}"), +]) +def test_stringify_set(a_set, max_len, stringified): + assert stringify_set(a_set, max_len) == stringified diff --git a/vladiate/validators.py b/vladiate/validators.py index 5166ec7..162c24c 100644 --- a/vladiate/validators.py +++ b/vladiate/validators.py @@ -212,11 +212,18 @@ def bad(self): pass -def stringify_set(a_set, max_len): - ''' Stringify `max_len` elements of `a_set` and count the remainings ''' +def stringify_set(a_set, max_len, max_sort_size=8192): + ''' Stringify `max_len` elements of `a_set` and count the remainings + + Small sets (len(a_set) <= max_sort_size) are displayed sorted. + Large sets won't be sorted for performance reasons. + This may result in an arbitrary ordering in the returned string. + ''' # Don't convert `a_set` to a list for performance reasons text = "{{{}}}".format(", ".join( - "'{}'".format(value) for value in islice(a_set, max_len) + "'{}'".format(value) for value in islice( + sorted(a_set) if len(a_set) <= max_sort_size else a_set, + max_len) )) if len(a_set) > max_len: text += " ({} more suppressed)".format(len(a_set) - max_len) From bd33377fff8ea42d923c4a00cc533a0519cffa09 Mon Sep 17 00:00:00 2001 From: Hans Kirchner Date: Tue, 2 Jan 2018 12:03:39 +0100 Subject: [PATCH 6/8] Remove previously introduced test case for set validator --- vladiate/test/test_validators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vladiate/test/test_validators.py b/vladiate/test/test_validators.py index c083e27..b6d7714 100644 --- a/vladiate/test/test_validators.py +++ b/vladiate/test/test_validators.py @@ -85,7 +85,6 @@ def test_set_validator_works(field_set, field): ([], 'bar'), (['foo'], 'bar'), (['foo', 'bar'], 'baz'), - ([str(x) for x in range(200)], "notanumber"), ]) def test_set_validator_fails(field_set, field): validator = SetValidator(field_set) From d355defe455e9aa16143b58498264eac1653cde1 Mon Sep 17 00:00:00 2001 From: Hans Kirchner Date: Tue, 2 Jan 2018 13:51:17 +0100 Subject: [PATCH 7/8] Add parameter checks for stringify_set --- vladiate/test/test_validators.py | 8 ++++++++ vladiate/validators.py | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/vladiate/test/test_validators.py b/vladiate/test/test_validators.py index b6d7714..31a5de1 100644 --- a/vladiate/test/test_validators.py +++ b/vladiate/test/test_validators.py @@ -224,6 +224,14 @@ def test_all_validators_support_empty_ok(validator_class, args): ({'A', 'B', 'C'}, 2, "{'A', 'B'} (1 more suppressed)"), ({'A', 'B', 'C'}, 0, "{} (3 more suppressed)"), ({}, 5, "{}"), + ({}, 0, "{}"), ]) def test_stringify_set(a_set, max_len, stringified): assert stringify_set(a_set, max_len) == stringified + + +def test_stringify_set_invalid_params(): + with pytest.raises(ValueError): + stringify_set({}, -1, 10) + with pytest.raises(ValueError): + stringify_set({}, 10, -1) diff --git a/vladiate/validators.py b/vladiate/validators.py index 162c24c..b1b778c 100644 --- a/vladiate/validators.py +++ b/vladiate/validators.py @@ -219,6 +219,11 @@ def stringify_set(a_set, max_len, max_sort_size=8192): Large sets won't be sorted for performance reasons. This may result in an arbitrary ordering in the returned string. ''' + if max_len < 0: + raise ValueError("max_len must be non-negative: {}".format(max_len)) + if max_sort_size < 0: + raise ValueError( + "max_sort_size must be non-negative: {}".format(max_sort_size)) # Don't convert `a_set` to a list for performance reasons text = "{{{}}}".format(", ".join( "'{}'".format(value) for value in islice( From aeab86f7173cab76ae3af46443387a122069f5c8 Mon Sep 17 00:00:00 2001 From: Hans Kirchner Date: Tue, 2 Jan 2018 17:21:44 +0100 Subject: [PATCH 8/8] Rename stringify_set to _stringify_set --- vladiate/test/test_validators.py | 8 ++++---- vladiate/validators.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vladiate/test/test_validators.py b/vladiate/test/test_validators.py index 31a5de1..58e53ef 100644 --- a/vladiate/test/test_validators.py +++ b/vladiate/test/test_validators.py @@ -4,7 +4,7 @@ from ..validators import ( CastValidator, EmptyValidator, FloatValidator, Ignore, IntValidator, NotEmptyValidator, RangeValidator, RegexValidator, SetValidator, - UniqueValidator, Validator, stringify_set + UniqueValidator, Validator, _stringify_set ) from ..exceptions import BadValidatorException, ValidationException @@ -227,11 +227,11 @@ def test_all_validators_support_empty_ok(validator_class, args): ({}, 0, "{}"), ]) def test_stringify_set(a_set, max_len, stringified): - assert stringify_set(a_set, max_len) == stringified + assert _stringify_set(a_set, max_len) == stringified def test_stringify_set_invalid_params(): with pytest.raises(ValueError): - stringify_set({}, -1, 10) + _stringify_set({}, -1, 10) with pytest.raises(ValueError): - stringify_set({}, 10, -1) + _stringify_set({}, 10, -1) diff --git a/vladiate/validators.py b/vladiate/validators.py index b1b778c..7df538c 100644 --- a/vladiate/validators.py +++ b/vladiate/validators.py @@ -72,7 +72,7 @@ def validate(self, field, row={}): self.invalid_set.add(field) raise ValidationException( "'{}' is not in {}".format(field, - stringify_set(self.valid_set, 100))) + _stringify_set(self.valid_set, 100))) @property def bad(self): @@ -212,7 +212,7 @@ def bad(self): pass -def stringify_set(a_set, max_len, max_sort_size=8192): +def _stringify_set(a_set, max_len, max_sort_size=8192): ''' Stringify `max_len` elements of `a_set` and count the remainings Small sets (len(a_set) <= max_sort_size) are displayed sorted.