diff --git a/vladiate/main.py b/vladiate/main.py index 988bd8e..68164c1 100644 --- a/vladiate/main.py +++ b/vladiate/main.py @@ -57,6 +57,14 @@ def parse_args(): type=int, help="attempt to use this number of processes") + # Verbose mode + parser.add_argument( + '-v', '--verbose', + dest='verbose', + action='store_true', + default=False, + help="Print failure exceptions from validators as debug logs") + return parser.parse_args() @@ -148,9 +156,12 @@ def load_vladfile(path): return imported.__doc__, vlads -def _vladiate(vlad): +def _vladiate(args): + vlad = args[0] + verbose = args[1] global result_queue - result_queue.put(vlad(vlad.source, validators=vlad.validators).validate()) + result_queue.put(vlad(vlad.source, validators=vlad.validators, + verbose=verbose).validate()) result_queue = Queue() @@ -200,7 +211,7 @@ def main(): # return code if arguments.processes == 1: for vlad in vlad_classes: - vlad(source=vlad.source).validate() + vlad(source=vlad.source, verbose=arguments.verbose).validate() else: proc_pool = Pool( @@ -208,7 +219,8 @@ def main(): if arguments.processes <= len(vlad_classes) else len(vlad_classes) ) - proc_pool.map(_vladiate, vlad_classes) + proc_pool.map(_vladiate, zip(vlad_classes, + [arguments.verbose] * len(vlad_classes))) try: if not result_queue.get_nowait(): return os.EX_DATAERR diff --git a/vladiate/test/test_main.py b/vladiate/test/test_main.py index 0d6c182..f8fd915 100644 --- a/vladiate/test/test_main.py +++ b/vladiate/test/test_main.py @@ -72,7 +72,7 @@ class TestVlad(Vlad): def validate(self): return validate_result - _vladiate(TestVlad) + _vladiate((TestVlad, True)) assert put.calls == [ call(validate_result) @@ -108,6 +108,7 @@ def test_main_when_get_nowait_raises(monkeypatch): vladfile=stub(), vlads=['Something'], processes=2, + verbose=False, ) ) monkeypatch.setattr( @@ -151,6 +152,7 @@ def test_main_with_multiprocess(monkeypatch, get_nowait, expected): vladfile=stub(), vlads=['Something'], processes=2, + verbose=False, ) ) monkeypatch.setattr( @@ -182,6 +184,7 @@ def test_main_with_vlads_in_args(monkeypatch): vladfile=stub(), vlads=['Something'], processes=1, + verbose=False, ) ) monkeypatch.setattr( @@ -205,6 +208,7 @@ def test_main_no_vlads_in_args(monkeypatch): vladfile=stub(), vlads=[], processes=1, + verbose=False, ) ) monkeypatch.setattr( diff --git a/vladiate/test/test_vlads.py b/vladiate/test/test_vlads.py index 4b077ae..f2c8234 100644 --- a/vladiate/test/test_vlads.py +++ b/vladiate/test/test_vlads.py @@ -6,6 +6,7 @@ EmptyValidator, FloatValidator, NotEmptyValidator, SetValidator, UniqueValidator, ) +from ..exceptions import ValidationException def test_initialize_vlad(): @@ -116,6 +117,31 @@ class TestVlad(Vlad): assert not vlad.validate() assert vlad.validators['Column A'][0].fail_count == 3 assert vlad.validators['Column B'][0].fail_count == 3 + assert len(vlad.failures) == 0 + + +def test_verbose_and_fails_validation(): + source = LocalFile('vladiate/examples/vampires.csv') + + class TestVlad(Vlad): + validators = { + 'Column A': [ + EmptyValidator() + ], + 'Column B': [ + SetValidator(['Vampire']) + ] + } + + vlad = TestVlad(source=source, verbose=True) + assert not vlad.validate() + ve = [ValidationException] + assert list(map(type, vlad.failures['Column A'][0])) == ve + assert list(map(type, vlad.failures['Column A'][1])) == ve + assert list(map(type, vlad.failures['Column A'][2])) == ve + assert list(map(type, vlad.failures['Column B'][0])) == ve + assert list(map(type, vlad.failures['Column B'][1])) == [] + assert list(map(type, vlad.failures['Column B'][2])) == [] def test_gt_99_failures(): diff --git a/vladiate/vlad.py b/vladiate/vlad.py index 0233152..14f674d 100644 --- a/vladiate/vlad.py +++ b/vladiate/vlad.py @@ -9,7 +9,8 @@ class Vlad(object): def __init__(self, source, validators={}, default_validator=EmptyValidator, - delimiter=None, ignore_missing_validators=False): + delimiter=None, ignore_missing_validators=False, + verbose=False): self.logger = logs.logger self.failures = defaultdict(lambda: defaultdict(list)) self.missing_validators = None @@ -19,6 +20,7 @@ def __init__(self, source, validators={}, default_validator=EmptyValidator, self.delimiter = delimiter or getattr(self, 'delimiter', ',') self.line_count = 0 self.ignore_missing_validators = ignore_missing_validators + self.verbose = verbose self.validators.update({ field: [default_validator()] @@ -99,6 +101,8 @@ def validate(self): self._log_missing_fields() return False + any_failures = False + for line, row in enumerate(reader): self.line_count += 1 for field_name, field in row.items(): @@ -107,12 +111,15 @@ def validate(self): try: validator.validate(field, row=row) except ValidationException as e: - self.failures[field_name][line].append(e) + any_failures = True + if self.verbose: + self.failures[field_name][line].append(e) validator.fail_count += 1 - if self.failures: + if any_failures: self.logger.info("\033[0;31m" + "Failed :(" + "\033[0m") - self._log_debug_failures() + if self.verbose: + self._log_debug_failures() self._log_validator_failures() return False else: