From 54e99904d1dcf084b4c9b3779b6b01d6be211804 Mon Sep 17 00:00:00 2001 From: Ilja Heitlager Date: Wed, 3 May 2017 20:46:10 +0200 Subject: [PATCH] Small tweaks (#43) * adding additional tweak to phone cleaner * adding maxLength * update documentation --- docs/ref/contrib.txt | 1 + docs/ref/exceptions.txt | 13 +------------ src/data_migrator/contrib/dutch.py | 26 ++++++++++++++++++++------ src/data_migrator/contrib/read.py | 13 +++++++++---- src/data_migrator/emitters/base.py | 4 ++-- src/data_migrator/exceptions.py | 2 +- src/data_migrator/models/base.py | 2 +- src/data_migrator/models/fields.py | 4 +++- tests/test_contrib.py | 1 + tests/test_fields.py | 8 +++++++- 10 files changed, 46 insertions(+), 28 deletions(-) diff --git a/docs/ref/contrib.txt b/docs/ref/contrib.txt index b9c9505e..8e20e577 100644 --- a/docs/ref/contrib.txt +++ b/docs/ref/contrib.txt @@ -8,6 +8,7 @@ Contrib reference .. currentmodule: data_migrator.contrib .. automodule:: data_migrator.contrib + :members: contrib.read ============= diff --git a/docs/ref/exceptions.txt b/docs/ref/exceptions.txt index fe5a04cd..34656464 100644 --- a/docs/ref/exceptions.txt +++ b/docs/ref/exceptions.txt @@ -5,15 +5,4 @@ Exceptions reference .. currentmodule:: data_migrator.exceptions .. automodule:: data_migrator.exceptions - -.. autoexception:: data_migrator.exceptions.InternalException - -.. autoexception:: data_migrator.exceptions.DefinitionException - -.. autoexception:: data_migrator.exceptions.ValidationException - -.. autoexception:: data_migrator.exceptions.DataException - -.. autoexception:: data_migrator.exceptions.NonUniqueDataException - -.. autoexception:: data_migrator.exceptions.NullDataException + :members: diff --git a/src/data_migrator/contrib/dutch.py b/src/data_migrator/contrib/dutch.py index ff6748d6..edce3fab 100644 --- a/src/data_migrator/contrib/dutch.py +++ b/src/data_migrator/contrib/dutch.py @@ -17,10 +17,18 @@ def clean_phone(v): clean_phone clean phone numbers, replaces all characters and spaces adds dutch country code (+31) if no country code is provide - >>> [clean_phone(x) for x in ['00 31 6 - 20 20 20 20','06 20 20 20 20', - '020 -123 345 6','+440.203.020.23','+440a203a020a23'] - ['+31620202020','+31620202020','+31201233456','+4402030203', - '+4402030203'] + >>> clean_phone('00 31 6 - 20 20 20 20') + '+31620202020' + >>> clean_phone('06 20 20 20 20') + '+31620202020' + >>> clean_phone('020 -123 345 6') + '+31201233456' + >>> clean_phone('+440.203.020.23') + '+4402030203' + >>> clean_phone('+440a203a020a23') + '+4402030203' + >>> clean_phone('31 (6) - 20 20 20 20') + '+31620202020' Args: v (str): value to clean @@ -32,6 +40,8 @@ def clean_phone(v): v = _PHONE_CHARS.sub('', v) v = _INTERNATIONAL_ZERO_START.sub('+', v) v = _MUNICIPALY_ZERO_START.sub('+31', v) + if v.startswith('316'): + v = '+' + v return v @@ -41,8 +51,12 @@ def clean_phone(v): def clean_zip_code(v): '''Cleans a dutch zipcode - >>> [clean_zip_code(x) for x in ['1234 aa', '1234AB', '1234 Ba']] - ['1234AA', '1234AB', '1234BA'] + >>> clean_zip_code('1234 aa') + '1234AA' + >>> clean_zip_code('1234AB') + '1234AB' + >>> clean_zip_code('1234 Ba') + '1234BA' Args: v (str): zipcode to clean diff --git a/src/data_migrator/contrib/read.py b/src/data_migrator/contrib/read.py index f8553ea3..55aaabfe 100644 --- a/src/data_migrator/contrib/read.py +++ b/src/data_migrator/contrib/read.py @@ -10,8 +10,7 @@ def read_map_from_csv(key=0, value=1, f=None, delimiter="\t", header=True, as_list=False, unique=False): - ''' - Generates a map from a csv and adds some validation and list parsing. A + '''Generates a map from a csv and adds some validation and list parsing. A function that returns a map for MappingField to use as input in its MappingField.data_map. @@ -34,12 +33,18 @@ def read_map_from_csv(key=0, value=1, f=None, delimiter="\t", header=True, values for ``key`` as a list. Default is ``False``. unique (boolean): If ``True``, *data-migrator* will treat add all non unique values for ``key`` as a violation and raise a - ``DataException``. Default is ``False``. + :exc:`~.NonUniqueDataException`. Default is ``False``. header (boolean): If ``True``, *data-migrator* will treat row as a header column. Default is ``True`` Returns: map: a key, value map from the csv + + Raises: + :exc:`~.DefinitionException`: if key, value does not match or as_list + not set. + :exc:`~.NonUniqueDataException`: if data is not unique on the key. + ''' data_map = {} if not f: @@ -74,7 +79,7 @@ def read_map_from_csv(key=0, value=1, f=None, delimiter="\t", header=True, elif as_list: data_map[l[ki]] += v else: - raise DefinitionException('line %d - unqiue contraint failed, expecting as_list for %s:%s' % (i, l[ki], data_map[l[ki]])) + raise DefinitionException('line %d - unique contraint failed, expecting as_list for %s:%s' % (i, l[ki], data_map[l[ki]])) else: data_map[l[ki]] = v if f != sys.stdin: diff --git a/src/data_migrator/emitters/base.py b/src/data_migrator/emitters/base.py index 0ed3e271..5b1f84b5 100644 --- a/src/data_migrator/emitters/base.py +++ b/src/data_migrator/emitters/base.py @@ -3,7 +3,7 @@ class BaseEmitter(object): - '''Base for emitters of the data_migrator. + '''Base for emitters of the *data-migrator*. Attributes: manager (BaseManager): reference to the manager that is calling this @@ -11,7 +11,7 @@ class BaseEmitter(object): model_class (Model): reference to the model linked to the class extension (str): file extension for output file of this emitter - note: ``model_class`` and ``manager`` are linked together + note: :attr:`~.model_class` and :attr:`~.manager` are linked together ''' def __init__(self, extension=None, manager=None): diff --git a/src/data_migrator/exceptions.py b/src/data_migrator/exceptions.py index 862e53f4..83dc02ff 100644 --- a/src/data_migrator/exceptions.py +++ b/src/data_migrator/exceptions.py @@ -25,7 +25,7 @@ class DataException(Exception): class NonUniqueDataException(Exception): - """Non unique data exception""" + """Non unique data based on key found""" pass diff --git a/src/data_migrator/models/base.py b/src/data_migrator/models/base.py index 1a0eb89e..bdc4aae0 100644 --- a/src/data_migrator/models/base.py +++ b/src/data_migrator/models/base.py @@ -123,7 +123,7 @@ def update(self, **kwargs): self, so that methods can be chained Raises: - :class:`~.DataException`: raised if trying to set non defined field + :exc:`~.DataException`: raised if trying to set non defined field and strict model. ''' _meta = self.__class__._meta diff --git a/src/data_migrator/models/fields.py b/src/data_migrator/models/fields.py index e088ebf8..335477fa 100644 --- a/src/data_migrator/models/fields.py +++ b/src/data_migrator/models/fields.py @@ -37,7 +37,7 @@ def __init__(self, # key indicated key field self.key = key # fixed position in the row to read - self.max_length = max_length + self.max_length = max_length if isinstance(max_length, int) else None # name of this field (will be set in Model class construction) self.name = name # input string that defines null -> None @@ -115,6 +115,8 @@ def json_schema(self): t = {'type': t} if self.key: t['key'] = True + if self.max_length: + t['maxLength'] = self.max_length return {self.name: t} def _value(self, v): # pylint: disable=R0201 diff --git a/tests/test_contrib.py b/tests/test_contrib.py index 13742fcd..40622715 100644 --- a/tests/test_contrib.py +++ b/tests/test_contrib.py @@ -17,6 +17,7 @@ def test_phone(self): ('020 -123 345 6', '+31201233456'), ('+440.203.020.23', '+44020302023'), ('+440 ada 203.020 // 23', '+44020302023'), + ('31 (6) - 20 20 20 20', '+31620202020'), ] for i, o in l: self.assertEqual(o, clean_phone(i)) diff --git a/tests/test_fields.py b/tests/test_fields.py index 1efea274..c8b1f519 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -65,10 +65,16 @@ def test_exception_int(self): f = models.IntField(pos=0) self.assertRaises(ValueError, f.scan, row=["BLA", "20"]) + def test_max_length_notset(self): + f = models.StringField(pos=0, max_length='something wrong') + self.assertEqual(f.emit("blablabla"), "blablabla") + self.assertFalse(f.max_length, None) + def test_string_length(self): '''build in string trimming''' - f = models.StringField(pos=0, max_length=3) + f = models.StringField(pos=0, max_length=3, name='f') self.assertEqual(f.emit("blablabla"), "bla") + self.assertEqual(f.json_schema(), {'f': {'type': 'string', 'maxLength': 3}}) def test_null_string(self): '''dedicated null string fields'''