Small tweaks (#43)

* adding additional tweak to phone cleaner

* adding maxLength

* update documentation

docs/ref/contrib.txt

@@ -8,6 +8,7 @@ Contrib reference
 .. currentmodule: data_migrator.contrib
 
 .. automodule:: data_migrator.contrib
+   :members:
 
 contrib.read
 =============

docs/ref/exceptions.txt

@@ -5,15 +5,4 @@ Exceptions reference
 .. currentmodule:: data_migrator.exceptions
 
 .. automodule:: data_migrator.exceptions
-
-.. autoexception::  data_migrator.exceptions.InternalException
-
-.. autoexception::  data_migrator.exceptions.DefinitionException
-
-.. autoexception::  data_migrator.exceptions.ValidationException
-
-.. autoexception::  data_migrator.exceptions.DataException
-
-.. autoexception::  data_migrator.exceptions.NonUniqueDataException
-
-.. autoexception::  data_migrator.exceptions.NullDataException
+   :members:

src/data_migrator/contrib/dutch.py

@@ -17,10 +17,18 @@ def clean_phone(v):
     clean_phone clean phone numbers, replaces all characters and spaces
     adds dutch country code (+31) if no country code is provide
 
-        >>> [clean_phone(x) for x in ['00 31 6 - 20 20 20 20','06 20 20 20 20',
-        '020 -123 345 6','+440.203.020.23','+440a203a020a23']
-        ['+31620202020','+31620202020','+31201233456','+4402030203',
-        '+4402030203']
+        >>> clean_phone('00 31 6 - 20 20 20 20')
+        '+31620202020'
+        >>> clean_phone('06 20 20 20 20')
+        '+31620202020'
+        >>> clean_phone('020 -123 345 6')
+        '+31201233456'
+        >>> clean_phone('+440.203.020.23')
+        '+4402030203'
+        >>> clean_phone('+440a203a020a23')
+        '+4402030203'
+        >>> clean_phone('31 (6) - 20 20 20 20')
+        '+31620202020'
 
     Args:
         v (str): value to clean
@@ -32,6 +40,8 @@ def clean_phone(v):
     v = _PHONE_CHARS.sub('', v)
     v = _INTERNATIONAL_ZERO_START.sub('+', v)
     v = _MUNICIPALY_ZERO_START.sub('+31', v)
+    if v.startswith('316'):
+        v = '+' + v
     return v
 
 
@@ -41,8 +51,12 @@ def clean_phone(v):
 def clean_zip_code(v):
     '''Cleans a dutch zipcode
 
-        >>> [clean_zip_code(x) for x in ['1234 aa', '1234AB', '1234   Ba']]
-        ['1234AA', '1234AB', '1234BA']
+        >>> clean_zip_code('1234 aa')
+        '1234AA'
+        >>> clean_zip_code('1234AB')
+        '1234AB'
+        >>> clean_zip_code('1234   Ba')
+        '1234BA'
 
     Args:
         v (str): zipcode to clean

src/data_migrator/contrib/read.py

@@ -10,8 +10,7 @@
 
 def read_map_from_csv(key=0, value=1, f=None, delimiter="\t", header=True,
                       as_list=False, unique=False):
-    '''
-    Generates a map from a csv and adds some validation and list parsing. A
+    '''Generates a map from a csv and adds some validation and list parsing. A
     function that returns a map for MappingField to use as input in its
     MappingField.data_map.
 
@@ -34,12 +33,18 @@ def read_map_from_csv(key=0, value=1, f=None, delimiter="\t", header=True,
             values for ``key`` as a list. Default is ``False``.
         unique (boolean): If ``True``, *data-migrator* will treat add all non
             unique values for ``key`` as a violation and raise a
-            ``DataException``. Default is ``False``.
+            :exc:`~.NonUniqueDataException`. Default is ``False``.
         header (boolean): If ``True``, *data-migrator* will treat row as a
             header column. Default is ``True``
 
     Returns:
         map: a key, value map from the csv
+
+    Raises:
+        :exc:`~.DefinitionException`: if key, value does not match or as_list
+            not set.
+        :exc:`~.NonUniqueDataException`: if data is not unique on the key.
+
     '''
     data_map = {}
     if not f:
@@ -74,7 +79,7 @@ def read_map_from_csv(key=0, value=1, f=None, delimiter="\t", header=True,
             elif as_list:
                 data_map[l[ki]] += v
             else:
-                raise DefinitionException('line %d - unqiue contraint failed, expecting as_list for %s:%s' % (i, l[ki], data_map[l[ki]]))
+                raise DefinitionException('line %d - unique contraint failed, expecting as_list for %s:%s' % (i, l[ki], data_map[l[ki]]))
         else:
             data_map[l[ki]] = v
     if f != sys.stdin:

src/data_migrator/emitters/base.py

@@ -3,15 +3,15 @@
 
 
 class BaseEmitter(object):
-    '''Base for emitters of the data_migrator.
+    '''Base for emitters of the *data-migrator*.
 
     Attributes:
         manager (BaseManager): reference to the manager that is calling this
             emitter to export objects from that manager
         model_class (Model): reference to the model linked to the class
         extension (str): file extension for output file of this emitter
 
-    note: ``model_class`` and ``manager`` are linked together
+    note: :attr:`~.model_class` and :attr:`~.manager` are linked together
     '''
 
     def __init__(self, extension=None, manager=None):

src/data_migrator/exceptions.py

@@ -25,7 +25,7 @@ class DataException(Exception):
 
 
 class NonUniqueDataException(Exception):
-    """Non unique data exception"""
+    """Non unique data based on key found"""
     pass
 
 

src/data_migrator/models/base.py

@@ -123,7 +123,7 @@ def update(self, **kwargs):
             self, so that methods can be chained
 
         Raises:
-            :class:`~.DataException`: raised if trying to set non defined field
+            :exc:`~.DataException`: raised if trying to set non defined field
                 and strict model.
         '''
         _meta = self.__class__._meta

src/data_migrator/models/fields.py

@@ -37,7 +37,7 @@ def __init__(self,
         # key indicated key field
         self.key = key
         # fixed position in the row to read
-        self.max_length = max_length
+        self.max_length = max_length if isinstance(max_length, int) else None
         # name of this field (will be set in Model class construction)
         self.name = name
         # input string that defines null -> None
@@ -115,6 +115,8 @@ def json_schema(self):
         t = {'type': t}
         if self.key:
             t['key'] = True
+        if self.max_length:
+            t['maxLength'] = self.max_length
         return {self.name: t}
 
     def _value(self, v):  # pylint: disable=R0201

tests/test_contrib.py

@@ -17,6 +17,7 @@ def test_phone(self):
             ('020 -123 345 6', '+31201233456'),
             ('+440.203.020.23', '+44020302023'),
             ('+440 ada 203.020 // 23', '+44020302023'),
+            ('31 (6) - 20 20 20 20', '+31620202020'),
         ]
         for i, o in l:
             self.assertEqual(o, clean_phone(i))

tests/test_fields.py

@@ -65,10 +65,16 @@ def test_exception_int(self):
         f = models.IntField(pos=0)
         self.assertRaises(ValueError, f.scan, row=["BLA", "20"])
 
+    def test_max_length_notset(self):
+        f = models.StringField(pos=0, max_length='something wrong')
+        self.assertEqual(f.emit("blablabla"), "blablabla")
+        self.assertFalse(f.max_length, None)
+
     def test_string_length(self):
         '''build in string trimming'''
-        f = models.StringField(pos=0, max_length=3)
+        f = models.StringField(pos=0, max_length=3, name='f')
         self.assertEqual(f.emit("blablabla"), "bla")
+        self.assertEqual(f.json_schema(), {'f': {'type': 'string', 'maxLength': 3}})
 
     def test_null_string(self):
         '''dedicated null string fields'''