merge fix

Makefile

@@ -1,23 +1,28 @@
 # Make content for developer
 help:
-	@echo "setup			Instal requirements"
+	@echo "dev			Instal requirements"
 	@echo "test			Run suit test"
 	@echo "coverage		Run Coverage"
 	@echo "clean			Remove trash files"
 	@echo "send_package		Send Package to Pypi"
 	@echo "p3		Create docker with py3"
 	@echo "p2		Create docker with py2"
+	@echo "update		Update all local packages"
 
 
-setup:
-	pip install -r example/requirements-dev.txt
+dev:
+	pip install -r requirements.txt
+
+update:
+	pip freeze --local | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 pip install -U
+	pip freeze > requirements.txt
 
 test:
-	PYTHONPATH=`pwd` py.test -x
+	PYTHONPATH=`pwd` py.test -x -s
 
 coverage:
 	rm -rf htmlcov
-	py.test --cov=data_importer --cov-report html
+	PYTHONPATH=`pwd` py.test --cov=data_importer --cov-report html
 
 send_package:
 	python setup.py register sdist upload

data_importer/core/base.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 from . import default_settings
 DATA_IMPORTER_EXCEL_DECODER = default_settings.DATA_IMPORTER_EXCEL_DECODER
 DATA_IMPORTER_DECODER = default_settings.DATA_IMPORTER_DECODER

data_importer/importers/base.py

@@ -1,18 +1,17 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
+from __future__ import unicode_literals
 import os
 import re
 from django.db import transaction
 from django.db.models.fields import FieldDoesNotExist
 from django.core.exceptions import ValidationError
-from django.utils.encoding import force_unicode
+from django.utils.encoding import force_text
 from data_importer.core.descriptor import ReadDescriptor
 from data_importer.core.exceptions import StopImporter
 from data_importer.core.base import objclass2dict
 from data_importer.core.base import DATA_IMPORTER_EXCEL_DECODER
 from data_importer.core.base import DATA_IMPORTER_DECODER
 from collections import OrderedDict
+from io import IOBase
 
 
 ALPHABETIC = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
@@ -60,13 +59,13 @@ def to_unicode(bytestr):
         """
         Receive string bytestr and try to return a utf-8 string.
         """
-        if not isinstance(bytestr, str) and not isinstance(bytestr, unicode):
+        if not isinstance(bytestr, str):
             return bytestr
 
         try:
             decoded = bytestr.decode(DATA_IMPORTER_EXCEL_DECODER)  # default by excel csv
-        except UnicodeEncodeError:
-            decoded = force_unicode(bytestr, DATA_IMPORTER_DECODER)
+        except (UnicodeEncodeError, AttributeError):
+            decoded = force_text(bytestr, DATA_IMPORTER_DECODER)
 
         return decoded
 
@@ -76,14 +75,12 @@ def source(self):
         return self._source
 
     @source.setter
-    def source(self, source):
+    def source(self, source=None, encoding="ISO-8859-1"):
         """Open source to reader"""
-        if isinstance(source, file):
+        if isinstance(source, IOBase):
             self._source = source
         elif isinstance(source, str) and os.path.exists(source) and source.endswith('csv'):
-            self._source = open(source, 'rb')
-        elif isinstance(source, unicode) and os.path.exists(source) and source.endswith('csv'):
-            self._source = open(source, 'rb')
+            self._source = open(source, 'r', encoding=encoding)
         elif isinstance(source, list):
             self._source = source
         elif hasattr(source, 'file_upload'):  # for FileHistory instances
@@ -177,7 +174,7 @@ def clean_field(self, field_name, value):
                 pass  # do nothing if not find this field in model
             except ValidationError as msg:
                 default_msg = msg.messages[0].replace('This field', '')
-                new_msg = u'Field (%s) %s' % (field.name, default_msg)
+                new_msg = 'Field (%s) %s' % (field.name, default_msg)
                 raise ValidationError(new_msg)
 
         clean_function = getattr(self, 'clean_%s' % field_name, False)
@@ -246,14 +243,17 @@ def get_error_message(self, error, row=None, error_type=None):
         messages = ''
 
         if not error_type:
-            error_type = u"%s" % type(error).__name__
+            error_type = "%s" % type(error).__name__
 
         if hasattr(error, 'message') and error.message:
-            messages = u'%s' % error.message
+            messages = '%s' % error.message
 
         if hasattr(error, 'messages') and not messages:
             if error.messages:
-                messages = u','.join(error.messages)
+                messages = ','.join(error.messages)
+
+        if not messages:
+            messages = str(error)
 
         messages = re.sub('\'', '', messages)
         error_type = re.sub('\'', '', error_type)
@@ -336,6 +336,7 @@ def _read_file(self):
             reader = self._reader.read()
         else:
             reader = self._reader
+
         self.original_fields = self.fields
         if isinstance(self.fields, dict):
             self.fields = self.get_dict_fields(self.fields)
@@ -396,7 +397,7 @@ def convert_letter_to_number(letter):
 
     @staticmethod
     def convert_list_number_to_decimal_integer(list_number):
-        list_number_reversed = list(reversed(list_number))
+        list_number_reversed = list(reversed(list(list_number)))
         final_number = 0
         for number, exp in zip(list_number_reversed, range(len(list_number_reversed))):
             final_number += (number + DELAY) * (FACTOR ** exp)

data_importer/importers/csv_importer.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 from data_importer.importers.base import BaseImporter
 from data_importer.readers import CSVReader
 

data_importer/importers/generic.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from data_importer.readers.xls_reader import XLSReader
 from data_importer.readers.xlsx_reader import XLSXReader
 from data_importer.readers.csv_reader import CSVReader

data_importer/importers/xls_importer.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 from .base import BaseImporter
 from data_importer.readers import XLSReader
 

data_importer/importers/xml_importer.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 from data_importer.importers.base import BaseImporter
 from data_importer.readers.xml_reader import XMLReader
 

data_importer/readers/xls_reader.py

@@ -49,6 +49,6 @@ def convert_value(item, workbook):
         return item.value
 
     def read(self):
-        for i in xrange(0, self.worksheet.nrows):
+        for i in range(0, self.worksheet.nrows):
             values = [self.convert_value(cell, self.workbook) for cell in self.worksheet.row(i)]
             yield values

data_importer/south_migrations/0001_initial.py

@@ -9,28 +9,28 @@ class Migration(SchemaMigration):
 
     def forwards(self, orm):
         # Adding model 'FileHistory'
-        db.create_table(u'data_importer_filehistory', (
-            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+        db.create_table('data_importer_filehistory', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
             ('created_at', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
             ('updated_at', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
             ('active', self.gf('django.db.models.fields.BooleanField')(default=True, db_index=True)),
             ('content', self.gf('django.db.models.fields.files.FileField')(max_length=100)),
         ))
-        db.send_create_signal(u'data_importer', ['FileHistory'])
+        db.send_create_signal('data_importer', ['FileHistory'])
 
 
     def backwards(self, orm):
         # Deleting model 'FileHistory'
-        db.delete_table(u'data_importer_filehistory')
+        db.delete_table('data_importer_filehistory')
 
 
     models = {
-        u'data_importer.filehistory': {
+        'data_importer.filehistory': {
             'Meta': {'object_name': 'FileHistory'},
             'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
             'content': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
             'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
         }
     }

data_importer/south_migrations/0002_auto__add_field_filehistory_owner.py

@@ -9,37 +9,37 @@ class Migration(SchemaMigration):
 
     def forwards(self, orm):
         # Adding field 'FileHistory.owner'
-        db.add_column(u'data_importer_filehistory', 'owner',
+        db.add_column('data_importer_filehistory', 'owner',
                       self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], null=True),
                       keep_default=False)
 
 
     def backwards(self, orm):
         # Deleting field 'FileHistory.owner'
-        db.delete_column(u'data_importer_filehistory', 'owner_id')
+        db.delete_column('data_importer_filehistory', 'owner_id')
 
 
     models = {
-        u'auth.group': {
+        'auth.group': {
             'Meta': {'object_name': 'Group'},
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
             'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
         },
-        u'auth.permission': {
-            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
             'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
             'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
         },
-        u'auth.user': {
+        'auth.user': {
             'Meta': {'object_name': 'User'},
             'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
             'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
             'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
             'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
             'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
@@ -49,19 +49,19 @@ def backwards(self, orm):
             'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
             'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
         },
-        u'contenttypes.contenttype': {
+        'contenttypes.contenttype': {
             'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
             'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
             'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
         },
-        u'data_importer.filehistory': {
+        'data_importer.filehistory': {
             'Meta': {'object_name': 'FileHistory'},
             'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
             'content': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
             'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True'}),
             'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
         }

data_importer/south_migrations/0003_auto__add_field_filehistory_is_task__add_field_filehistory_status.py

@@ -9,45 +9,45 @@ class Migration(SchemaMigration):
 
     def forwards(self, orm):
         # Adding field 'FileHistory.is_task'
-        db.add_column(u'data_importer_filehistory', 'is_task',
+        db.add_column('data_importer_filehistory', 'is_task',
                       self.gf('django.db.models.fields.BooleanField')(default=False),
                       keep_default=False)
 
         # Adding field 'FileHistory.status'
-        db.add_column(u'data_importer_filehistory', 'status',
+        db.add_column('data_importer_filehistory', 'status',
                       self.gf('django.db.models.fields.IntegerField')(default=1),
                       keep_default=False)
 
 
     def backwards(self, orm):
         # Deleting field 'FileHistory.is_task'
-        db.delete_column(u'data_importer_filehistory', 'is_task')
+        db.delete_column('data_importer_filehistory', 'is_task')
 
         # Deleting field 'FileHistory.status'
-        db.delete_column(u'data_importer_filehistory', 'status')
+        db.delete_column('data_importer_filehistory', 'status')
 
 
     models = {
-        u'auth.group': {
+        'auth.group': {
             'Meta': {'object_name': 'Group'},
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
             'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
         },
-        u'auth.permission': {
-            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
             'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
             'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
         },
-        u'auth.user': {
+        'auth.user': {
             'Meta': {'object_name': 'User'},
             'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
             'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
             'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
             'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
             'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
@@ -57,19 +57,19 @@ def backwards(self, orm):
             'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
             'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
         },
-        u'contenttypes.contenttype': {
+        'contenttypes.contenttype': {
             'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
             'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
             'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
         },
-        u'data_importer.filehistory': {
+        'data_importer.filehistory': {
             'Meta': {'object_name': 'FileHistory'},
             'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'db_index': 'True'}),
             'content': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
             'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'is_task': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
             'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True'}),
             'status': ('django.db.models.fields.IntegerField', [], {'default': '1'}),