fix #1: skip empty rows in import data

pyexcel_io/base.py

@@ -271,3 +271,7 @@ def from_query_sets(column_names, query_sets):
             new_array.append(value)
         array.append(new_array)
     return array
+
+
+def is_empty_array(array):
+    return len([x for x in array if x != '']) == 0

pyexcel_io/constants.py

@@ -19,6 +19,7 @@
 MESSAGE_CANNOT_WRITE_FILE_TYPE_FORMATTER = "Cannot write content of file type %s to file %s"
 MESSAGE_CANNOT_READ_FILE_TYPE_FORMATTER = "Cannot read content of file type %s from file %s"
 MESSAGE_LOADING_FORMATTER = "The plugin for file type %s is not installed. Please install %s"
+MESSAGE_EMPTY_ARRAY = "One empty row is found"
 
 FILE_FORMAT_CSV = 'csv'
 FILE_FORMAT_TSV = 'tsv'

pyexcel_io/djangobook.py

@@ -8,12 +8,14 @@
     :license: New BSD License, see LICENSE for more details
 """
 from ._compact import OrderedDict
+from .constants import MESSAGE_EMPTY_ARRAY
 from .base import (
     BookReaderBase,
     SheetReaderBase,
     BookWriter,
     SheetWriter,
-    from_query_sets
+    from_query_sets,
+    is_empty_array
 )
 
 
@@ -72,9 +74,12 @@ def __init__(self, model, batch_size=None):
         self.objs = []
 
     def write_row(self, array):
-        self.objs.append(self.mymodel(**dict(
-            zip(self.column_names, self.initializer(array))
-        )))
+        if is_empty_array(array):
+            print(MESSAGE_EMPTY_ARRAY)
+        else:
+            self.objs.append(self.mymodel(**dict(
+                zip(self.column_names, self.initializer(array))
+            )))
 
     def close(self):
         self.mymodel.objects.bulk_create(self.objs, batch_size=self.batch_size)

pyexcel_io/sqlbook.py

@@ -8,13 +8,17 @@
     :license: New BSD License, see LICENSE for more details
 """
 from ._compact import OrderedDict
-from .constants import MESSAGE_INVALID_PARAMETERS
+from .constants import (
+    MESSAGE_INVALID_PARAMETERS,
+    MESSAGE_EMPTY_ARRAY
+)
 from .base import (
     BookReaderBase,
     SheetReaderBase,
     BookWriter,
     SheetWriter,
-    from_query_sets
+    from_query_sets,
+    is_empty_array
 )
 
 
@@ -47,11 +51,11 @@ def __init__(self, session=None, tables=None):
         for table in tables:
             sqltablereader = SQLTableReader(session, table)
             self.my_sheets[sqltablereader.name] = sqltablereader.to_array()
-            
+
     def sheets(self):
         return self.my_sheets
 
-        
+
 class SQLTableWriter(SheetWriter):
     """Write to a table
     """
@@ -71,6 +75,12 @@ def __init__(self, session, table_params):
             self.mapdict = None
 
     def write_row(self, array):
+        if is_empty_array(array):
+            print(MESSAGE_EMPTY_ARRAY)
+        else:
+            self._write_row(array)
+
+    def _write_row(self, array):
         row = dict(zip(self.column_names, array))
         if self.initializer:
             o = self.initializer(row)

tests/test_base.py

@@ -1,11 +1,19 @@
 from pyexcel_io import (
     SheetReaderBase, SheetReader, BookReader,
     SheetWriter, BookWriter, NamedContent,
-    BookReaderBase, SheetWriterBase
+    BookReaderBase, SheetWriterBase,
 )
+from pyexcel_io.base import is_empty_array
 from nose.tools import raises
 
 
+def test_is_empty_array():
+    a=["", "", "", ""]
+    assert is_empty_array(a) == True
+    b=[1, "", "", ""]
+    assert is_empty_array(b) == False
+
+
 class ArrayReader(SheetReader):
     @property
     def name(self):
@@ -230,4 +238,4 @@ def test_book_writer(self):
         writer = DictWriter("afile")
         writer.write(content)
         writer.close()
-        assert writer.dict == content
+        assert writer.dict == content

tests/test_django_book.py

@@ -61,6 +61,20 @@ def test_sheet_save_to_django_model(self):
         writer.close()
         assert model.objects.objs == self.result
 
+    def test_sheet_save_to_django_model_with_empty_array(self):
+        model=FakeDjangoModel()
+        data  = [
+            ["X", "Y", "Z"],
+            ['', '', ''],
+            [1, 2, 3],
+            [4, 5, 6]
+        ]
+        writer = DjangoModelWriter([model, data[0], None, None])
+        writer.write_array(data[1:])
+        writer.close()
+        print self.result
+        assert model.objects.objs == self.result
+
     def test_sheet_save_to_django_model_3(self):
         model=FakeDjangoModel()
         def wrapper(row):

tests/test_sql_book.py

@@ -121,6 +121,23 @@ def test_one_table(self):
         assert results == self.results
         mysession.close()
 
+    def test_one_table_with_empty_rows(self):
+        mysession = Session()
+        data = [
+            ['birth', 'id', 'name', 'weight'],
+            ['', '', ''],
+            [datetime.date(2014, 11, 11), 0, 'Adam', 11.25],
+            [datetime.date(2014, 11, 12), 1, 'Smith', 12.25]
+        ]
+        writer = SQLTableWriter(mysession,
+                                [Pyexcel,data[0], None, None])
+        writer.write_array(data[1:])
+        writer.close()
+        query_sets=mysession.query(Pyexcel).all()
+        results = from_query_sets(data[0], query_sets)
+        assert results == self.results
+        mysession.close()
+
     def test_one_table_using_mapdict_as_array(self):
         mysession = Session()
         self.data = [