Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Fixed tests.

  • Loading branch information...
commit 8714b1a8d7ba5b169a573868e34ceb46cea4c746 1 parent 607c951
@chronossc authored
View
16 data_importer/base.py
@@ -31,7 +31,7 @@ class BaseImporter(object):
reader = None
loaded = False
errors = SortedDict() # {lineNum:list(set([error1,error2])),...}
-
+
def __init__(self,import_file,reader=None,reader_kwargs={}):
self._validation_results = SortedDict()
self.set_logger()
@@ -41,7 +41,7 @@ def __init__(self,import_file,reader=None,reader_kwargs={}):
if settings.DEBUG:
self.logger.setLevel(logging.DEBUG)
else:self.logger.setLevel(logging.INFO)
-
+
def _validate_class(self):
"""
Somethings here is important, as sample we need fields :)
@@ -104,8 +104,12 @@ def set_logger(self):
logging.basicConfig(format=u'%(asctime)s :: %(levelname)s :: %(message)s')
self.logger = logging.getLogger('%s_importer' % self.__class__.__name__)
+ # remove handlers that can come with logger
+ for h in self.logger.handlers:
+ self.logger.removeHandler(h)
self.logger.propagate = False
+ # set defined handlers as defined in self.get_logger_handlers
for h,hargs,hkwargs in handlers:
self.logger.addHandler(h(*hargs,**hkwargs))
@@ -139,12 +143,12 @@ def is_valid(self):
def _clean_all(self):
self.errors = SortedDict()
- for i,row in enumerate(self.reader,2):
+ for i,row in enumerate(self.reader,1):
self._clean(i,row)
def _iter_clean_all(self):
self.errors = SortedDict()
- for i,row in enumerate(self.reader,2):
+ for i,row in enumerate(self.reader,1):
yield i,self._clean(i,row)
def _clean(self,i,_row):
@@ -163,7 +167,7 @@ def _clean(self,i,_row):
DON'T CALL THIS METHOD DIRECTLY!!
This method should be called by self._clean_all
-
+
"""
if i in self._validation_results:
@@ -258,7 +262,7 @@ def save(self,i,row):
if row:
self.logger.info(_(u"Line %s saved successfully"),i)
return row
-
+
def post_save_all(self):
"""
# TODO: use signals
View
3  data_importer/readers/csv_reader.py
@@ -13,5 +13,6 @@ def set_reader(self):
def get_items(self):
for row in self._reader:
- #if not row: continue
+ print row
+ if not row: continue # invalid lines are ignored
yield self.get_item(row)
View
15 data_importer/readers/xls_reader.py
@@ -38,7 +38,7 @@ def headers(self):
return self._headers
def get_value(self,item,**kwargs):
- """
+ """
Handle different value types for XLS. Item is a cell object.
"""
@@ -46,11 +46,18 @@ def get_value(self,item,**kwargs):
if item.ctype == XL_CELL_DATE:
return datetime.datetime(*xlrd.xldate_as_tuple(item.value, self._workbook.datemode))
+ if item.ctype == XL_CELL_NUMBER:
+ if item.value % 1 == 0: # integers
+ return str(int(item.value))
+ else:
+ return str(item.value)
+
return item.value
def get_items(self):
for r in range(1,self.nrows):
values = [self.get_value(self._reader.cell(r,c)) for c in range(self.ncols)]
+ if not all(values): continue # empty lines are ignored
yield self.get_item(values)
class XLSXReader(XLSReader):
@@ -61,7 +68,7 @@ def set_reader(self):
self._reader = self._workbook.worksheets[self._workbook.get_sheet_names().index(self._sheet_name)]
else:
self._reader = self._workbook.worksheets[0]
-
+
@property
def headers(self):
if not self._headers:
@@ -69,18 +76,18 @@ def headers(self):
return self._headers
def get_value(self,item,**kwargs):
- """
+ """
Handle different value types for XLSX. Item is a cell object.
"""
# Thx to Augusto C Men to point fast solution for XLS/XLSX dates
if item.is_date() and isinstance(item,(int,float)):
return datetime.date(1899,12,30) + datetime.timedelta(days=item)
-
return item.value
def get_items(self):
for row in self._reader.rows[1:]:
values = [self.get_value(c) for c in list(row)]
+ if not all(values): continue # empty lines are ignored
yield self.get_item(values)
View
14 data_importer/tests/__init__.py
@@ -33,7 +33,7 @@ def setUpClassData(klass):
'csv_invalid_cpf_sheet': os.path.join(mod_dir,'fixtures','csv_invalid_cpf_sheet.csv'), # used to get validation errors
'csv_sheet': os.path.join(mod_dir,'fixtures','csv_sheet.csv'),
'xls_sheet': os.path.join(mod_dir,'fixtures','xls_sheet.xls'),
- 'xlsx_sheet': os.path.join(mod_dir,'fixtures','xlsx_sheet.xlsx'),
+ 'xlsx_sheet': os.path.join(mod_dir,'fixtures','xlsx_sheet.xlsx'),
}
klass.f_headers = ['cpf','field3','field4','field5']
@@ -50,7 +50,7 @@ class ReaderTest(TestCase):
Test various readers results against already know data.
"""
-
+
def setUp(self):
setUpClassData(self)
@@ -154,7 +154,7 @@ def test_mock_log_instance_handlers(self):
instances = []
for i in importer.logger.handlers:
self.assertTrue(i.__class__.__name__ not in instances,u"More than one logger with same class found in importer.logger.handlers")
- instances.append(i.__class__.__name__)
+ instances.append(i.__class__.__name__)
class ImportersValidationsTests(TestCase):
"""
@@ -187,7 +187,6 @@ def test_invalid_errors(self):
importer = SimpleValidationsImporter(self.files['csv_invalid_cpf_sheet'])
self.assertTrue(not importer.is_valid(),u"Should return False to is_valid()")
self.assertNotEquals(SortedDict(),importer.errors) # importer.errors shouldn't be empty
-
for i in importer.errors:
self.assertEquals(True,i in self.invalid_lines)
for k,v in importer.errors[i].items():
@@ -238,9 +237,10 @@ def test_invalid_required_field_DB(self):
self.assertTrue(isinstance(importer.logger.handlers[0],DBLoggingHandler),u"Logger handler isn't DBLoggingHandler")
errors = Error.objects.all()
- self.assertTrue(u"RequiredFieldValidationsImporterDB_importer :: error :: Line 1, field field3: Field field3 is required!" in \
- str(errors[0]),u"Weird string for this test, check loggers.")
+
self.assertTrue(u"RequiredFieldValidationsImporterDB_importer :: error :: Line 1, field cpf: Field cpf is required!" in \
+ str(errors[0]),u"Weird string for this test, check loggers.")
+ self.assertTrue(u"RequiredFieldValidationsImporterDB_importer :: error :: Line 1, field field3: Field field3 is required!" in \
str(errors[1]),u"Weird string for this test, check loggers.")
self.assertTrue(u"RequiredFieldValidationsImporterDB_importer :: error :: Line 2, field cpf: Invalid CPF number." in \
str(errors[2]),u"Weird string for this test, check loggers.")
@@ -256,7 +256,7 @@ def test_save(self):
for i,know_data in enumerate(self.data_invalid):
data = results[i]
if not data and i+1 in invalid_lines: # dict data starts from 1
- self.assertEquals(data,False)
+ self.assertEquals(data,None)
else:
for k,v in know_data.items():
# RequiredFieldValidationsImporterDB should return field
View
12 data_importer/tests/importers.py
@@ -22,10 +22,10 @@ class SimpleValidationsImporter(BaseImportWithFields):
"""
Since all tested importers will validate same data, we can write only one
with validate methods.
- This importer doesn't implement save method that return data as dict in
+ This importer doesn't implement save method that return data as dict in
BaseImporter.save and doesn't put any field as required.
"""
- def clean_cpf(self,val):
+ def clean_cpf(self,val,row):
# field isn't required in this class!
if not val: return val
try:
@@ -39,7 +39,7 @@ class SimpleValidationsImporterDB(SimpleValidationsImporter):
"""
Since all tested importers will validate same data, we can write only one
with validate methods.
- This importer doesn't implement save method that return data as list of dict in
+ This importer doesn't implement save method that return data as list of dict in
BaseImporter.save and doesn't put any field as required.
"""
@@ -50,12 +50,12 @@ class RequiredFieldValidationsImporter(SimpleValidationsImporter):
"""
Since all tested importers will validate same data, we can write only one
with validate methods.
- This importer doesn't implement save method that return data as list of dict in
+ This importer doesn't implement save method that return data as list of dict in
BaseImporter.save but use CPF and FIELD3 fields as required.
"""
required_fields = ['cpf','field3']
- def clean_cpf(self,val):
+ def clean_cpf(self,val,row):
try:
val = CPF(val)
except ValueError,msg:
@@ -67,7 +67,7 @@ class RequiredFieldValidationsImporterDB(RequiredFieldValidationsImporter):
"""
Since all tested importers will validate same data, we can write only one
with validate methods.
- This importer doesn't implement save method that return data as list of dict in
+ This importer doesn't implement save method that return data as list of dict in
BaseImporter.save but use CPF and FIELD3 fields as required.
"""
View
2  data_importer/tests/mocks.py
@@ -16,7 +16,7 @@ def reset(self):
self.messages = {
'debug':[],
'info':[],
- 'warninig':[],
+ 'warning':[],
'error':[],
'critical':[],
}
View
2  sampleproject/settings.py
@@ -100,4 +100,4 @@
'data_importer.tests', # important to test logger to DB
)
-#TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
+TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
Please sign in to comment.
Something went wrong with that request. Please try again.