Merge pull request #435 from ethanwhite/convert-float-nulls

Convert numeric nulls when also converting string nulls

lib/cleanup.py

@@ -1,9 +1,17 @@
+def floatable(value):
+    """Check if a value can be converted to a float"""
+    try:
+        float(value)
+        return True
+    except ValueError:
+        return False
+
 def correct_invalid_value(value, args):
     """This cleanup function replaces null indicators with None."""
     try:
         if value in [item for item in args["nulls"]]:
             return None
-        if float(value) in [float(item) for item in args["nulls"]]:
+        if float(value) in [float(item) for item in args["nulls"] if floatable(item)]:
             return None
         return value
     except:

test/test_retriever.py

@@ -6,6 +6,7 @@
 from retriever.lib.table import Table
 from retriever.lib.templates import BasicTextTemplate
 from retriever.lib.tools import getmd5
+from retriever.lib.cleanup import correct_invalid_value
 from retriever import DATA_WRITE_PATH
 from nose.tools import with_setup
 
@@ -50,7 +51,12 @@ def test_auto_get_delimiter_semicolon():
     test_engine.auto_get_delimiter("a;b;c;,d")
     assert test_engine.table.delimiter == ";"
 
+def test_correct_invalid_value_string():
+    assert correct_invalid_value('NA', {'nulls': ['NA', '-999']}) == None
 
+def test_correct_invalid_value_number():
+    assert correct_invalid_value(-999, {'nulls': ['NA', '-999']}) == None
+
 def test_create_db_statement():
     """Test creating the create database SQL statement"""
     assert test_engine.create_db_statement() == 'CREATE DATABASE test_abc'