Merge pull request #291 from alphagov/parse-csv-numbers

Parse csv numbers

backdrop/core/upload/parse_csv.py

@@ -4,15 +4,12 @@
 
 
 def parse_csv(incoming_data):
+    reader = unicode_csv_reader(
+        ignore_comment_lines(lines(incoming_data)), "utf-8")
     return [
-        ignore_empty_rows(
-            ignore_comment_column(
-                unicode_csv_reader(
-                    ignore_comment_lines(lines(incoming_data)), "utf-8"
-                )
-            )
-        )
-    ]
+        parse_cells_as_numbers(
+            ignore_empty_rows(
+                ignore_comment_column(reader)))]
 
 
 def lines(stream):
@@ -26,7 +23,37 @@ def ignore_empty_rows(rows):
 
 
 def is_empty_row(row):
-    return all(not v for v in row)
+    """Returns True if all cells in a row evaluate to False
+
+    >>> is_empty_row(['', False, ''])
+    True
+    >>> is_empty_row(['', 'a', ''])
+    False
+    """
+    return not any(row)
+
+
+def parse_cells_as_numbers(rows):
+    return [[parse_as_number(cell) for cell in row] for row in rows]
+
+
+def parse_as_number(cell):
+    """Convert a string to an int or a float if it can be
+
+    >>> parse_as_number("1")
+    1
+    >>> parse_as_number("1.1")
+    1.1
+    >>> parse_as_number("foo")
+    'foo'
+    """
+    try:
+        return int(cell)
+    except ValueError:
+        try:
+            return float(cell)
+        except ValueError:
+            return cell
 
 
 def ignore_comment_lines(reader):

features/admin/csv_upload.feature

@@ -18,8 +18,8 @@ Feature: CSV Upload
     And   I click "Upload"
     Then  the "my_data_set" data_set should contain in any order:
           """
-          {"name": "Pawel", "age": "27", "nationality": "Polish"}
-          {"name": "Max", "age": "35", "nationality": "Italian"}
+          {"name": "Pawel", "age": 27, "nationality": "Polish"}
+          {"name": "Max", "age": 35, "nationality": "Italian"}
           """
 
   Scenario: UTF8 characters

tests/core/upload/test_parse_csv.py

@@ -126,6 +126,19 @@ def test_preserve_newlines_in_quoted_values(self):
             ["value", "value\nwith newline"],
         )))
 
+    def test_parsing_numbers_in_cells(self):
+        csv = u"int,float,string\n12,12.1,a string"
+
+        csv_stream = _string_io(csv, "utf-8")
+
+        data = parse_csv(csv_stream)
+
+        print(list(data[0]))
+        assert_that(data, only_contains(only_contains(
+            ["int", "float", "string"],
+            [12, 12.1, "a string"],
+        )))
+
 
 class LinesGeneratorTest(unittest.TestCase):
     def test_handles_CR_LF_and_CRLF(self):