Merge pull request #542 from onyxfish/dedupe

in2csv: Reduce code duplication

csvkit/convert/__init__.py

@@ -1,21 +1,16 @@
 #!/usr/bin/env python
 
+import agate
+import agateexcel
+import dbf
 import six
 
-from csvkit.convert.csvitself import csv2csv
 from csvkit.convert.fixed import fixed2csv
 from csvkit.convert.geojs import geojson2csv
-from csvkit.convert.js import json2csv
-from csvkit.convert.xls import xls2csv
-from csvkit.convert.xlsx import xlsx2csv
 
-SUPPORTED_FORMATS = ['fixed', 'xls', 'xlsx', 'csv', 'json', 'geojson', 'ndjson']
+agateexcel.patch()
 
-# DBF is supported for Python 2 only
-if six.PY2:
-    from csvkit.convert.dbase import dbf2csv
-
-    SUPPORTED_FORMATS.append('dbf')
+SUPPORTED_FORMATS = ['csv', 'dbf', 'geojson', 'json', 'ndjson', 'fixed', 'xls', 'xlsx']
 
 
 def convert(f, format, schema=None, key=None, **kwargs):
@@ -33,22 +28,30 @@ def convert(f, format, schema=None, key=None, **kwargs):
             raise ValueError('schema must not be null when format is "fixed"')
 
         return fixed2csv(f, schema, **kwargs)
-    elif format == 'xls':
-        return xls2csv(f, **kwargs)
-    elif format == 'xlsx':
-        return xlsx2csv(f, **kwargs)
-    elif format == 'json':
-        return json2csv(f, key, **kwargs)
-    elif format == 'ndjson':
-        return json2csv(f, key, newline=True, **kwargs)
     elif format == 'geojson':
         return geojson2csv(f, **kwargs)
-    elif format == 'csv':
-        return csv2csv(f, **kwargs)
-    elif format == 'dbf':
-        if six.PY3:
-            raise ValueError('format "dbf" is not supported forthis version of Python.')
-        return dbf2csv(f, **kwargs)
+    elif format in ('csv', 'dbf', 'json', 'ndjson', 'xls', 'xlsx'):
+        if format == 'csv':
+            table = agate.Table.from_csv(f, **kwargs)
+        elif format == 'json':
+            table = agate.Table.from_json(f, key=key, **kwargs)
+        elif format == 'ndjson':
+            table = agate.Table.from_json(f, key=key, newline=True, **kwargs)
+        elif format == 'xls':
+            table = agate.Table.from_xls(f, sheet=kwargs.get('sheet', None))
+        elif format == 'xlsx':
+            table = agate.Table.from_xlsx(f, sheet=kwargs.get('sheet', None))
+        elif format == 'dbf':
+            with dbf.Table(f.name) as db:
+                column_names = db.field_names
+                table = agate.Table(db, column_names)
+
+        output = six.StringIO()
+        table.to_csv(output)
+        result = output.getvalue()
+        output.close()
+
+        return result
     else:
         raise ValueError('format "%s" is not supported' % format)
 

tests/test_utilities/test_in2csv.py

@@ -15,40 +15,46 @@
 
 
 class TestIn2CSV(unittest.TestCase):
+    def assertConverted(self, input_format, input_filename, output_filename, additional_args=[]):
+        args = ['-f', input_format, input_filename] + additional_args
+        output_file = six.StringIO()
+
+        utility = In2CSV(args, output_file)
+        utility.main()
+
+        target_output = open(output_filename, 'r').read()
+        self.assertEqual(output_file.getvalue(), target_output)
 
     def test_launch_new_instance(self):
         with patch.object(sys, 'argv', ['in2csv', 'examples/dummy.csv']):
             launch_new_instance()
 
-    def test_convert_xls(self):
-        args = ['-f', 'xls', 'examples/test.xls']
-        output_file = six.StringIO()
+    def test_convert_csv(self):
+        self.assertConverted('csv', 'examples/testfixed_converted.csv', 'examples/testfixed_converted.csv')
 
-        utility = In2CSV(args, output_file)
-        utility.main()
+    def test_convert_csv(self):
+        self.assertConverted('dbf', 'examples/testdbf.dbf', 'examples/testdbf_converted.csv')
 
-        target_output = open('examples/testxls_converted.csv', 'r').read()
-        self.assertEqual(output_file.getvalue(), target_output)
+    def test_convert_json(self):
+        self.assertConverted('json', 'examples/testjson.json', 'examples/testjson_converted.csv')
 
-    def test_convert_xlsx(self):
-        args = ['-f', 'xlsx', 'examples/test.xlsx']
-        output_file = six.StringIO()
+    def test_convert_ndjson(self):
+        self.assertConverted('ndjson', 'examples/testjson_multiline.json', 'examples/testjson_multiline_converted.csv')
 
-        utility = In2CSV(args, output_file)
-        utility.main()
+    def test_convert_nested_json(self):
+        self.assertConverted('json', 'examples/testjson_nested.json', 'examples/testjson_nested_converted.csv')
 
-        target_output = open('examples/testxlsx_converted.csv', 'r').read()
-        self.assertEqual(output_file.getvalue(), target_output)
+    def test_convert_xls(self):
+        self.assertConverted('xls', 'examples/test.xls', 'examples/testxls_converted.csv')
 
-    def test_convert_specific_xls_sheet(self):
-        args = ['-f', 'xls', '--sheet', 'data', 'examples/sheets.xls']
-        output_file = six.StringIO()
+    def test_convert_xls_with_sheet(self):
+        self.assertConverted('xls', 'examples/sheets.xls', 'examples/testxls_converted.csv', ['--sheet', 'data'])
 
-        utility = In2CSV(args, output_file)
-        utility.main()
+    def test_convert_xlsx(self):
+        self.assertConverted('xlsx', 'examples/test.xlsx', 'examples/testxlsx_converted.csv')
 
-        target_output = open('examples/testxls_converted.csv', 'r').read()
-        self.assertEqual(output_file.getvalue(), target_output)
+    def test_convert_xlsx(self):
+        self.assertConverted('xlsx', 'examples/sheets.xlsx', 'examples/testxlsx_converted.csv', ['--sheet', 'data'])
 
     def test_csv_no_headers(self):
         args = ['--no-header-row', 'examples/no_header_row.csv']
@@ -60,33 +66,3 @@ def test_csv_no_headers(self):
         output = output_file.getvalue()
 
         self.assertTrue('A,B,C' in output)
-
-    def test_convert_json(self):
-        args = ['examples/testjson.json']
-        output_file = six.StringIO()
-
-        utility = In2CSV(args, output_file)
-        utility.main()
-
-        target_output = open('examples/testjson_converted.csv', 'r').read()
-        self.assertEqual(output_file.getvalue(), target_output)
-
-    def test_convert_ndjson(self):
-        args = ['examples/testjson_multiline.json', '-f', 'ndjson']
-        output_file = six.StringIO()
-
-        utility = In2CSV(args, output_file)
-        utility.main()
-
-        target_output = open('examples/testjson_multiline_converted.csv', 'r').read()
-        self.assertEqual(output_file.getvalue(), target_output)
-
-    def test_convert_nested_json(self):
-        args = ['examples/testjson_nested.json']
-        output_file = six.StringIO()
-
-        utility = In2CSV(args, output_file)
-        utility.main()
-
-        target_output = open('examples/testjson_nested_converted.csv', 'r').read()
-        self.assertEqual(output_file.getvalue(), target_output)