Merge 85cc864 into 8639457

.travis.yml

@@ -14,6 +14,8 @@ install:
   - if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then pip install -r requirements-py3.txt; fi
   - if [[ $TRAVIS_PYTHON_VERSION == "3.5" ]]; then pip install -r requirements-py3.txt; fi
   - pip install coveralls
+before_script:
+  - psql -U postgres -c 'CREATE DATABASE dummy_test'
 script:
   - nosetests --with-coverage --cover-package=csvkit
 after_success:

CHANGELOG

@@ -27,6 +27,7 @@ Fixes:
 * csvgrep can match multiline values.
 * csvgrep supports --no-header-row.
 * csvsql supports custom SQLAlchemy dialects.
+* csvsql correctly escapes `%` characters in SQL queries.
 * csvstack supports stacking a single file.
 * FilteringCSVReader's any_match argument works correctly.
 * A file of two empty rows won't raise an error.

csvkit/utilities/sql2csv.py

@@ -51,7 +51,10 @@ def main(self):
             for line in self.args.file:
                 query += line
 
-        rows = conn.execute(query)
+        # Must escape '%'.
+        # @see https://github.com/onyxfish/csvkit/issues/440
+        # @see https://bitbucket.org/zzzeek/sqlalchemy/commits/5bc1f17cb53248e7cea609693a3b2a9bb702545b
+        rows = conn.execute(query.replace('%', '%%'))
         output = agate.writer(self.output_file, **self.writer_kwargs)
 
         if not self.args.no_header_row:

requirements-py3.txt

@@ -1,6 +1,7 @@
 coverage>=3.5.1b1
 dbf>=0.96.005
 nose>=1.1.2
+psycopg2
 python-dateutil>=2.2
 six>=1.6.1
 sphinx>=1.0.7

tests/test_utilities/test_sql2csv.py

@@ -24,19 +24,22 @@ def test_launch_new_instance(self):
             launch_new_instance()
 
     def setUp(self):
-        self.db_file = "foo.db"
+        self.db_file = 'foo.db'
 
     def tearDown(self):
         try:
             os.remove(self.db_file)
         except OSError:
             pass
 
-    def csvsql(self, csv_file):
+    def csvsql(self, csv_file, db=None):
         """
         Load test data into the DB and return it as a string for comparison.
         """
-        args = ['--db', "sqlite:///" + self.db_file, '--table', 'foo', '--insert', csv_file]
+        if not db:
+            db = 'sqlite:///' + self.db_file
+
+        args = ['--db', db, '--table', 'foo', '--insert', csv_file]
 
         utility = CSVSQL(args)
         utility.main()
@@ -56,7 +59,7 @@ def test_query(self):
 
     def test_stdin(self):
         output_file = six.StringIO()
-        input_file = six.StringIO("select cast(3.1415 * 13.37 as integer) as answer")
+        input_file = six.StringIO('select cast(3.1415 * 13.37 as integer) as answer')
 
         with stdin_as_string(input_file):
             utility = SQL2CSV([], output_file)
@@ -69,7 +72,7 @@ def test_stdin(self):
     def test_stdin_with_query(self):
         args = ['--query', 'select 6*9 as question']
         output_file = six.StringIO()
-        input_file = six.StringIO("select cast(3.1415 * 13.37 as integer) as answer")
+        input_file = six.StringIO('select cast(3.1415 * 13.37 as integer) as answer')
 
         with stdin_as_string(input_file):
             utility = SQL2CSV(args, output_file)
@@ -81,7 +84,7 @@ def test_stdin_with_query(self):
 
     def test_unicode(self):
         target_output = self.csvsql('examples/test_utf8.csv')
-        args = ['--db', "sqlite:///" + self.db_file, '--query', 'select * from foo']
+        args = ['--db', 'sqlite:///' + self.db_file, '--query', 'select * from foo']
         output_file = six.StringIO()
 
         utility = SQL2CSV(args, output_file)
@@ -91,7 +94,7 @@ def test_unicode(self):
 
     def test_no_header_row(self):
         self.csvsql('examples/dummy.csv')
-        args = ['--db', "sqlite:///" + self.db_file, '--no-header-row', '--query', 'select * from foo']
+        args = ['--db', 'sqlite:///' + self.db_file, '--no-header-row', '--query', 'select * from foo']
         output_file = six.StringIO()
         utility = SQL2CSV(args, output_file)
         utility.main()
@@ -102,7 +105,7 @@ def test_no_header_row(self):
 
     def test_linenumbers(self):
         self.csvsql('examples/dummy.csv')
-        args = ['--db', "sqlite:///" + self.db_file, '--linenumbers', '--query', 'select * from foo']
+        args = ['--db', 'sqlite:///' + self.db_file, '--linenumbers', '--query', 'select * from foo']
         output_file = six.StringIO()
         utility = SQL2CSV(args, output_file)
         utility.main()
@@ -111,13 +114,24 @@ def test_linenumbers(self):
         self.assertTrue('line_number,a,b,c' in csv)
         self.assertTrue('1,1,2,3' in csv)
 
-    def test_wilcard(self):
-        self.csvsql('examples/dummy.csv')
-        args = ['--db', "sqlite:///" + self.db_file, '--query', "select * from foo where a LIKE '%'"]
+    def test_wilcard_on_sqlite(self):
+        self.csvsql('examples/iris.csv')
+        args = ['--db', 'sqlite:///' + self.db_file, '--query', "select * from foo where species LIKE '%'"]
         output_file = six.StringIO()
         utility = SQL2CSV(args, output_file)
         utility.main()
         csv = output_file.getvalue()
 
-        self.assertTrue('a,b,c' in csv)
-        self.assertTrue('1,2,3' in csv)
+        self.assertTrue('sepal_length,sepal_width,petal_length,petal_width,species' in csv)
+        self.assertTrue('5.1,3.5,1.4,0.2,Iris-setosa' in csv)
+
+    def test_wilcard_on_postgresql(self):
+        self.csvsql('examples/iris.csv', 'postgres:///dummy_test')
+        args = ['--db', 'postgres:///dummy_test', '--query', "select * from foo where species LIKE '%'"]
+        output_file = six.StringIO()
+        utility = SQL2CSV(args, output_file)
+        utility.main()
+        csv = output_file.getvalue()
+
+        self.assertTrue('sepal_length,sepal_width,petal_length,petal_width,species' in csv)
+        self.assertTrue('5.1,3.5,1.4,0.2,Iris-setosa' in csv)