diff --git a/.travis.yml b/.travis.yml index 660da207a..d338afc01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,8 @@ install: - if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then pip install -r requirements-py3.txt; fi - if [[ $TRAVIS_PYTHON_VERSION == "3.5" ]]; then pip install -r requirements-py3.txt; fi - pip install coveralls +before_script: + - psql -U postgres -c 'CREATE DATABASE dummy_test' script: - nosetests --with-coverage --cover-package=csvkit after_success: diff --git a/CHANGELOG b/CHANGELOG index 698e52370..8deb86652 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -27,6 +27,7 @@ Fixes: * csvgrep can match multiline values. * csvgrep supports --no-header-row. * csvsql supports custom SQLAlchemy dialects. +* csvsql correctly escapes `%` characters in SQL queries. * csvstack supports stacking a single file. * FilteringCSVReader's any_match argument works correctly. * A file of two empty rows won't raise an error. diff --git a/csvkit/utilities/sql2csv.py b/csvkit/utilities/sql2csv.py index c828d66f8..91dff7f27 100644 --- a/csvkit/utilities/sql2csv.py +++ b/csvkit/utilities/sql2csv.py @@ -51,7 +51,10 @@ def main(self): for line in self.args.file: query += line - rows = conn.execute(query) + # Must escape '%'. + # @see https://github.com/onyxfish/csvkit/issues/440 + # @see https://bitbucket.org/zzzeek/sqlalchemy/commits/5bc1f17cb53248e7cea609693a3b2a9bb702545b + rows = conn.execute(query.replace('%', '%%')) output = agate.writer(self.output_file, **self.writer_kwargs) if not self.args.no_header_row: diff --git a/requirements-py3.txt b/requirements-py3.txt index b40a1c795..d23c824c0 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,6 +1,7 @@ coverage>=3.5.1b1 dbf>=0.96.005 nose>=1.1.2 +psycopg2 python-dateutil>=2.2 six>=1.6.1 sphinx>=1.0.7 diff --git a/tests/test_utilities/test_sql2csv.py b/tests/test_utilities/test_sql2csv.py index 568e3d485..877d70f71 100644 --- a/tests/test_utilities/test_sql2csv.py +++ b/tests/test_utilities/test_sql2csv.py @@ -24,7 +24,7 @@ def test_launch_new_instance(self): launch_new_instance() def setUp(self): - self.db_file = "foo.db" + self.db_file = 'foo.db' def tearDown(self): try: @@ -32,11 +32,14 @@ def tearDown(self): except OSError: pass - def csvsql(self, csv_file): + def csvsql(self, csv_file, db=None): """ Load test data into the DB and return it as a string for comparison. """ - args = ['--db', "sqlite:///" + self.db_file, '--table', 'foo', '--insert', csv_file] + if not db: + db = 'sqlite:///' + self.db_file + + args = ['--db', db, '--table', 'foo', '--insert', csv_file] utility = CSVSQL(args) utility.main() @@ -56,7 +59,7 @@ def test_query(self): def test_stdin(self): output_file = six.StringIO() - input_file = six.StringIO("select cast(3.1415 * 13.37 as integer) as answer") + input_file = six.StringIO('select cast(3.1415 * 13.37 as integer) as answer') with stdin_as_string(input_file): utility = SQL2CSV([], output_file) @@ -69,7 +72,7 @@ def test_stdin(self): def test_stdin_with_query(self): args = ['--query', 'select 6*9 as question'] output_file = six.StringIO() - input_file = six.StringIO("select cast(3.1415 * 13.37 as integer) as answer") + input_file = six.StringIO('select cast(3.1415 * 13.37 as integer) as answer') with stdin_as_string(input_file): utility = SQL2CSV(args, output_file) @@ -81,7 +84,7 @@ def test_stdin_with_query(self): def test_unicode(self): target_output = self.csvsql('examples/test_utf8.csv') - args = ['--db', "sqlite:///" + self.db_file, '--query', 'select * from foo'] + args = ['--db', 'sqlite:///' + self.db_file, '--query', 'select * from foo'] output_file = six.StringIO() utility = SQL2CSV(args, output_file) @@ -91,7 +94,7 @@ def test_unicode(self): def test_no_header_row(self): self.csvsql('examples/dummy.csv') - args = ['--db', "sqlite:///" + self.db_file, '--no-header-row', '--query', 'select * from foo'] + args = ['--db', 'sqlite:///' + self.db_file, '--no-header-row', '--query', 'select * from foo'] output_file = six.StringIO() utility = SQL2CSV(args, output_file) utility.main() @@ -102,7 +105,7 @@ def test_no_header_row(self): def test_linenumbers(self): self.csvsql('examples/dummy.csv') - args = ['--db', "sqlite:///" + self.db_file, '--linenumbers', '--query', 'select * from foo'] + args = ['--db', 'sqlite:///' + self.db_file, '--linenumbers', '--query', 'select * from foo'] output_file = six.StringIO() utility = SQL2CSV(args, output_file) utility.main() @@ -111,13 +114,24 @@ def test_linenumbers(self): self.assertTrue('line_number,a,b,c' in csv) self.assertTrue('1,1,2,3' in csv) - def test_wilcard(self): - self.csvsql('examples/dummy.csv') - args = ['--db', "sqlite:///" + self.db_file, '--query', "select * from foo where a LIKE '%'"] + def test_wilcard_on_sqlite(self): + self.csvsql('examples/iris.csv') + args = ['--db', 'sqlite:///' + self.db_file, '--query', "select * from foo where species LIKE '%'"] output_file = six.StringIO() utility = SQL2CSV(args, output_file) utility.main() csv = output_file.getvalue() - self.assertTrue('a,b,c' in csv) - self.assertTrue('1,2,3' in csv) + self.assertTrue('sepal_length,sepal_width,petal_length,petal_width,species' in csv) + self.assertTrue('5.1,3.5,1.4,0.2,Iris-setosa' in csv) + + def test_wilcard_on_postgresql(self): + self.csvsql('examples/iris.csv', 'postgres:///dummy_test') + args = ['--db', 'postgres:///dummy_test', '--query', "select * from foo where species LIKE '%'"] + output_file = six.StringIO() + utility = SQL2CSV(args, output_file) + utility.main() + csv = output_file.getvalue() + + self.assertTrue('sepal_length,sepal_width,petal_length,petal_width,species' in csv) + self.assertTrue('5.1,3.5,1.4,0.2,Iris-setosa' in csv)