From 3bf2aab325fbe59a2e7aefef513adc1b52292ad4 Mon Sep 17 00:00:00 2001
From: David Read <david.read@hackneyworkshop.com>
Date: Tue, 22 Oct 2019 23:47:54 +0100
Subject: [PATCH] Fix exceptions seen with percent in column name eg datastore
 search filter

---
 ckanext/datastore/backend/postgres.py  |  9 +++++++--
 ckanext/datastore/tests/test_search.py | 22 ++++++++++++++++++++++
 ckanext/datastore/tests/test_upsert.py | 12 ++++++------
 3 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py
index 44d78030097..d8601e7da5a 100644
--- a/ckanext/datastore/backend/postgres.py
+++ b/ckanext/datastore/backend/postgres.py
@@ -365,6 +365,10 @@ def _where_clauses(data_dict, fields_types):
         if field not in fields_types:
             continue
         field_array_type = _is_array_type(fields_types[field])
+        # "%" needs to be escaped as "%%" in any query going to
+        # connection.execute, otherwise it will think the "%" is for
+        # substituting a bind parameter
+        field = field.replace('%', '%%')
         if isinstance(value, list) and not field_array_type:
             clause_str = (u'"{0}" in ({1})'.format(field,
                           ','.join(['%s'] * len(value))))
@@ -1148,8 +1152,9 @@ def upsert_data(context, data_dict):
                                     WHERE ({primary_key}) = ({primary_value}));
                 '''.format(
                     res_id=data_dict['resource_id'],
-                    columns=u', '.join([u'"{0}"'.format(field)
-                                        for field in used_field_names]),
+                    columns=u', '.join([
+                        u'"{0}"'.format(field.replace('%', '%%'))
+                        for field in used_field_names]),
                     values=u', '.join(['%s::nested'
                                        if field['type'] == 'nested' else '%s'
                                        for field in used_fields]),
diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py
index bf0af321f2b..3a47c2feb0f 100644
--- a/ckanext/datastore/tests/test_search.py
+++ b/ckanext/datastore/tests/test_search.py
@@ -436,6 +436,28 @@ def test_search_limit_config_combination(self):
                                           {u'the year': 2015, u'_id': 2}])
         assert_equals(result['limit'], 2)
 
+    def test_search_filter_with_percent_in_column_name(self):
+        resource = factories.Resource()
+        data = {
+            'resource_id': resource['id'],
+            'force': True,
+            'primary_key': 'id',
+            'fields': [{'id': 'id', 'type': 'text'},
+                       {'id': 'bo%ok', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}],
+            'records': [
+                {'id': '1%',
+                 'bo%ok': u'El Nino',
+                 'author': 'Torres'}],
+        }
+        helpers.call_action('datastore_create', **data)
+
+        search_data = {
+            'resource_id': resource['id'],
+            'filters': {u'bo%ok': 'El Nino'}}
+        result = helpers.call_action('datastore_search', **search_data)
+        assert_equals(result['total'], 1)
+
 
 class TestDatastoreSearchLegacyTests(DatastoreLegacyTestBase):
     sysadmin_user = None
diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py
index 162ae658226..641af179c3f 100644
--- a/ckanext/datastore/tests/test_upsert.py
+++ b/ckanext/datastore/tests/test_upsert.py
@@ -212,11 +212,11 @@ def test_percent(self):
             'force': True,
             'primary_key': 'id',
             'fields': [{'id': 'id', 'type': 'text'},
-                       {'id': 'book', 'type': 'text'},
+                       {'id': 'bo%ok', 'type': 'text'},
                        {'id': 'author', 'type': 'text'}],
             'records': [
                 {'id': '1%',
-                 'book': u'El Niño',
+                 'bo%ok': u'El Niño',
                  'author': 'Torres'}],
         }
         helpers.call_action('datastore_create', **data)
@@ -227,18 +227,18 @@ def test_percent(self):
             'method': 'upsert',
             'records': [
                 {'id': '1%',
-                 'book': u'The % boy',
+                 'bo%ok': u'The % boy',
                  'author': u'F Torres'},
                 {'id': '2%',
-                 'book': u'Gu%ide',
+                 'bo%ok': u'Gu%ide',
                  'author': u'Adams'}],
         }
         helpers.call_action('datastore_upsert', **data)
 
         search_result = _search(resource['id'])
         assert_equal(search_result['total'], 2)
-        assert_equal(search_result['records'][0]['book'], 'The % boy')
-        assert_equal(search_result['records'][1]['book'], 'Gu%ide')
+        assert_equal(search_result['records'][0]['bo%ok'], 'The % boy')
+        assert_equal(search_result['records'][1]['bo%ok'], 'Gu%ide')
 
     def test_missing_key(self):
         resource = factories.Resource()