diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index f2b4de917f9..bebda375346 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -78,7 +78,10 @@ def _is_valid_field_name(name): return True -_is_valid_table_name = _is_valid_field_name +def _is_valid_table_name(name): + if '%' in name: + return False + return _is_valid_field_name(name) def _validate_int(i, field_name): @@ -278,7 +281,7 @@ def create_table(context, data_dict): }) fields = datastore_fields + supplied_fields + extra_fields - sql_fields = u", ".join([u'"{0}" {1}'.format(f['id'], f['type']) + sql_fields = u", ".join([u'"{0}" {1}'.format(f['id'].replace('%', '%%'), f['type']) for f in fields]) sql_string = u'CREATE TABLE "{0}" ({1});'.format( @@ -341,7 +344,7 @@ def create_indexes(context, data_dict): 'index': [('The field {0} is not a valid column name.').format( index)] }) - fields_string = u','.join(['"%s"' % field for field in fields]) + fields_string = u','.join(['"%s"' % field.replace('%', '%%') for field in fields]) sql_index_strings.append(sql_index_string.format( res_id=data_dict['resource_id'], unique='', fields=fields_string)) @@ -363,7 +366,7 @@ def create_indexes(context, data_dict): if primary_key: sql_index_strings.append(sql_index_string.format( res_id=data_dict['resource_id'], unique='unique', - fields=u','.join(['"%s"' % field for field in primary_key]))) + fields=u','.join(['"%s"' % field.replace('%', '%%') for field in primary_key]))) map(context['connection'].execute, sql_index_strings) @@ -388,7 +391,7 @@ def _drop_indexes(context, data_dict, unique=False): indexes_to_drop = context['connection'].execute( sql_get_index_string, data_dict['resource_id']).fetchall() for index in indexes_to_drop: - context['connection'].execute(sql_drop_index.format(index[0])) + context['connection'].execute(sql_drop_index.format(index[0]).replace('%', '%%')) def alter_table(context, data_dict): @@ -440,7 +443,7 @@ def alter_table(context, data_dict): for field in new_fields: sql = 'ALTER TABLE "{0}" ADD "{1}" {2}'.format( data_dict['resource_id'], - field['id'], + field['id'].replace('%', '%%'), field['type']) context['connection'].execute(sql) @@ -465,7 +468,7 @@ def upsert_data(context, data_dict): fields = _get_fields(context, data_dict) field_names = _pluck('id', fields) records = data_dict['records'] - sql_columns = ", ".join(['"%s"' % name for name in field_names] + sql_columns = ", ".join(['"%s"' % name.replace('%', '%%') for name in field_names] + ['"_full_text"']) if method == INSERT: diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py index 50026881d79..346a7c44280 100644 --- a/ckanext/datastore/tests/test_datastore.py +++ b/ckanext/datastore/tests/test_datastore.py @@ -80,7 +80,20 @@ def test_create_invalid_alias_name(self): resource = model.Package.get('annakarenina').resources[0] data = { 'resource_id': resource.id, - 'aliases': 'foo"bar', + 'aliases': u'foo"bar', + 'fields': [{'id': 'book', 'type': 'text'}, + {'id': 'author', 'type': 'text'}] + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + assert res_dict['success'] is False + + data = { + 'resource_id': resource.id, + 'aliases': u'fo%25bar', 'fields': [{'id': 'book', 'type': 'text'}, {'id': 'author', 'type': 'text'}] } @@ -215,13 +228,13 @@ def test_create_basic(self): data = { 'resource_id': resource.id, 'aliases': aliases, - 'fields': [{'id': 'book', 'type': 'text'}, + 'fields': [{'id': 'boo%k', 'type': 'text'}, {'id': 'author', 'type': 'json'}], - 'indexes': [['book', 'author'], 'book'], + 'indexes': [['boo%k', 'author'], 'boo%k'], 'records': [ - {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']}, - {'book': 'annakarenina', 'author': ['tolstoy', 'putin']}, - {'book': 'warandpeace'}] # treat author as null + {'boo%k': 'crime', 'author': ['tolstoy', 'dostoevsky']}, + {'boo%k': 'annakarenina', 'author': ['tolstoy', 'putin']}, + {'boo%k': 'warandpeace'}] # treat author as null } ### Firstly test to see if resource things it has datastore table postparams = '%s=1' % json.dumps({'id': resource.id}) @@ -248,7 +261,7 @@ def test_create_basic(self): assert results.rowcount == 3 for i, row in enumerate(results): - assert data['records'][i].get('book') == row['book'] + assert data['records'][i].get('boo%k') == row['boo%k'] assert data['records'][i].get('author') == ( json.loads(row['author'][0]) if row['author'] else None) @@ -288,7 +301,7 @@ def test_create_basic(self): ####### insert again simple data2 = { 'resource_id': resource.id, - 'records': [{'book': 'hagji murat', 'author': ['tolstoy']}] + 'records': [{'boo%k': 'hagji murat', 'author': ['tolstoy']}] } postparams = '%s=1' % json.dumps(data2) @@ -306,7 +319,7 @@ def test_create_basic(self): all_data = data['records'] + data2['records'] for i, row in enumerate(results): - assert all_data[i].get('book') == row['book'] + assert all_data[i].get('boo%k') == row['boo%k'] assert all_data[i].get('author') == ( json.loads(row['author'][0]) if row['author'] else None) @@ -319,7 +332,7 @@ def test_create_basic(self): ####### insert again extra field data3 = { 'resource_id': resource.id, - 'records': [{'book': 'crime and punsihment', + 'records': [{'boo%k': 'crime and punsihment', 'author': ['dostoevsky'], 'rating': 'good'}], 'indexes': ['rating'] } @@ -339,7 +352,7 @@ def test_create_basic(self): all_data = data['records'] + data2['records'] + data3['records'] for i, row in enumerate(results): - assert all_data[i].get('book') == row['book'], (i, all_data[i].get('book'), row['book']) + assert all_data[i].get('boo%k') == row['boo%k'], (i, all_data[i].get('boo%k'), row['boo%k']) assert all_data[i].get('author') == (json.loads(row['author'][0]) if row['author'] else None) results = c.execute('''select * from "{0}" where _full_text @@ to_tsquery('dostoevsky') '''.format(resource.id)) @@ -350,8 +363,8 @@ def test_create_basic(self): ####### insert again which will fail because of unique book name data4 = { 'resource_id': resource.id, - 'records': [{'book': 'warandpeace'}], - 'primary_key': 'book' + 'records': [{'boo%k': 'warandpeace'}], + 'primary_key': 'boo%k' } postparams = '%s=1' % json.dumps(data4)