diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index d74067d4928..e327b338d66 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -3,6 +3,7 @@ import pylons import sqlalchemy +import ckan.lib.base as base import ckan.lib.navl.dictization_functions import ckan.logic as logic import ckan.plugins as p @@ -215,6 +216,73 @@ def datastore_upsert(context, data_dict): return result +def datastore_info(context, data_dict): + ''' + Returns information about the data imported, such as column names + and types. + + :rtype: A dictionary describing the columns and their types. + :param id: Id of the resource we want info about + :type id: A UUID + ''' + def _type_lookup(t): + if t in ['numeric', 'integer']: + return 'number' + + if t.startswith('timestamp'): + return "date" + + return "text" + + p.toolkit.check_access('datastore_info', context, data_dict) + + resource_id = _get_or_bust(data_dict, 'id') + resource = p.toolkit.get_action('resource_show')(context, {'id':resource_id}) + + data_dict['connection_url'] = pylons.config['ckan.datastore.read_url'] + + resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" + WHERE name = :id AND alias_of IS NULL''') + results = db._get_engine(data_dict).execute(resources_sql, id=resource_id) + res_exists = results.rowcount > 0 + if not res_exists: + raise p.toolkit.ObjectNotFound(p.toolkit._( + u'Resource "{0}" was not found.'.format(resource_id) + )) + + info = {'schema': {}, 'meta': {}} + + schema_results = None + meta_results = None + try: + schema_sql = sqlalchemy.text(u''' + SELECT column_name, data_type + FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = :resource_id; + ''') + schema_results = db._get_engine(data_dict).execute(schema_sql, resource_id=resource_id) + for row in schema_results.fetchall(): + k = row[0] + v = row[1] + if k.startswith('_'): # Skip internal rows + continue + info['schema'][k] = _type_lookup(v) + + # We need to make sure the resource_id is a valid resource_id before we use it like + # this, we have done that above. + meta_sql = sqlalchemy.text(u''' + SELECT count(_id) FROM "{}"; + '''.format(resource_id)) + meta_results = db._get_engine(data_dict).execute(meta_sql, resource_id=resource_id) + info['meta']['count'] = meta_results.fetchone()[0] + finally: + if schema_results: + schema_results.close() + if meta_results: + meta_results.close() + + return info + + def datastore_delete(context, data_dict): '''Deletes a table or a set of records from the DataStore. diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py index 73d78e03b84..bce4de74254 100644 --- a/ckanext/datastore/logic/auth.py +++ b/ckanext/datastore/logic/auth.py @@ -38,6 +38,11 @@ def datastore_delete(context, data_dict): return datastore_auth(context, data_dict) +@p.toolkit.auth_allow_anonymous_access +def datastore_info(context, data_dict): + return datastore_auth(context, data_dict, 'resource_show') + + @p.toolkit.auth_allow_anonymous_access def datastore_search(context, data_dict): return datastore_auth(context, data_dict, 'resource_show') diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 2380ce17c0f..a7db61dbd52 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -243,6 +243,7 @@ def get_actions(self): 'datastore_upsert': action.datastore_upsert, 'datastore_delete': action.datastore_delete, 'datastore_search': action.datastore_search, + 'datastore_info': action.datastore_info, } if not self.legacy_mode: actions.update({ @@ -255,6 +256,7 @@ def get_auth_functions(self): return {'datastore_create': auth.datastore_create, 'datastore_upsert': auth.datastore_upsert, 'datastore_delete': auth.datastore_delete, + 'datastore_info': auth.datastore_info, 'datastore_search': auth.datastore_search, 'datastore_search_sql': auth.datastore_search_sql, 'datastore_change_permissions': auth.datastore_change_permissions} diff --git a/ckanext/datastore/tests/test_info.py b/ckanext/datastore/tests/test_info.py new file mode 100644 index 00000000000..37b44faa199 --- /dev/null +++ b/ckanext/datastore/tests/test_info.py @@ -0,0 +1,57 @@ +import json +import nose +import pprint + +import pylons +import sqlalchemy.orm as orm + +import ckan.plugins as p +import ckan.lib.create_test_data as ctd +import ckan.model as model +import ckan.tests as tests + +import ckanext.datastore.db as db +from ckanext.datastore.tests.helpers import extract, rebuild_all_dbs + +import ckan.new_tests.helpers as helpers +import ckan.new_tests.factories as factories + +assert_equals = nose.tools.assert_equals +assert_raises = nose.tools.assert_raises + + +class TestDatastoreInfo(object): + @classmethod + def setup_class(cls): + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") + plugin = p.load('datastore') + if plugin.legacy_mode: + # make sure we undo adding the plugin + p.unload('datastore') + raise nose.SkipTest("Info is not supported in legacy mode") + + @classmethod + def teardown_class(cls): + p.unload('datastore') + helpers.reset_db() + + def test_info_success(self): + resource = factories.Resource() + data = { + 'resource_id': resource['id'], + 'force': True, + 'records': [ + {'from': 'Brazil', 'to': 'Brazil', 'num': 2}, + {'from': 'Brazil', 'to': 'Italy', 'num': 22} + ], + } + result = helpers.call_action('datastore_create', **data) + + info = helpers.call_action('datastore_info', id=resource['id']) + + assert info['meta']['count'] == 2, info['meta'] + assert len(info['schema']) == 3 + assert info['schema']['to'] == 'text' + assert info['schema']['from'] == 'text' + assert info['schema']['num'] == 'number', info['schema']