diff --git a/bin/travis-install-dependencies b/bin/travis-install-dependencies index 8b9b1430d8f..77514786c2e 100755 --- a/bin/travis-install-dependencies +++ b/bin/travis-install-dependencies @@ -29,6 +29,7 @@ sudo -E -u postgres ./bin/postgres_init/1_create_ckan_db.sh sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh export PIP_USE_MIRRORS=true +pip install -r requirement-setuptools.txt --allow-all-external pip install -r requirements.txt --allow-all-external pip install -r dev-requirements.txt --allow-all-external diff --git a/circle.yml b/circle.yml index d9fff638dea..e560a8057a0 100644 --- a/circle.yml +++ b/circle.yml @@ -23,6 +23,7 @@ dependencies: && chmod +x ~/.local/bin/circleci-matrix" override: + - pip install -r requirement-setuptools.txt - pip install -r requirements.txt - pip install -r dev-requirements.txt - python setup.py develop diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index e711fea8561..40b7ef2b5b3 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -382,13 +382,16 @@ def bulk_process(self, id): data_dict = {'id': id, 'type': group_type} try: + self._check_access('bulk_update_public', context, {'org_id': id}) # Do not query for the group datasets when dictizing, as they will # be ignored and get requested on the controller anyway data_dict['include_datasets'] = False c.group_dict = self._action('group_show')(context, data_dict) c.group = context['group'] - except (NotFound, NotAuthorized): + except NotFound: abort(404, _('Group not found')) + except NotAuthorized: + abort(403, _('User %r not authorized to edit %s') % (c.user, id)) if not c.group_dict['is_organization']: # FIXME: better error @@ -634,14 +637,21 @@ def members(self, id): 'user': c.user} try: + data_dict = {'id': id} + check_access('group_edit_permissions', context, data_dict) c.members = self._action('member_list')( context, {'id': id, 'object_type': 'user'} ) - data_dict = {'id': id} data_dict['include_datasets'] = False c.group_dict = self._action('group_show')(context, data_dict) - except (NotFound, NotAuthorized): + except NotFound: abort(404, _('Group not found')) + except NotAuthorized: + abort( + 403, + _('User %r not authorized to edit members of %s') % ( + c.user, id)) + return self._render_template('group/members.html', group_type) def member_new(self, id): diff --git a/ckan/lib/navl/dictization_functions.py b/ckan/lib/navl/dictization_functions.py index ff4bae72cd5..c0ffcf65246 100644 --- a/ckan/lib/navl/dictization_functions.py +++ b/ckan/lib/navl/dictization_functions.py @@ -47,9 +47,17 @@ class State(object): class DictizationError(Exception): def __str__(self): + return unicode(self).encode('utf8') + + def __unicode__(self): + if hasattr(self, 'error') and self.error: + return u'{}: {}'.format(self.__class__.__name__, repr(self.error)) + return self.__class__.__name__ + + def __repr__(self): if hasattr(self, 'error') and self.error: - return repr(self.error) - return '' + return '<{} {}>'.format(self.__class__.__name__, repr(self.error)) + return '<{}>'.format(self.__class__.__name__) class Invalid(DictizationError): diff --git a/ckan/lib/navl/validators.py b/ckan/lib/navl/validators.py index 6e508c846a8..cfcb1a2d5d2 100644 --- a/ckan/lib/navl/validators.py +++ b/ckan/lib/navl/validators.py @@ -117,3 +117,9 @@ def convert_int(value, context): except ValueError: raise Invalid(_('Please enter an integer value')) +def unicode_only(value): + '''Accept only unicode values''' + + if not isinstance(value, unicode): + raise Invalid(_('Must be a Unicode string value')) + return value diff --git a/ckan/logic/auth/update.py b/ckan/logic/auth/update.py index 3d0e86c25cb..ae75f8cedb8 100644 --- a/ckan/logic/auth/update.py +++ b/ckan/logic/auth/update.py @@ -153,14 +153,15 @@ def group_edit_permissions(context, data_dict): user = context['user'] group = logic_auth.get_group_object(context, data_dict) - authorized = authz.has_user_permission_for_group_or_org(group.id, - user, - 'update') + authorized = authz.has_user_permission_for_group_or_org( + group.id, user, 'update') if not authorized: - return {'success': False, - 'msg': _('User %s not authorized to edit permissions of group %s') % - (str(user), group.id)} + return { + 'success': False, + 'msg': _('User %s not authorized to' + ' edit permissions of group %s') % + (str(user), group.id)} else: return {'success': True} diff --git a/ckan/templates/organization/members.html b/ckan/templates/organization/members.html index e9a2bf54c79..2f2e8130f42 100644 --- a/ckan/templates/organization/members.html +++ b/ckan/templates/organization/members.html @@ -10,7 +10,9 @@ {% endblock %} {% block primary_content_inner %} -

{{ _('{0} members'.format(c.members|length)) }}

+ {% set count = c.members|length %} + {% set members_count = ungettext('{count} member', '{count} members', count).format(count=count) %} +

{{ members_count }}

diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index be0bba84aa6..a7ffbc0b3b8 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -37,7 +37,7 @@ {{ _('View') }} {% elif res.resource_type == 'api' %} {{ _('API Endpoint') }} - {% elif not res.has_views or not res.can_be_previewed %} + {% elif (not res.has_views or not res.can_be_previewed) and not res.url_type == 'upload' %} {{ _('Go to resource') }} {% else %} {{ _('Download') }} diff --git a/ckan/templates/package/snippets/resource_item.html b/ckan/templates/package/snippets/resource_item.html index 8e437da6f7d..992397e10f4 100644 --- a/ckan/templates/package/snippets/resource_item.html +++ b/ckan/templates/package/snippets/resource_item.html @@ -39,7 +39,7 @@ {% if res.url and h.is_url(res.url) %}
  • - {% if res.has_views %} + {% if res.has_views or res.url_type == 'upload' %} {{ _('Download') }} {% else %} diff --git a/ckan/tests/controllers/test_group.py b/ckan/tests/controllers/test_group.py index 67e06c92ce9..a0fdc00c13f 100644 --- a/ckan/tests/controllers/test_group.py +++ b/ckan/tests/controllers/test_group.py @@ -282,9 +282,12 @@ def test_membership_list(self): group = self._create_group(user_one['name'], other_users) + env = {'REMOTE_USER': user_one['name'].encode('ascii')} + member_list_url = url_for(controller='group', action='members', id=group['id']) - member_list_response = app.get(member_list_url) + member_list_response = app.get( + member_list_url, extra_environ=env) assert_true('2 members' in member_list_response) @@ -375,7 +378,7 @@ def test_remove_member(self): env = {'REMOTE_USER': user_one['name'].encode('ascii')} remove_response = app.post(remove_url, extra_environ=env, status=302) # redirected to member list after removal - remove_response = remove_response.follow() + remove_response = remove_response.follow(extra_environ=env) assert_true('Group member has been deleted.' in remove_response) assert_true('1 members' in remove_response) diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index 0d0e80a64db..49d099f9fc1 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -179,7 +179,8 @@ class FunctionalTestBase(object): Allows configuration changes by overriding _apply_config_changes and resetting the CKAN config after your test class has run. It creates a webtest.TestApp at self.app for your class to use to make HTTP requests - to the CKAN web UI or API. + to the CKAN web UI or API. Also loads plugins defined by + _load_plugins in the class definition. If you're overriding methods that this class provides, like setup_class() and teardown_class(), make sure to use super() to call this class's methods @@ -196,10 +197,13 @@ def _get_test_app(cls): # leading _ because nose is terrible @classmethod def setup_class(cls): + import ckan.plugins as p # Make a copy of the Pylons config, so we can restore it in teardown. cls._original_config = dict(config) cls._apply_config_changes(config) cls._get_test_app() + for plugin in getattr(cls, '_load_plugins', []): + p.load(plugin) @classmethod def _apply_config_changes(cls, cfg): @@ -214,6 +218,9 @@ def setup(self): @classmethod def teardown_class(cls): + import ckan.plugins as p + for plugin in reversed(getattr(cls, '_load_plugins', [])): + p.unload(plugin) # Restore the Pylons config to its original values, in case any tests # changed any config settings. config.clear() diff --git a/ckan/tests/lib/navl/test_dictization_functions.py b/ckan/tests/lib/navl/test_dictization_functions.py index 0e475b5b32e..c984ee98b18 100644 --- a/ckan/tests/lib/navl/test_dictization_functions.py +++ b/ckan/tests/lib/navl/test_dictization_functions.py @@ -1,7 +1,7 @@ # encoding: utf-8 import nose -from ckan.lib.navl.dictization_functions import validate +from ckan.lib.navl.dictization_functions import validate, Invalid eq_ = nose.tools.eq_ @@ -49,3 +49,44 @@ def my_validator(key, data, errors, context): context = {} data, errors = validate(data_dict, schema, context) + + +class TestDictizationError(object): + + def test_str_error(self): + err_obj = Invalid('Some ascii error') + eq_(str(err_obj), "Invalid: 'Some ascii error'") + + def test_unicode_error(self): + err_obj = Invalid('Some ascii error') + eq_(unicode(err_obj), u"Invalid: 'Some ascii error'") + + def test_repr_error(self): + err_obj = Invalid('Some ascii error') + eq_(repr(err_obj), "") + + # Error msgs should be ascii, but let's just see what happens for unicode + + def test_str_unicode_error(self): + err_obj = Invalid(u'Some unicode \xa3 error') + eq_(str(err_obj), "Invalid: u'Some unicode \\xa3 error'") + + def test_unicode_unicode_error(self): + err_obj = Invalid(u'Some unicode \xa3 error') + eq_(unicode(err_obj), "Invalid: u'Some unicode \\xa3 error'") + + def test_repr_unicode_error(self): + err_obj = Invalid(u'Some unicode \xa3 error') + eq_(repr(err_obj), "") + + def test_str_blank(self): + err_obj = Invalid('') + eq_(str(err_obj), "Invalid") + + def test_unicode_blank(self): + err_obj = Invalid('') + eq_(unicode(err_obj), u"Invalid") + + def test_repr_blank(self): + err_obj = Invalid('') + eq_(repr(err_obj), "") diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index ed157734a69..7d34b7f42ba 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -94,7 +94,7 @@ def datapusher_submit(context, data_dict): if existing_task.get('state') == 'pending': updated = datetime.datetime.strptime( existing_task['last_updated'], '%Y-%m-%dT%H:%M:%S.%f') - time_since_last_updated = datetime.datetime.now() - updated + time_since_last_updated = datetime.datetime.utcnow() - updated if time_since_last_updated > assume_task_stale_after: # it's been a while since the job was last updated - it's more # likely something went wrong with it and the state wasn't diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index d603ca386ab..cdf4024374d 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -68,8 +68,7 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) diff --git a/ckanext/datapusher/tests/test_interfaces.py b/ckanext/datapusher/tests/test_interfaces.py index 49b8efd7fad..8439dbff45c 100644 --- a/ckanext/datapusher/tests/test_interfaces.py +++ b/ckanext/datapusher/tests/test_interfaces.py @@ -61,8 +61,7 @@ def setup_class(cls): cls.sysadmin_user = factories.User(name='testsysadmin', sysadmin=True) cls.normal_user = factories.User(name='annafan') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index ed01ce95bfd..ad5da9b216f 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -98,9 +98,16 @@ def _is_valid_table_name(name): return _is_valid_field_name(name) -def _get_engine(data_dict): +def get_read_engine(): + return _get_engine_from_url(config['ckan.datastore.read_url']) + + +def get_write_engine(): + return _get_engine_from_url(config['ckan.datastore.write_url']) + + +def _get_engine_from_url(connection_url): '''Get either read or write engine.''' - connection_url = data_dict['connection_url'] engine = _engines.get(connection_url) if not engine: @@ -127,9 +134,7 @@ def _cache_types(context): log.info("Create nested type. Native JSON: {0!r}".format( native_json)) - data_dict = { - 'connection_url': config['ckan.datastore.write_url']} - engine = _get_engine(data_dict) + engine = get_write_engine() with engine.begin() as connection: connection.execute( 'CREATE TYPE "nested" AS (json {0}, extra text)'.format( @@ -714,6 +719,9 @@ def upsert_data(context, data_dict): toolkit._("The data was invalid (for example: a numeric value " "is out of range or was inserted into a text field)." )) + except sqlalchemy.exc.DatabaseError as err: + raise ValidationError( + {u'records': [_programming_error_summary(err)]}) elif method in [_UPDATE, _UPSERT]: unique_keys = _get_unique_key(context, data_dict) @@ -774,8 +782,13 @@ def upsert_data(context, data_dict): [u'"{0}"'.format(part) for part in unique_keys]), primary_value=u','.join(["%s"] * len(unique_keys)) ) - results = context['connection'].execute( - sql_string, used_values + [full_text] + unique_values) + try: + results = context['connection'].execute( + sql_string, used_values + [full_text] + unique_values) + except sqlalchemy.exc.DatabaseError as err: + raise ValidationError({ + u'records': [_programming_error_summary(err)], + u'_records_row': num}) # validate that exactly one row has been updated if results.rowcount != 1: @@ -803,9 +816,14 @@ def upsert_data(context, data_dict): for part in unique_keys]), primary_value=u','.join(["%s"] * len(unique_keys)) ) - context['connection'].execute( - sql_string, - (used_values + [full_text] + unique_values) * 2) + try: + context['connection'].execute( + sql_string, + (used_values + [full_text] + unique_values) * 2) + except sqlalchemy.exc.DatabaseError as err: + raise ValidationError({ + u'records': [_programming_error_summary(err)], + u'_records_row': num}) def _get_unique_key(context, data_dict): @@ -967,7 +985,6 @@ def validate(context, data_dict): fields_types) # Remove default elements in data_dict - del data_dict_copy['connection_url'] del data_dict_copy['resource_id'] data_dict_copy.pop('id', None) data_dict_copy.pop('include_total', None) @@ -1116,7 +1133,7 @@ def create(context, data_dict): :raises InvalidDataError: if there is an invalid value in the given data ''' - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) _cache_types(context) @@ -1136,6 +1153,11 @@ def create(context, data_dict): create_table(context, data_dict) else: alter_table(context, data_dict) + if 'triggers' in data_dict: + _create_triggers( + context['connection'], + data_dict['resource_id'], + data_dict['triggers']) insert_data(context, data_dict) create_indexes(context, data_dict) create_alias(context, data_dict) @@ -1173,6 +1195,37 @@ def create(context, data_dict): context['connection'].close() +def _create_triggers(connection, resource_id, triggers): + u''' + Delete existing triggers on table then create triggers + + Currently our schema requires "before insert or update" + triggers run on each row, so we're not reading "when" + or "for_each" parameters from triggers list. + ''' + existing = connection.execute( + u'SELECT tgname FROM pg_trigger WHERE tgrelid = %s::regclass', + resource_id) + sql_list = ( + [u'DROP TRIGGER {name} ON {table}'.format( + name=datastore_helpers.identifier(r[0]), + table=datastore_helpers.identifier(resource_id)) + for r in existing] + + [u'''CREATE TRIGGER {name} + BEFORE INSERT OR UPDATE ON {table} + FOR EACH ROW EXECUTE PROCEDURE {function}()'''.format( + # 1000 triggers per table should be plenty + name=datastore_helpers.identifier(u't%03d' % i), + table=datastore_helpers.identifier(resource_id), + function=datastore_helpers.identifier(t['function'])) + for i, t in enumerate(triggers)]) + try: + if sql_list: + connection.execute(u';\n'.join(sql_list)) + except ProgrammingError as pe: + raise ValidationError({u'triggers': [_programming_error_summary(pe)]}) + + def upsert(context, data_dict): ''' This method combines upsert insert and update on the datastore. The method @@ -1181,7 +1234,7 @@ def upsert(context, data_dict): Any error results in total failure! For now pass back the actual error. Should be transactional. ''' - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) @@ -1224,7 +1277,7 @@ def upsert(context, data_dict): def delete(context, data_dict): - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() _cache_types(context) @@ -1248,7 +1301,7 @@ def delete(context, data_dict): def search(context, data_dict): - engine = _get_engine(data_dict) + engine = get_read_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) _cache_types(context) @@ -1275,7 +1328,7 @@ def search(context, data_dict): def search_sql(context, data_dict): - engine = _get_engine(data_dict) + engine = get_read_engine() context['connection'] = engine.connect() timeout = context.get('query_timeout', _TIMEOUT) _cache_types(context) @@ -1367,7 +1420,7 @@ def _change_privilege(context, data_dict, what): def make_private(context, data_dict): log.info('Making resource {resource_id!r} private'.format(**data_dict)) - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() trans = context['connection'].begin() try: @@ -1379,7 +1432,7 @@ def make_private(context, data_dict): def make_public(context, data_dict): log.info('Making resource {resource_id!r} public'.format(**data_dict)) - engine = _get_engine(data_dict) + engine = get_write_engine() context['connection'] = engine.connect() trans = context['connection'].begin() try: @@ -1390,12 +1443,70 @@ def make_public(context, data_dict): def get_all_resources_ids_in_datastore(): - read_url = config.get('ckan.datastore.read_url') - write_url = config.get('ckan.datastore.write_url') - data_dict = { - 'connection_url': read_url or write_url - } resources_sql = sqlalchemy.text(u'''SELECT name FROM "_table_metadata" WHERE alias_of IS NULL''') - query = _get_engine(data_dict).execute(resources_sql) + query = get_read_engine().execute(resources_sql) return [q[0] for q in query.fetchall()] + + +def create_function(name, arguments, rettype, definition, or_replace): + sql = u''' + CREATE {or_replace} FUNCTION + {name}({args}) RETURNS {rettype} AS {definition} + LANGUAGE plpgsql;'''.format( + or_replace=u'OR REPLACE' if or_replace else u'', + name=datastore_helpers.identifier(name), + args=u', '.join( + u'{argname} {argtype}'.format( + argname=datastore_helpers.identifier(a['argname']), + argtype=datastore_helpers.identifier(a['argtype'])) + for a in arguments), + rettype=datastore_helpers.identifier(rettype), + definition=datastore_helpers.literal_string(definition)) + + try: + _write_engine_execute(sql) + except ProgrammingError as pe: + key = ( + u'name' if pe.args[0].startswith('(ProgrammingError) function') + else u'definition') + raise ValidationError({key: [_programming_error_summary(pe)]}) + + +def drop_function(name, if_exists): + sql = u''' + DROP FUNCTION {if_exists} {name}(); + '''.format( + if_exists=u'IF EXISTS' if if_exists else u'', + name=datastore_helpers.identifier(name)) + + try: + _write_engine_execute(sql) + except ProgrammingError as pe: + raise ValidationError({u'name': [_programming_error_summary(pe)]}) + + +def _write_engine_execute(sql): + connection = get_write_engine().connect() + # No special meaning for '%' in sql parameter: + connection = connection.execution_options(no_parameters=True) + trans = connection.begin() + try: + connection.execute(sql) + trans.commit() + except Exception: + trans.rollback() + raise + finally: + connection.close() + + +def _programming_error_summary(pe): + u''' + return the text description of a sqlalchemy DatabaseError + without the actual SQL included, for raising as a + ValidationError to send back to API users + ''' + # first line only, after the '(ProgrammingError)' text + message = pe.args[0].split('\n')[0].decode('utf8') + return message.split(u') ', 1)[-1] diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 27e74599668..4f49bb9373f 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -55,6 +55,13 @@ def datastore_create(context, data_dict): :type primary_key: list or comma separated string :param indexes: indexes on table (optional) :type indexes: list or comma separated string + :param triggers: trigger functions to apply to this table on update/insert. + functions may be created with + :meth:`~ckanext.datastore.logic.action.datastore_function_create`. + eg: [ + {"function": "trigger_clean_reference"}, + {"function": "trigger_check_codes"}] + :type triggers: list of dictionaries Please note that setting the ``aliases``, ``indexes`` or ``primary_key`` replaces the exising aliases or constraints. Setting ``records`` appends the provided records to the resource. @@ -122,8 +129,6 @@ def datastore_create(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = config['ckan.datastore.write_url'] - # validate aliases aliases = datastore_helpers.get_list(data_dict.get('aliases', [])) for alias in aliases: @@ -153,7 +158,6 @@ def datastore_create(context, data_dict): result.pop('id', None) result.pop('private', None) - result.pop('connection_url') return result @@ -207,12 +211,10 @@ def datastore_upsert(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = config['ckan.datastore.write_url'] - res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + results = db.get_write_engine().execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: @@ -222,7 +224,6 @@ def datastore_upsert(context, data_dict): result = db.upsert(context, data_dict) result.pop('id', None) - result.pop('connection_url') return result @@ -249,11 +250,9 @@ def _type_lookup(t): resource_id = _get_or_bust(data_dict, 'id') resource = p.toolkit.get_action('resource_show')(context, {'id':resource_id}) - data_dict['connection_url'] = config['ckan.datastore.read_url'] - resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=resource_id) + results = db.get_read_engine().execute(resources_sql, id=resource_id) res_exists = results.rowcount > 0 if not res_exists: raise p.toolkit.ObjectNotFound(p.toolkit._( @@ -269,7 +268,7 @@ def _type_lookup(t): SELECT column_name, data_type FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = :resource_id; ''') - schema_results = db._get_engine(data_dict).execute(schema_sql, resource_id=resource_id) + schema_results = db.get_read_engine().execute(schema_sql, resource_id=resource_id) for row in schema_results.fetchall(): k = row[0] v = row[1] @@ -282,7 +281,7 @@ def _type_lookup(t): meta_sql = sqlalchemy.text(u''' SELECT count(_id) FROM "{0}"; '''.format(resource_id)) - meta_results = db._get_engine(data_dict).execute(meta_sql, resource_id=resource_id) + meta_results = db.get_read_engine().execute(meta_sql, resource_id=resource_id) info['meta']['count'] = meta_results.fetchone()[0] finally: if schema_results: @@ -334,12 +333,10 @@ def datastore_delete(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = config['ckan.datastore.write_url'] - res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + results = db.get_read_engine().execute(resources_sql, id=res_id) res_exists = results.rowcount > 0 if not res_exists: @@ -361,7 +358,6 @@ def datastore_delete(context, data_dict): set_datastore_active_flag(model, data_dict, False) result.pop('id', None) - result.pop('connection_url') return result @@ -431,11 +427,13 @@ def datastore_search(context, data_dict): raise p.toolkit.ValidationError(errors) res_id = data_dict['resource_id'] - data_dict['connection_url'] = config['ckan.datastore.write_url'] resources_sql = sqlalchemy.text(u'''SELECT alias_of FROM "_table_metadata" WHERE name = :id''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + # XXX: write connection because of private tables, we + # should be able to make this read once we stop using pg + # permissions enforcement + results = db.get_write_engine().execute(resources_sql, id=res_id) # Resource only has to exist in the datastore (because it could be an alias) if not results.rowcount > 0: @@ -453,7 +451,6 @@ def datastore_search(context, data_dict): result = db.search(context, data_dict) result.pop('id', None) - result.pop('connection_url') return result @@ -495,11 +492,8 @@ def datastore_search_sql(context, data_dict): p.toolkit.check_access('datastore_search_sql', context, data_dict) - data_dict['connection_url'] = config['ckan.datastore.read_url'] - result = db.search_sql(context, data_dict) result.pop('id', None) - result.pop('connection_url') return result @@ -518,8 +512,6 @@ def datastore_make_private(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - data_dict['connection_url'] = config['ckan.datastore.write_url'] - if not _resource_exists(context, data_dict): raise p.toolkit.ObjectNotFound(p.toolkit._( u'Resource "{0}" was not found.'.format(res_id) @@ -544,8 +536,6 @@ def datastore_make_public(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - data_dict['connection_url'] = config['ckan.datastore.write_url'] - if not _resource_exists(context, data_dict): raise p.toolkit.ObjectNotFound(p.toolkit._( u'Resource "{0}" was not found.'.format(res_id) @@ -615,7 +605,7 @@ def _resource_exists(context, data_dict): resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') - results = db._get_engine(data_dict).execute(resources_sql, id=res_id) + results = db.get_read_engine().execute(resources_sql, id=res_id) return results.rowcount > 0 @@ -630,3 +620,42 @@ def _check_read_only(context, resource_id): 'read-only': ['Cannot edit read-only resource. Either pass' '"force=True" or change url-type to "datastore"'] }) + + +@logic.validate(dsschema.datastore_function_create_schema) +def datastore_function_create(context, data_dict): + u''' + Create a trigger function for use with datastore_create + + :param name: function name + :type name: string + :param or_replace: True to replace if function already exists + (default: False) + :type or_replace: bool + :param rettype: set to 'trigger' + (only trigger functions may be created at this time) + :type rettype: string + :param definition: PL/pgSQL function body for trigger function + :type definition: string + ''' + p.toolkit.check_access('datastore_function_create', context, data_dict) + + db.create_function( + name=data_dict['name'], + arguments=data_dict.get('arguments', []), + rettype=data_dict['rettype'], + definition=data_dict['definition'], + or_replace=data_dict['or_replace']) + + +@logic.validate(dsschema.datastore_function_delete_schema) +def datastore_function_delete(context, data_dict): + u''' + Delete a trigger function + + :param name: function name + :type name: string + ''' + p.toolkit.check_access('datastore_function_delete', context, data_dict) + + db.drop_function(data_dict['name'], data_dict['if_exists']) diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py index fc54f82bd5e..824a73ee46b 100644 --- a/ckanext/datastore/logic/auth.py +++ b/ckanext/datastore/logic/auth.py @@ -58,3 +58,12 @@ def datastore_search_sql(context, data_dict): def datastore_change_permissions(context, data_dict): return datastore_auth(context, data_dict) + + +def datastore_function_create(context, data_dict): + '''sysadmin-only: functions can be used to skip access checks''' + return {'success': False} + + +def datastore_function_delete(context, data_dict): + return {'success': False} diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index d2eecc4caf9..e2072ea11c8 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -16,6 +16,7 @@ boolean_validator = get_validator('boolean_validator') int_validator = get_validator('int_validator') OneOf = get_validator('OneOf') +unicode_only = get_validator('unicode_only') default = get_validator('default') @@ -107,6 +108,17 @@ def datastore_create_schema(): }, 'primary_key': [ignore_missing, list_of_strings_or_string], 'indexes': [ignore_missing, list_of_strings_or_string], + 'triggers': { + 'when': [ + default(u'before insert or update'), + unicode_only, + OneOf([u'before insert or update'])], + 'for_each': [ + default(u'row'), + unicode_only, + OneOf([u'row'])], + 'function': [not_empty, unicode_only], + }, '__junk': [empty], '__before': [rename('id', 'resource_id')] } @@ -155,3 +167,24 @@ def datastore_search_schema(): '__before': [rename('id', 'resource_id')] } return schema + + +def datastore_function_create_schema(): + return { + 'name': [unicode_only, not_empty], + 'or_replace': [default(False), boolean_validator], + # we're only exposing functions for triggers at the moment + 'arguments': { + 'argname': [unicode_only, not_empty], + 'argtype': [unicode_only, not_empty], + }, + 'rettype': [default(u'void'), unicode_only], + 'definition': [unicode_only], + } + + +def datastore_function_delete_schema(): + return { + 'name': [unicode_only, not_empty], + 'if_exists': [default(False), boolean_validator], + } diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index dc35bd96f28..33155a76f0b 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -34,9 +34,7 @@ def _is_legacy_mode(config): Returns True if `ckan.datastore.read_url` is not set in the provided config object or CKAN is running on Postgres < 9.x ''' - write_url = config.get('ckan.datastore.write_url') - - engine = db._get_engine({'connection_url': write_url}) + engine = db.get_write_engine() connection = engine.connect() return (not config.get('ckan.datastore.read_url') or @@ -112,8 +110,7 @@ def configure(self, config): else: self.read_url = self.config['ckan.datastore.read_url'] - self.read_engine = db._get_engine( - {'connection_url': self.read_url}) + self.read_engine = db.get_read_engine() if not model.engine_is_pg(self.read_engine): log.warn('We detected that you do not use a PostgreSQL ' 'database. The DataStore will NOT work and DataStore ' @@ -140,7 +137,6 @@ def notify(self, entity, operation=None): for resource in entity.resources: try: func(context, { - 'connection_url': self.write_url, 'resource_id': resource.id}) except p.toolkit.ObjectNotFound: pass @@ -174,7 +170,7 @@ def _is_read_only_database(self): ''' Returns True if no connection has CREATE privileges on the public schema. This is the case if replication is enabled.''' for url in [self.ckan_url, self.write_url, self.read_url]: - connection = db._get_engine({'connection_url': url}).connect() + connection = db._get_engine_from_url(url).connect() try: sql = u"SELECT has_schema_privilege('public', 'CREATE')" is_writable = connection.execute(sql).first()[0] @@ -200,8 +196,7 @@ def _read_connection_has_correct_privileges(self): only user. A table is created by the write user to test the read only user. ''' - write_connection = db._get_engine( - {'connection_url': self.write_url}).connect() + write_connection = db.get_write_engine().connect() read_connection_user = sa_url.make_url(self.read_url).username drop_foo_sql = u'DROP TABLE IF EXISTS _foo' @@ -222,12 +217,15 @@ def _read_connection_has_correct_privileges(self): return True def get_actions(self): - actions = {'datastore_create': action.datastore_create, - 'datastore_upsert': action.datastore_upsert, - 'datastore_delete': action.datastore_delete, - 'datastore_search': action.datastore_search, - 'datastore_info': action.datastore_info, - } + actions = { + 'datastore_create': action.datastore_create, + 'datastore_upsert': action.datastore_upsert, + 'datastore_delete': action.datastore_delete, + 'datastore_search': action.datastore_search, + 'datastore_info': action.datastore_info, + 'datastore_function_create': action.datastore_function_create, + 'datastore_function_delete': action.datastore_function_delete, + } if not self.legacy_mode: if self.enable_sql_search: # Only enable search_sql if the config does not disable it @@ -239,13 +237,17 @@ def get_actions(self): return actions def get_auth_functions(self): - return {'datastore_create': auth.datastore_create, - 'datastore_upsert': auth.datastore_upsert, - 'datastore_delete': auth.datastore_delete, - 'datastore_info': auth.datastore_info, - 'datastore_search': auth.datastore_search, - 'datastore_search_sql': auth.datastore_search_sql, - 'datastore_change_permissions': auth.datastore_change_permissions} + return { + 'datastore_create': auth.datastore_create, + 'datastore_upsert': auth.datastore_upsert, + 'datastore_delete': auth.datastore_delete, + 'datastore_info': auth.datastore_info, + 'datastore_search': auth.datastore_search, + 'datastore_search_sql': auth.datastore_search_sql, + 'datastore_change_permissions': auth.datastore_change_permissions, + 'datastore_function_create': auth.datastore_function_create, + 'datastore_function_delete': auth.datastore_function_delete, + } def before_map(self, m): m.connect( diff --git a/ckanext/datastore/tests/helpers.py b/ckanext/datastore/tests/helpers.py index 7e707a0e75e..277253b82a6 100644 --- a/ckanext/datastore/tests/helpers.py +++ b/ckanext/datastore/tests/helpers.py @@ -1,9 +1,13 @@ # encoding: utf-8 +from sqlalchemy import orm + import ckan.model as model import ckan.lib.cli as cli import ckan.plugins as p +from ckan.tests.helpers import FunctionalTestBase +from ckanext.datastore import db def extract(d, keys): @@ -17,6 +21,17 @@ def clear_db(Session): results = c.execute(drop_tables) for result in results: c.execute(result[0]) + + drop_functions_sql = u''' + SELECT 'drop function ' || quote_ident(proname) || '();' + FROM pg_proc + INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid) + WHERE ns.nspname = 'public' + ''' + drop_functions = u''.join(r[0] for r in c.execute(drop_functions_sql)) + if drop_functions: + c.execute(drop_functions) + Session.commit() Session.remove() @@ -42,3 +57,13 @@ def set_url_type(resources, user): context, {'id': resource.id}) resource['url_type'] = 'datastore' p.toolkit.get_action('resource_update')(context, resource) + + +class DatastoreFunctionalTestBase(FunctionalTestBase): + _load_plugins = (u'datastore', ) + + @classmethod + def setup_class(cls): + engine = db.get_write_engine() + rebuild_all_dbs(orm.scoped_session(orm.sessionmaker(bind=engine))) + super(DatastoreFunctionalTestBase, cls).setup_class() diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index a95bcf68ef8..ba8c94adf08 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -18,7 +18,9 @@ import ckan.tests.factories as factories import ckanext.datastore.db as db -from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type +from ckanext.datastore.tests.helpers import ( + rebuild_all_dbs, set_url_type, DatastoreFunctionalTestBase) +from ckan.plugins.toolkit import ValidationError class TestDatastoreCreateNewTests(object): @@ -193,8 +195,7 @@ def _get_index_names(self, resource_id): return [result[0] for result in results] def _execute_sql(self, sql, *args): - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() session = orm.scoped_session(orm.sessionmaker(bind=engine)) return session.connection().execute(sql, *args) @@ -255,8 +256,7 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) @@ -907,3 +907,201 @@ def test_datastore_create_with_invalid_data_value(self): assert res_dict['success'] is False assert res_dict['error']['__type'] == 'Validation Error' assert res_dict['error']['message'].startswith('The data was invalid') + + +class TestDatastoreFunctionCreate(DatastoreFunctionalTestBase): + def test_nop_trigger(self): + helpers.call_action( + u'datastore_function_create', + name=u'test_nop', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + + def test_invalid_definition(self): + try: + helpers.call_action( + u'datastore_function_create', + name=u'test_invalid_def', + rettype=u'trigger', + definition=u'HELLO WORLD') + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'definition': + [u'syntax error at or near "HELLO"']}) + else: + assert 0, u'no validation error' + + def test_redefined_trigger(self): + helpers.call_action( + u'datastore_function_create', + name=u'test_redefined', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + try: + helpers.call_action( + u'datastore_function_create', + name=u'test_redefined', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'name':[ + u'function "test_redefined" already exists ' + u'with same argument types']}) + else: + assert 0, u'no validation error' + + def test_redefined_with_or_replace_trigger(self): + helpers.call_action( + u'datastore_function_create', + name=u'test_replaceme', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + helpers.call_action( + u'datastore_function_create', + name=u'test_replaceme', + or_replace=True, + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + + +class TestDatastoreCreateTriggers(DatastoreFunctionalTestBase): + def test_create_with_missing_trigger(self): + ds = factories.Dataset() + + try: + helpers.call_action( + u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{u'id': u'spam', u'type': u'text'}], + records=[{u'spam': u'SPAM'}, {u'spam': u'EGGS'}], + triggers=[{u'function': u'no_such_trigger_function'}]) + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'triggers':[ + u'function no_such_trigger_function() does not exist']}) + else: + assert 0, u'no validation error' + + def test_create_trigger_applies_to_records(self): + ds = factories.Dataset() + + helpers.call_action( + u'datastore_function_create', + name=u'spamify_trigger', + rettype=u'trigger', + definition=u''' + BEGIN + NEW.spam := 'spam spam ' || NEW.spam || ' spam'; + RETURN NEW; + END;''') + res = helpers.call_action( + u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{u'id': u'spam', u'type': u'text'}], + records=[{u'spam': u'SPAM'}, {u'spam': u'EGGS'}], + triggers=[{u'function': u'spamify_trigger'}]) + assert_equal( + helpers.call_action( + u'datastore_search', + fields=[u'spam'], + resource_id=res['resource_id'])['records'], + [ + {u'spam': u'spam spam SPAM spam'}, + {u'spam': u'spam spam EGGS spam'}]) + + def test_upsert_trigger_applies_to_records(self): + ds = factories.Dataset() + + helpers.call_action( + u'datastore_function_create', + name=u'more_spam_trigger', + rettype=u'trigger', + definition=u''' + BEGIN + NEW.spam := 'spam spam ' || NEW.spam || ' spam'; + RETURN NEW; + END;''') + res = helpers.call_action( + u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{u'id': u'spam', u'type': u'text'}], + triggers=[{u'function': u'more_spam_trigger'}]) + helpers.call_action( + u'datastore_upsert', + method=u'insert', + resource_id=res['resource_id'], + records=[{u'spam': u'BEANS'}, {u'spam': u'SPAM'}]) + assert_equal( + helpers.call_action( + u'datastore_search', + fields=[u'spam'], + resource_id=res['resource_id'])['records'], + [ + {u'spam': u'spam spam BEANS spam'}, + {u'spam': u'spam spam SPAM spam'}]) + + def test_create_trigger_exception(self): + ds = factories.Dataset() + + helpers.call_action( + u'datastore_function_create', + name=u'spamexception_trigger', + rettype=u'trigger', + definition=u''' + BEGIN + IF NEW.spam != 'spam' THEN + RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam; + END IF; + RETURN NEW; + END;''') + try: + res = helpers.call_action( + u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{u'id': u'spam', u'type': u'text'}], + records=[{u'spam': u'spam'}, {u'spam': u'EGGS'}], + triggers=[{u'function': u'spamexception_trigger'}]) + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'records':[ + u'"EGGS"? Yeeeeccch!']}) + else: + assert 0, u'no validation error' + + def test_upsert_trigger_exception(self): + ds = factories.Dataset() + + helpers.call_action( + u'datastore_function_create', + name=u'spamonly_trigger', + rettype=u'trigger', + definition=u''' + BEGIN + IF NEW.spam != 'spam' THEN + RAISE EXCEPTION '"%"? Yeeeeccch!', NEW.spam; + END IF; + RETURN NEW; + END;''') + res = helpers.call_action( + u'datastore_create', + resource={u'package_id': ds['id']}, + fields=[{u'id': u'spam', u'type': u'text'}], + triggers=[{u'function': u'spamonly_trigger'}]) + try: + helpers.call_action( + u'datastore_upsert', + method=u'insert', + resource_id=res['resource_id'], + records=[{u'spam': u'spam'}, {u'spam': u'BEANS'}]) + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'records':[ + u'"BEANS"? Yeeeeccch!']}) + else: + assert 0, u'no validation error' diff --git a/ckanext/datastore/tests/test_delete.py b/ckanext/datastore/tests/test_delete.py index f5c14fd4944..4c806808358 100644 --- a/ckanext/datastore/tests/test_delete.py +++ b/ckanext/datastore/tests/test_delete.py @@ -2,6 +2,7 @@ import json import nose +from nose.tools import assert_equal import sqlalchemy import sqlalchemy.orm as orm @@ -10,13 +11,16 @@ import ckan.lib.create_test_data as ctd import ckan.model as model import ckan.tests.legacy as tests +from ckan.tests import helpers from ckan.common import config +from ckan.plugins.toolkit import ValidationError import ckan.tests.factories as factories import ckan.tests.helpers as helpers from ckan.logic import NotFound import ckanext.datastore.db as db -from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type +from ckanext.datastore.tests.helpers import ( + rebuild_all_dbs, set_url_type, DatastoreFunctionalTestBase) assert_raises = nose.tools.assert_raises @@ -47,8 +51,7 @@ def setup_class(cls): 'rating with %': '42%'}] } - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) @@ -259,3 +262,33 @@ def test_delete_with_blank_filters(self): assert(len(results['result']['records']) == 0) self._delete() + + +class TestDatastoreFunctionDelete(DatastoreFunctionalTestBase): + def test_create_delete(self): + helpers.call_action( + u'datastore_function_create', + name=u'test_nop', + rettype=u'trigger', + definition=u'BEGIN RETURN NEW; END;') + helpers.call_action( + u'datastore_function_delete', + name=u'test_nop') + + def test_delete_nonexistant(self): + try: + helpers.call_action( + u'datastore_function_delete', + name=u'test_not_there') + except ValidationError as ve: + assert_equal( + ve.error_dict, + {u'name': [u'function test_not_there() does not exist']}) + else: + assert 0, u'no validation error' + + def test_delete_if_exitst(self): + helpers.call_action( + u'datastore_function_delete', + name=u'test_not_there_either', + if_exists=True) diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index c460db873de..5db9ac33fd5 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -90,8 +90,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine({ - 'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod diff --git a/ckanext/datastore/tests/test_helpers.py b/ckanext/datastore/tests/test_helpers.py index 9cc05c3dded..eed67000c20 100644 --- a/ckanext/datastore/tests/test_helpers.py +++ b/ckanext/datastore/tests/test_helpers.py @@ -70,9 +70,7 @@ def setup_class(cls): if not config.get('ckan.datastore.read_url'): raise nose.SkipTest('Datastore runs on legacy mode, skipping...') - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']} - ) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) datastore_test_helpers.clear_db(cls.Session) diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py index c472b52bdc3..f1b0fe64c48 100644 --- a/ckanext/datastore/tests/test_search.py +++ b/ckanext/datastore/tests/test_search.py @@ -165,9 +165,7 @@ def setup_class(cls): u'characters': None, u'rating with %': u'99%'}] - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']} - ) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -838,8 +836,7 @@ def setup_class(cls): u'published': None}] cls.expected_join_results = [{u'first': 1, u'second': 1}, {u'first': 1, u'second': 2}] - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -941,8 +938,7 @@ def test_read_private(self): 'user': self.sysadmin_user.name, 'model': model} data_dict = { - 'resource_id': self.data['resource_id'], - 'connection_url': config['ckan.datastore.write_url']} + 'resource_id': self.data['resource_id']} p.toolkit.get_action('datastore_make_private')(context, data_dict) query = 'SELECT * FROM "{0}"'.format(self.data['resource_id']) data = {'sql': query} diff --git a/ckanext/datastore/tests/test_unit.py b/ckanext/datastore/tests/test_unit.py index c02d30c60d6..2a74e3386cd 100644 --- a/ckanext/datastore/tests/test_unit.py +++ b/ckanext/datastore/tests/test_unit.py @@ -35,8 +35,7 @@ def test_is_valid_table_name(self): def test_pg_version_check(self): if not tests.is_datastore_supported(): raise nose.SkipTest("Datastore not supported") - engine = db._get_engine( - {'connection_url': config['sqlalchemy.url']}) + engine = db._get_engine_from_url(config['sqlalchemy.url']) connection = engine.connect() assert db._pg_version_is_at_least(connection, '8.0') assert not db._pg_version_is_at_least(connection, '10.0') diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py index b7a09bf7dae..e798bec38dc 100644 --- a/ckanext/datastore/tests/test_upsert.py +++ b/ckanext/datastore/tests/test_upsert.py @@ -113,8 +113,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -365,8 +364,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -472,8 +470,7 @@ def setup_class(cls): res_dict = json.loads(res.body) assert res_dict['success'] is True - engine = db._get_engine( - {'connection_url': config['ckan.datastore.write_url']}) + engine = db.get_write_engine() cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 78ca3a948fd..51ffd3bbc56 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -12,17 +12,11 @@ ckan: db: container_name: db image: ckan/postgresql:latest - ports: - - "5432:5432" solr: container_name: solr image: ckan/solr:latest - ports: - - "8983:8983" redis: container_name: redis image: redis:latest - ports: - - "6379:6379" diff --git a/doc/conf.py b/doc/conf.py index 3ed57700fbb..7a5692d42b8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -52,7 +52,7 @@ .. |storage_path| replace:: |storage_parent_dir|/default .. |reload_apache| replace:: sudo service apache2 reload .. |restart_apache| replace:: sudo service apache2 restart -.. |restart_solr| replace:: sudo service jetty restart +.. |restart_solr| replace:: sudo service jetty8 restart .. |solr| replace:: Solr .. |restructuredtext| replace:: reStructuredText .. |nginx| replace:: Nginx diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index b1b1790d314..9fd0f995d22 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -245,6 +245,10 @@ Data can be written incrementally to the DataStore through the API. New data can inserted, existing data can be updated or deleted. You can also add a new column to an existing table even if the DataStore resource already contains some data. +Triggers may be added to enforce validation, clean data as it is loaded or +even record record histories. Triggers are PL/pgSQL functions that must be +created by a sysadmin. + You will notice that we tried to keep the layer between the underlying PostgreSQL database and the API as thin as possible to allow you to use the features you would expect from a powerful database management system. diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index c022946a9e7..e8c93a3e133 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -5,10 +5,10 @@ Installing CKAN from source =========================== This section describes how to install CKAN from source. Although -:doc:`install-from-package` is simpler, it requires Ubuntu 14.04 64-bit or -Ubuntu 12.04 64-bit. Installing CKAN from source works with other versions of -Ubuntu and with other operating systems (e.g. RedHat, Fedora, CentOS, OS X). -If you install CKAN from source on your own operating system, please share your +:doc:`install-from-package` is simpler, it requires Ubuntu 16.04 64-bit or +Ubuntu 14.04 64-bit. Installing CKAN from source works with other versions of +Ubuntu and with other operating systems (e.g. RedHat, Fedora, CentOS, OS X). If +you install CKAN from source on your own operating system, please share your experiences on our `How to Install CKAN `_ wiki page. @@ -21,7 +21,11 @@ work on CKAN. -------------------------------- If you're using a Debian-based operating system (such as Ubuntu) install the -required packages with this command:: +required packages with this command for Ubuntu 16.04:: + + sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-8-jdk redis-server + +or for Ubuntu 14.04:: sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-6-jdk redis-server @@ -41,7 +45,7 @@ virtualenv `The virtual Python environment builder `_ Apache Solr `A search platform `_ Jetty `An HTTP server `_ (used for Solr). -OpenJDK 6 JDK `The Java Development Kit `_ +OpenJDK JDK `The Java Development Kit `_ (used by Jetty) Redis `An in-memory data structure store `_ ===================== =============================================== @@ -94,7 +98,13 @@ a. Create a Python `virtual environment `_ |activate| -b. Install the CKAN source code into your virtualenv. +b. Install the recommended version of 'setuptools': + + .. parsed-literal:: + + pip install -r |virtualenv|/src/ckan/requirement-setuptools.txt + +c. Install the CKAN source code into your virtualenv. To install the latest stable release of CKAN (CKAN |latest_release_version|), run: @@ -116,7 +126,7 @@ b. Install the CKAN source code into your virtualenv. production websites! Only install this version if you're doing CKAN development. -c. Install the Python modules that CKAN requires into your virtualenv: +d. Install the Python modules that CKAN requires into your virtualenv: .. versionchanged:: 2.1 In CKAN 2.0 and earlier the requirement file was called @@ -126,7 +136,7 @@ c. Install the Python modules that CKAN requires into your virtualenv: pip install -r |virtualenv|/src/ckan/requirements.txt -d. Deactivate and reactivate your virtualenv, to make sure you're using the +e. Deactivate and reactivate your virtualenv, to make sure you're using the virtualenv's copies of commands like ``paster`` rather than any system-wide installed copies: @@ -321,9 +331,13 @@ If ``javac`` isn't installed, do:: and then restart Solr: -.. parsed-literal:: +For Ubuntu 16.04:: - |restart_solr| + sudo service jetty8 restart + +or for Ubuntu 14.04:: + + sudo service jetty restart AttributeError: 'module' object has no attribute 'css/main.debug.css' --------------------------------------------------------------------- @@ -331,11 +345,22 @@ AttributeError: 'module' object has no attribute 'css/main.debug.css' This error is likely to show up when `debug` is set to `True`. To fix this error, install frontend dependencies. See :doc:`/contributing/frontend/index`. -After installing the dependencies, run `bin/less` and then start paster server +After installing the dependencies, run ``bin/less`` and then start paster server again. If you do not want to compile CSS, you can also copy the main.css to -main.debug.css to get CKAN running. +main.debug.css to get CKAN running:: cp /usr/lib/ckan/default/src/ckan/ckan/public/base/css/main.css \ /usr/lib/ckan/default/src/ckan/ckan/public/base/css/main.debug.css + +JSP support not configured +-------------------------- + +This is seen occasionally with Jetty and Ubuntu 14.04. It requires a solr-jetty fix:: + + cd /tmp + wget https://launchpad.net/~vshn/+archive/ubuntu/solr/+files/solr-jetty-jsp-fix_1.0.2_all.deb + sudo dpkg -i solr-jetty-jsp-fix_1.0.2_all.deb + sudo service jetty restart + diff --git a/doc/maintaining/installing/solr.rst b/doc/maintaining/installing/solr.rst index 2bd057537c1..5da3851ce85 100644 --- a/doc/maintaining/installing/solr.rst +++ b/doc/maintaining/installing/solr.rst @@ -12,8 +12,8 @@ installed, we need to install and configure Solr. server, but CKAN doesn't require Jetty - you can deploy Solr to another web server, such as Tomcat, if that's convenient on your operating system. -#. Edit the Jetty configuration file (``/etc/default/jetty``) and change the - following variables:: +#. Edit the Jetty configuration file (``/etc/default/jetty8`` or + ``/etc/default/jetty``) and change the following variables:: NO_START=0 # (line 4) JETTY_HOST=127.0.0.1 # (line 16) @@ -26,9 +26,20 @@ installed, we need to install and configure Solr. change it to the relevant host or to 0.0.0.0 (and probably set up your firewall accordingly). - Start the Jetty server:: + Start or restart the Jetty server. - sudo service jetty start + For Ubuntu 16.04:: + + sudo service jetty8 restart + + Or for Ubuntu 14.04:: + + sudo service jetty restart + + .. note:: + + Ignore any warning that it wasn't already running - some Ubuntu + distributions choose not to start Jetty on install, but it's not important. You should now see a welcome page from Solr if you open http://localhost:8983/solr/ in your web browser (replace localhost with @@ -57,11 +68,15 @@ installed, we need to install and configure Solr. Now restart Solr: - .. parsed-literal:: + For Ubuntu 16.04:: + + sudo service jetty8 restart + + or for Ubuntu 14.04:: - |restart_solr| + sudo service jetty restart - and check that Solr is running by opening http://localhost:8983/solr/. + Check that Solr is running by opening http://localhost:8983/solr/. #. Finally, change the :ref:`solr_url` setting in your :ref:`config_file` (|production.ini|) to diff --git a/requirement-setuptools.txt b/requirement-setuptools.txt new file mode 100644 index 00000000000..44e41169c85 --- /dev/null +++ b/requirement-setuptools.txt @@ -0,0 +1 @@ +setuptools==20.4 \ No newline at end of file diff --git a/setup.py b/setup.py index 273b6d528f7..a973a71eca7 100644 --- a/setup.py +++ b/setup.py @@ -6,15 +6,31 @@ del os.link try: - from setuptools import setup, find_packages + from setuptools import (setup, find_packages, + __version__ as setuptools_version) except ImportError: from ez_setup import use_setuptools use_setuptools() - from setuptools import setup, find_packages + from setuptools import (setup, find_packages, + __version__ as setuptools_version) from ckan import (__version__, __description__, __long_description__, __license__) +MIN_SETUPTOOLS_VERSION = 20.4 +assert setuptools_version >= str(MIN_SETUPTOOLS_VERSION) and \ + int(setuptools_version.split('.')[0]) >= int(MIN_SETUPTOOLS_VERSION),\ + ('setuptools version error' + '\nYou need a newer version of setuptools.\n' + 'You have {current}, you need at least {minimum}' + '\nInstall the recommended version:\n' + ' pip install -r requirement-setuptools.txt\n' + 'and then try again to install ckan into your python environment.'.format( + current=setuptools_version, + minimum=MIN_SETUPTOOLS_VERSION + )) + + entry_points = { 'nose.plugins.0.10': [ 'main = ckan.ckan_nose_plugin:CkanNose',