From 1eb9d94077bd7d0dc586fefcb0d6b49042606b6c Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 19 Jun 2013 15:27:33 +0200 Subject: [PATCH 001/269] [#1011] Fix a 500 to a 404 Catch Genshi TemplateNotFound errors when rendering the package read page. This can happen if an IDatasetForm plugin returns a custom package read template, e.g. 'read.html', but the user has requested the dataset in RDF format and the plugin does not provide a corresponding 'read.rdf' template. (CKAN will take the path to read.html, replace the filename extension with rdf, try to render this non-existing template file, and crash.) Replace this crash with a proper 404. Add to the IDatasetForm docs, explaining how to provide RDF templates for datasets. Fixes #1011. --- ckan/controllers/package.py | 11 ++++++++++- ckan/plugins/interfaces.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 65541ee33bf..693ceb851a2 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -5,6 +5,7 @@ from pylons import config from genshi.template import MarkupTemplate from genshi.template.text import NewTextTemplate +import genshi.template.loader from paste.deploy.converters import asbool import ckan.logic as logic @@ -352,7 +353,15 @@ def read(self, id, format='html'): template = self._read_template(package_type) template = template[:template.index('.') + 1] + format - return render(template, loader_class=loader) + try: + return render(template, loader_class=loader) + except genshi.template.loader.TemplateNotFound: + msg = _("Viewing {package_type} datasets in {format} format is " + "not supported (template file {file} not found).".format( + package_type=package_type, format=format, file=template)) + abort(404, msg) + + assert False, "We should never get here" def history(self, id): package_type = self._get_package_type(id.split('@')[0]) diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index a37d3c96ae8..826fec921cd 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -676,6 +676,15 @@ def read_template(self): The path should be relative to the plugin's templates dir, e.g. ``'package/read.html'``. + If the user requests the dataset in a format other than HTML + (CKAN supports returning datasets in RDF or N3 format by appending .rdf + or .n3 to the dataset read URL, see :doc:`linked-data-and-rdf`) then + CKAN will try to render + a template file with the same path as returned by this function, + but a different filename extension, e.g. ``'package/read.rdf'``. + If your extension doesn't have this RDF version of the template + file, the user will get a 404 error. + :rtype: string ''' From 25c80f16f57ecc34fda2d52bbd8d3a41e0b33d66 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Sat, 6 Jul 2013 12:05:59 +0200 Subject: [PATCH 002/269] Add coverage reports with coveralls --- .coveragerc | 3 +++ .travis.yml | 4 ++++ bin/travis-build | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000000..28f34e39273 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = /ckan/migration/* +source = ckan, ckanext diff --git a/.travis.yml b/.travis.yml index af450298b6c..b4f63b6c669 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,8 @@ python: env: - PGVERSION=9.1 - PGVERSION=8.4 +install: + - pip install coveralls --use-mirrors script: ./bin/travis-build notifications: irc: @@ -14,3 +16,5 @@ notifications: on_failure: change template: - "%{repository} %{branch} %{commit} %{build_url} %{author}: %{message}" +after_success: + - coveralls diff --git a/bin/travis-build b/bin/travis-build index df7c398d1b6..de7599cdddd 100755 --- a/bin/travis-build +++ b/bin/travis-build @@ -48,4 +48,4 @@ fi cat test-core.ini # And finally, run the tests -nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext +nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext --with-coverage --cover-package=ckanext --cover-package=ckan From 73308db08242a6a2ee72079b6cc87ce4b0705761 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 17 Jul 2013 15:49:13 +0200 Subject: [PATCH 003/269] [#1117] Add new_tests dir with first new logic tests Eventually (before this branch is merged into master) new_tests should become tests and the current tests should become legacy_tests. --- ckan/new_tests/__init__.py | 0 ckan/new_tests/data.py | 7 ++ ckan/new_tests/helpers.py | 49 ++++++++++++++ ckan/new_tests/logic/__init__.py | 0 ckan/new_tests/logic/action/__init__.py | 0 ckan/new_tests/logic/action/test_update.py | 79 ++++++++++++++++++++++ 6 files changed, 135 insertions(+) create mode 100644 ckan/new_tests/__init__.py create mode 100644 ckan/new_tests/data.py create mode 100644 ckan/new_tests/helpers.py create mode 100644 ckan/new_tests/logic/__init__.py create mode 100644 ckan/new_tests/logic/action/__init__.py create mode 100644 ckan/new_tests/logic/action/test_update.py diff --git a/ckan/new_tests/__init__.py b/ckan/new_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckan/new_tests/data.py b/ckan/new_tests/data.py new file mode 100644 index 00000000000..d5d76a8f800 --- /dev/null +++ b/ckan/new_tests/data.py @@ -0,0 +1,7 @@ +'''A collection of static data for use in tests. + +''' +TYPICAL_USER = {'name': 'fred', + 'email': 'fred@fred.com', + 'password': 'wilma', + } diff --git a/ckan/new_tests/helpers.py b/ckan/new_tests/helpers.py new file mode 100644 index 00000000000..7ad0bd00781 --- /dev/null +++ b/ckan/new_tests/helpers.py @@ -0,0 +1,49 @@ +'''A collection of test helper functions. + +''' +import ckan.model as model +import ckan.logic as logic + + +def reset_db(): + '''Reset CKAN's database. + + Each test module or class should call this function in its setup method. + + ''' + # Close any database connections that have been left open. + # This prevents CKAN from hanging waiting for some unclosed connection. + model.Session.close_all() + + # Clean out any data from the db. This prevents tests failing due to data + # leftover from other tests or from previous test runs. + model.repo.clean_db() + + # Initialize the db. This prevents CKAN from crashing if the tests are run + # and the test db has not been initialized. + model.repo.init_db() + + +def call_action(action_name, context=None, **kwargs): + '''Call the given ckan.logic.action function with the given context + and params. + + For example: + + call_action('user_create', name='seanh', email='seanh@seanh.com', + password='pass') + + This is just a nicer way for user code to call action functions, nicer than + either calling the action function directly or via get_action(). + + If accepted this function should eventually be moved to + ckan.logic.call_action() and the current get_action() function should be + deprecated. The tests may still need their own wrapper function for + logic.call_action(), e.g. to insert 'ignore_auth': True into the context. + + ''' + if context is None: + context = {} + context.setdefault('user', '127.0.0.1') + context.setdefault('ignore_auth', True) + return logic.get_action(action_name)(context=context, data_dict=kwargs) diff --git a/ckan/new_tests/logic/__init__.py b/ckan/new_tests/logic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckan/new_tests/logic/action/__init__.py b/ckan/new_tests/logic/action/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py new file mode 100644 index 00000000000..6a7ecfd0d8f --- /dev/null +++ b/ckan/new_tests/logic/action/test_update.py @@ -0,0 +1,79 @@ +'''Unit tests for ckan/logic/action/update.py. + +''' +import ckan.new_tests.helpers as helpers +import ckan.new_tests.data as data + + +def setup_module(): + helpers.reset_db() + +# Tests for user_update: +# +# - Test for success: +# - Typical values +# - Edge cases +# - If multiple params, test with different combinations +# - Test for failure: +# - Common mistakes +# - Bizarre input +# - Unicode +# - Cover the interface +# - Can we somehow mock user_create? +# +# Correct and incorrect ID + + +def test_user_update(): + '''Test that updating a user's metadata fields works successfully. + + ''' + # Prepare + user = helpers.call_action('user_create', **data.TYPICAL_USER) + + # Execute + user['name'] = 'updated_name' + user['about'] = 'updated_about' + helpers.call_action('user_update', **user) + + # Assert + updated_user = helpers.call_action('user_show', id=user['id']) + # Note that we check each updated field seperately, we don't compare the + # entire dicts, only assert what we're actually testing. + assert user['name'] == updated_user['name'] + assert user['about'] == updated_user['about'] + + +def test_user_update_with_invalid_id(): + pass + + +def test_user_update_with_nonexistent_id(): + pass + + +def test_user_update_with_no_id(): + pass + + +def test_user_update_with_custom_schema(): + pass + + +def test_user_update_with_invalid_name(): + pass + + +def test_user_update_with_invalid_password(): + pass + + +def test_user_update_with_deferred_commit(): + pass + + +# TODO: Valid and invalid values for other fields. + + +def test_user_update_activity_stream(): + pass From 3491cd695ec0c9758586d219c027165256fd7012 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 17 Jul 2013 19:14:05 +0200 Subject: [PATCH 004/269] [#1117] Add new tests for updating user names and passwords ...and also for what happens when you call user_update without a valid id. --- ckan/new_tests/logic/action/test_update.py | 211 ++++++++++++++------- 1 file changed, 139 insertions(+), 72 deletions(-) diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index 6a7ecfd0d8f..d17a4092541 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -1,79 +1,146 @@ '''Unit tests for ckan/logic/action/update.py. ''' +import nose.tools + +import ckan.logic as logic import ckan.new_tests.helpers as helpers import ckan.new_tests.data as data -def setup_module(): - helpers.reset_db() - -# Tests for user_update: -# -# - Test for success: -# - Typical values -# - Edge cases -# - If multiple params, test with different combinations -# - Test for failure: -# - Common mistakes -# - Bizarre input -# - Unicode -# - Cover the interface -# - Can we somehow mock user_create? -# -# Correct and incorrect ID - - -def test_user_update(): - '''Test that updating a user's metadata fields works successfully. - - ''' - # Prepare - user = helpers.call_action('user_create', **data.TYPICAL_USER) - - # Execute - user['name'] = 'updated_name' - user['about'] = 'updated_about' - helpers.call_action('user_update', **user) - - # Assert - updated_user = helpers.call_action('user_show', id=user['id']) - # Note that we check each updated field seperately, we don't compare the - # entire dicts, only assert what we're actually testing. - assert user['name'] == updated_user['name'] - assert user['about'] == updated_user['about'] - - -def test_user_update_with_invalid_id(): - pass - - -def test_user_update_with_nonexistent_id(): - pass - - -def test_user_update_with_no_id(): - pass - - -def test_user_update_with_custom_schema(): - pass - - -def test_user_update_with_invalid_name(): - pass - - -def test_user_update_with_invalid_password(): - pass - - -def test_user_update_with_deferred_commit(): - pass - - -# TODO: Valid and invalid values for other fields. - - -def test_user_update_activity_stream(): - pass +class TestClass(object): + + @classmethod + def setup_class(cls): + # Initialize the test db (if it isn't already) and clean out any data + # left in it. + helpers.reset_db() + + def setup(self): + import ckan.model as model + # Reset the db before each test method. + model.repo.rebuild_db() + + def test_update_user_name(self): + '''Test that updating a user's name works successfully.''' + + # The canonical form of a test has four steps: + # 1. Setup anything preconditions needed for the test. + # 2. Call the function that's being tested, once only. + # 3. Make assertions about the return value and/or side-effects of + # of the function that's being tested. + # 4. Do absolutely nothing else! + + # 1. Setup. + user = helpers.call_action('user_create', **data.TYPICAL_USER) + + # 2. Call the function that is being tested, once only. + # Note we have to pass the email address and password (in plain text!) + # to user_update even though we're not updating those fields, otherwise + # validation fails. + helpers.call_action('user_update', id=user['name'], + email=user['email'], + password=data.TYPICAL_USER['password'], + name='updated', + ) + + # 3. Make assertions about the return value and/or side-effects. + updated_user = helpers.call_action('user_show', id=user['id']) + # Note that we check just the field we were trying to update, not the + # entire dict, only assert what we're actually testing. + assert updated_user['name'] == 'updated' + + # 4. Do absolutely nothing else! + + def test_user_update_with_id_that_does_not_exist(self): + # FIXME: Does this need to be more realistic? + # - Actually create a user before trying to update it with the wrong + # id? + # - Actually pass other params (name, about..) to user_update? + with nose.tools.assert_raises(logic.NotFound) as context: + helpers.call_action('user_update', + id="there's no user with this id") + # TODO: Could assert the actual error message, not just the exception? + # (Could also do this with many of the tests below.) + + def test_user_update_with_no_id(self): + with nose.tools.assert_raises(logic.ValidationError) as context: + helpers.call_action('user_update') + + def test_user_update_with_invalid_name(self): + user = helpers.call_action('user_create', **data.TYPICAL_USER) + + # FIXME: This actually breaks the canonical test form rule by calling + # the function-under-test multiple times. + # Breaking this up into multiple tests would actually be clearer in + # terms of each test testing one thing and the names of the test + # methods documenting what the test does and how the function is + # supposed to behave. + # BUT the test code would be very repetitive. + invalid_names = ('', 'a', False, 0, -1, 23, 'new', 'edit', 'search', + 'a'*200, 'Hi!', ) + for name in invalid_names: + user['name'] = name + with nose.tools.assert_raises(logic.ValidationError) as context: + helpers.call_action('user_update', **user) + + def test_user_update_to_name_that_already_exists(self): + fred = helpers.call_action('user_create', **data.TYPICAL_USER) + bob = helpers.call_action('user_create', name='bob', + email='bob@bob.com', password='pass') + + # Try to update fred and change his user name to bob, which is already + # bob's user name + fred['name'] = bob['name'] + with nose.tools.assert_raises(logic.ValidationError) as context: + helpers.call_action('user_update', **fred) + + def test_update_user_password(self): + '''Test that updating a user's password works successfully.''' + + user = helpers.call_action('user_create', **data.TYPICAL_USER) + + # Note we have to pass the email address to user_update even though + # we're not updating it, otherwise validation fails. + helpers.call_action('user_update', id=user['name'], + email=user['email'], + password='new password', + ) + + # user_show() never returns the user's password, so we have to access + # the model directly to test it. + import ckan.model as model + updated_user = model.User.get(user['id']) + assert updated_user.validate_password('new password') + + def test_user_update_with_short_password(self): + user = helpers.call_action('user_create', **data.TYPICAL_USER) + + user['password'] = 'xxx' # This password is too short. + with nose.tools.assert_raises(logic.ValidationError) as context: + helpers.call_action('user_update', **user) + + def test_user_update_with_empty_password(self): + user = helpers.call_action('user_create', **data.TYPICAL_USER) + + user['password'] = '' + with nose.tools.assert_raises(logic.ValidationError) as context: + helpers.call_action('user_update', **user) + + def test_user_update_with_null_password(self): + user = helpers.call_action('user_create', **data.TYPICAL_USER) + + user['password'] = None + with nose.tools.assert_raises(logic.ValidationError) as context: + helpers.call_action('user_update', **user) + + # TODO: Valid and invalid values for the rest of the user model's fields. + + def test_user_update_activity_stream(self): + pass + + def test_user_update_with_custom_schema(self): + pass + + def test_user_update_with_deferred_commit(self): + pass From bc3baa951de74a61a96c10c2726b991d7ee88894 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 17 Jul 2013 19:17:01 +0200 Subject: [PATCH 005/269] [#1117] user_update don't crash on non-string name Don't crash if the user tries to update their user name to a value that is not a string (which they can do, using the API) --- ckan/logic/validators.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index f9de157795c..78d37eb9f08 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -276,6 +276,9 @@ def extras_unicode_convert(extras, context): name_match = re.compile('[a-z0-9_\-]*$') def name_validator(val, context): + if not isinstance(val, basestring): + raise Invalid(_('Names must be strings')) + # check basic textual rules if val in ['new', 'edit', 'search']: raise Invalid(_('That name cannot be used')) @@ -478,6 +481,9 @@ def user_name_validator(key, data, errors, context): session = context["session"] user = context.get("user_obj") + if not isinstance(data[key], basestring): + raise Invalid(_('User names must be strings')) + query = session.query(model.User.name).filter_by(name=data[key]) if user: user_id = user.id From f3165ce46cf2c2f16a79013fb23f79bdd821ca90 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 17 Jul 2013 19:17:38 +0200 Subject: [PATCH 006/269] [#1117] user_update: don't crash on non-string password Fix a crash that happens if the user tries to update their password to a value that isn't a string (which they can do, using the API) --- ckan/logic/validators.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 78d37eb9f08..5874f89d81c 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -507,7 +507,11 @@ def user_both_passwords_entered(key, data, errors, context): def user_password_validator(key, data, errors, context): value = data[key] - if not value == '' and not isinstance(value, Missing) and not len(value) >= 4: + if isinstance(value, Missing): + errors[('password',)].append(_('Your password must be 4 characters or longer')) + elif not isinstance(value, basestring): + errors[('password',)].append(_('Passwords must be strings')) + elif len(value) < 4: errors[('password',)].append(_('Your password must be 4 characters or longer')) def user_passwords_match(key, data, errors, context): From 4d2d54a6e3622cdf4526ab59002e19114e7776c0 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 18 Jul 2013 13:03:55 +0200 Subject: [PATCH 007/269] [#1011] Refactor an exception raise and catch Raise and catch a CKAN exception, instead of a Genshi one. --- ckan/controllers/package.py | 4 ++-- ckan/lib/base.py | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 693ceb851a2..d73ab4ee740 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -5,7 +5,6 @@ from pylons import config from genshi.template import MarkupTemplate from genshi.template.text import NewTextTemplate -import genshi.template.loader from paste.deploy.converters import asbool import ckan.logic as logic @@ -20,6 +19,7 @@ import ckan.lib.datapreview as datapreview import ckan.lib.plugins import ckan.plugins as p +import lib.render from ckan.common import OrderedDict, _, json, request, c, g, response from home import CACHE_PARAMETERS @@ -355,7 +355,7 @@ def read(self, id, format='html'): try: return render(template, loader_class=loader) - except genshi.template.loader.TemplateNotFound: + except lib.render.TemplateNotFound: msg = _("Viewing {package_type} datasets in {format} format is " "not supported (template file {file} not found).".format( package_type=package_type, format=format, file=template)) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 7227734ddf4..914b42741ea 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -103,11 +103,7 @@ def render_template(): # we remove it so any bad templates crash and burn del globs['url'] - try: - template_path, template_type = lib.render.template_info(template_name) - except lib.render.TemplateNotFound: - template_type = 'genshi' - template_path = '' + template_path, template_type = lib.render.template_info(template_name) # snippets should not pass the context # but allow for legacy genshi templates From 41a1a727cbecfa8331b45b16bd2570c11b98af07 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 18 Jul 2013 16:01:59 +0200 Subject: [PATCH 008/269] [#1117] Typo --- ckan/new_tests/logic/action/test_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index d17a4092541..8d48190248a 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -25,7 +25,7 @@ def test_update_user_name(self): '''Test that updating a user's name works successfully.''' # The canonical form of a test has four steps: - # 1. Setup anything preconditions needed for the test. + # 1. Setup any preconditions needed for the test. # 2. Call the function that's being tested, once only. # 3. Make assertions about the return value and/or side-effects of # of the function that's being tested. From 6d9f3b0ff9c2b2509cbdfb088fb76a5d161eba7e Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 18 Jul 2013 17:01:56 +0200 Subject: [PATCH 009/269] [#1117] Make various improvements to test_update.py - Add some notes to the top of test_update.py about what I'm trying to do with it, these will end up in the testing guidelines - Make the static test data read-only - Rename some of the test methods so that they're all test_user_update_* - Add test_user_update_activity() test - Other small tweaks --- ckan/new_tests/data.py | 15 ++- ckan/new_tests/logic/action/test_update.py | 122 ++++++++++++++++----- 2 files changed, 103 insertions(+), 34 deletions(-) diff --git a/ckan/new_tests/data.py b/ckan/new_tests/data.py index d5d76a8f800..e7e9f120e1c 100644 --- a/ckan/new_tests/data.py +++ b/ckan/new_tests/data.py @@ -1,7 +1,14 @@ '''A collection of static data for use in tests. +These are functions that return objects because the data is intended to be +read-only: if they were simply module-level objects then one test may modify +eg. a dict or list and then break the tests that follow it. + ''' -TYPICAL_USER = {'name': 'fred', - 'email': 'fred@fred.com', - 'password': 'wilma', - } + + +def typical_user(): + return {'name': 'fred', + 'email': 'fred@fred.com', + 'password': 'wilma', + } diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index 8d48190248a..335100a2c00 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -1,6 +1,44 @@ '''Unit tests for ckan/logic/action/update.py. +Notes: + +- All user_update tests are named test_user_update_* + +- Using the canonical test form from the Pylons guidelines + (i.e. each method tests One Thing and the method name explains what it tests) + +- Not testing auth here, that can be done in ckan.tests.logic.auth.*.py + +- But I am testing validation here, because some action functions do some of + their own validation, not all the validation is done in the schemas + +- The tests for each action function try to cover: + - Testing for success: + - Typical values + - Edge cases + - Multiple parameters in different combinations + - Testing for failure: + - Common mistakes + - Bizarre input + - Unicode + +- Cover the interface of the function (i.e. test all params and features) + +- Not storing anything (e.g. test data) against self in the test class, + instead have each test method call helper functions to get any test data + it needs + + - I think it would be okay to create *read-only* data against self in + setup_class though + +- The tests are not ordered, and each one can be run on its own (db is rebuilt + after each test method) + +- Within reason, keep tests as clear and simple as possible even if it means + they get repetitive + ''' +import datetime import nose.tools import ckan.logic as logic @@ -8,6 +46,15 @@ import ckan.new_tests.data as data +def datetime_from_string(s): + '''Return a standard datetime.datetime object initialised from a string in + the same format used for timestamps in dictized activities (the format + produced by datetime.datetime.isoformat()) + + ''' + return datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S.%f') + + class TestClass(object): @classmethod @@ -21,7 +68,7 @@ def setup(self): # Reset the db before each test method. model.repo.rebuild_db() - def test_update_user_name(self): + def test_user_update_name(self): '''Test that updating a user's name works successfully.''' # The canonical form of a test has four steps: @@ -32,15 +79,15 @@ def test_update_user_name(self): # 4. Do absolutely nothing else! # 1. Setup. - user = helpers.call_action('user_create', **data.TYPICAL_USER) + user = helpers.call_action('user_create', **data.typical_user()) # 2. Call the function that is being tested, once only. - # Note we have to pass the email address and password (in plain text!) - # to user_update even though we're not updating those fields, otherwise - # validation fails. + # FIXME we have to pass the email address and password to user_update + # even though we're not updating those fields, otherwise validation + # fails. helpers.call_action('user_update', id=user['name'], email=user['email'], - password=data.TYPICAL_USER['password'], + password=data.typical_user()['password'], name='updated', ) @@ -53,30 +100,22 @@ def test_update_user_name(self): # 4. Do absolutely nothing else! def test_user_update_with_id_that_does_not_exist(self): - # FIXME: Does this need to be more realistic? - # - Actually create a user before trying to update it with the wrong - # id? - # - Actually pass other params (name, about..) to user_update? + user_dict = data.typical_user() + user_dict['id'] = "there's no user with this id" with nose.tools.assert_raises(logic.NotFound) as context: - helpers.call_action('user_update', - id="there's no user with this id") + helpers.call_action('user_update', **user_dict) # TODO: Could assert the actual error message, not just the exception? # (Could also do this with many of the tests below.) def test_user_update_with_no_id(self): + user_dict = data.typical_user() + assert 'id' not in user_dict with nose.tools.assert_raises(logic.ValidationError) as context: - helpers.call_action('user_update') + helpers.call_action('user_update', **user_dict) def test_user_update_with_invalid_name(self): - user = helpers.call_action('user_create', **data.TYPICAL_USER) - - # FIXME: This actually breaks the canonical test form rule by calling - # the function-under-test multiple times. - # Breaking this up into multiple tests would actually be clearer in - # terms of each test testing one thing and the names of the test - # methods documenting what the test does and how the function is - # supposed to behave. - # BUT the test code would be very repetitive. + user = helpers.call_action('user_create', **data.typical_user()) + invalid_names = ('', 'a', False, 0, -1, 23, 'new', 'edit', 'search', 'a'*200, 'Hi!', ) for name in invalid_names: @@ -85,7 +124,7 @@ def test_user_update_with_invalid_name(self): helpers.call_action('user_update', **user) def test_user_update_to_name_that_already_exists(self): - fred = helpers.call_action('user_create', **data.TYPICAL_USER) + fred = helpers.call_action('user_create', **data.typical_user()) bob = helpers.call_action('user_create', name='bob', email='bob@bob.com', password='pass') @@ -95,12 +134,12 @@ def test_user_update_to_name_that_already_exists(self): with nose.tools.assert_raises(logic.ValidationError) as context: helpers.call_action('user_update', **fred) - def test_update_user_password(self): + def test_user_update_password(self): '''Test that updating a user's password works successfully.''' - user = helpers.call_action('user_create', **data.TYPICAL_USER) + user = helpers.call_action('user_create', **data.typical_user()) - # Note we have to pass the email address to user_update even though + # FIXME we have to pass the email address to user_update even though # we're not updating it, otherwise validation fails. helpers.call_action('user_update', id=user['name'], email=user['email'], @@ -114,21 +153,21 @@ def test_update_user_password(self): assert updated_user.validate_password('new password') def test_user_update_with_short_password(self): - user = helpers.call_action('user_create', **data.TYPICAL_USER) + user = helpers.call_action('user_create', **data.typical_user()) user['password'] = 'xxx' # This password is too short. with nose.tools.assert_raises(logic.ValidationError) as context: helpers.call_action('user_update', **user) def test_user_update_with_empty_password(self): - user = helpers.call_action('user_create', **data.TYPICAL_USER) + user = helpers.call_action('user_create', **data.typical_user()) user['password'] = '' with nose.tools.assert_raises(logic.ValidationError) as context: helpers.call_action('user_update', **user) def test_user_update_with_null_password(self): - user = helpers.call_action('user_create', **data.TYPICAL_USER) + user = helpers.call_action('user_create', **data.typical_user()) user['password'] = None with nose.tools.assert_raises(logic.ValidationError) as context: @@ -136,8 +175,31 @@ def test_user_update_with_null_password(self): # TODO: Valid and invalid values for the rest of the user model's fields. + def test_user_update_activity_stream(self): - pass + '''Test that the right activity is emitted when updating a user.''' + + user = helpers.call_action('user_create', **data.typical_user()) + before = datetime.datetime.now() + + # FIXME we have to pass the email address and password to user_update + # even though we're not updating those fields, otherwise validation + # fails. + helpers.call_action('user_update', id=user['name'], + email=user['email'], + password=data.typical_user()['password'], + name='updated', + ) + + activity_stream = helpers.call_action('user_activity_list', + id=user['id']) + latest_activity = activity_stream[0] + assert latest_activity['activity_type'] == 'changed user' + assert latest_activity['object_id'] == user['id'] + assert latest_activity['user_id'] == user['id'] + after = datetime.datetime.now() + timestamp = datetime_from_string(latest_activity['timestamp']) + assert timestamp >= before and timestamp <= after def test_user_update_with_custom_schema(self): pass From 683852c555928f1cff99e5863e63d8568f93a9e9 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 18 Jul 2013 17:05:04 +0200 Subject: [PATCH 010/269] [#1117] Add some more user_update invalid password tests --- ckan/new_tests/logic/action/test_update.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index 335100a2c00..90c7d3c0800 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -173,8 +173,15 @@ def test_user_update_with_null_password(self): with nose.tools.assert_raises(logic.ValidationError) as context: helpers.call_action('user_update', **user) - # TODO: Valid and invalid values for the rest of the user model's fields. + def test_user_update_with_invalid_password(self): + user = helpers.call_action('user_create', **data.typical_user()) + + for password in (False, -1, 23, 30.7): + user['password'] = password + with nose.tools.assert_raises(logic.ValidationError) as context: + helpers.call_action('user_update', **user) + # TODO: Valid and invalid values for the rest of the user model's fields. def test_user_update_activity_stream(self): '''Test that the right activity is emitted when updating a user.''' From df72fcbd336e29f3efa3b2d221dcd439c183762e Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 18 Jul 2013 17:06:02 +0200 Subject: [PATCH 011/269] [#1117] Fix some bad indentation --- ckan/new_tests/logic/action/test_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index 90c7d3c0800..113717a7545 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -126,7 +126,7 @@ def test_user_update_with_invalid_name(self): def test_user_update_to_name_that_already_exists(self): fred = helpers.call_action('user_create', **data.typical_user()) bob = helpers.call_action('user_create', name='bob', - email='bob@bob.com', password='pass') + email='bob@bob.com', password='pass') # Try to update fred and change his user name to bob, which is already # bob's user name From 7d1bdc511058e19f61080593f26a04b06f86bb43 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 18 Jul 2013 18:11:56 +0200 Subject: [PATCH 012/269] [#1117] Add user_update custom schema test --- ckan/new_tests/logic/action/test_update.py | 38 +++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index 113717a7545..2725aa7c473 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -209,7 +209,43 @@ def test_user_update_activity_stream(self): assert timestamp >= before and timestamp <= after def test_user_update_with_custom_schema(self): - pass + '''Test that custom schemas passed to user_update do get used. + + user_update allows a custom validation schema to be passed to it in the + context dict. This is just a simple test that if you pass a custom + schema user_update does at least call a custom method that's given in + the custom schema. We assume this means it did use the custom schema + instead of the default one for validation, so user_update's custom + schema feature does work. + + ''' + import mock + import ckan.logic.schema + + user = helpers.call_action('user_create', **data.typical_user()) + + # A mock validator method, it doesn't do anything but it records what + # params it gets called with and how many times. + mock_validator = mock.MagicMock() + + # Build a custom schema by taking the default schema and adding our + # mock method to its 'id' field. + schema = ckan.logic.schema.default_update_user_schema() + schema['id'].append(mock_validator) + + # Call user_update and pass our custom schema in the context. + # FIXME: We have to pass email and password even though we're not + # trying to update them, or validation fails. + helpers.call_action('user_update', context={'schema': schema}, + id=user['name'], email=user['email'], + password=data.typical_user()['password'], + name='updated', + ) + + # Since we passed user['name'] to user_update as the 'id' param, + # our mock validator method should have been called once with + # user['name'] as arg. + mock_validator.assert_called_once_with(user['name']) def test_user_update_with_deferred_commit(self): pass From 6276cb2e143bef2e108dc4c612c32832926f9b79 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 19 Jul 2013 14:51:44 +0200 Subject: [PATCH 013/269] [#1117] Add test for user_update's defer_commit option --- ckan/new_tests/logic/action/test_update.py | 77 +++++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index 2725aa7c473..489fa82e15b 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -39,7 +39,9 @@ ''' import datetime + import nose.tools +import mock import ckan.logic as logic import ckan.new_tests.helpers as helpers @@ -68,6 +70,13 @@ def setup(self): # Reset the db before each test method. model.repo.rebuild_db() + def teardown(self): + # Since some of the test methods below use the mock module to patch + # things, we use this teardown() method to remove remove all patches. + # (This makes sure the patches always get removed even if the test + # method aborts with an exception or something.) + mock.patch.stopall() + def test_user_update_name(self): '''Test that updating a user's name works successfully.''' @@ -219,7 +228,6 @@ def test_user_update_with_custom_schema(self): schema feature does work. ''' - import mock import ckan.logic.schema user = helpers.call_action('user_create', **data.typical_user()) @@ -248,4 +256,69 @@ def test_user_update_with_custom_schema(self): mock_validator.assert_called_once_with(user['name']) def test_user_update_with_deferred_commit(self): - pass + '''Test that user_update()'s deferred_commit option works. + + In this test we mock out the rest of CKAN and test the user_update() + action function in isolation. What we're testing is simply that when + called with 'deferred_commit': True in its context, user_update() does + not call ckan.model.repo.commit(). + + ''' + # Patch ckan.model, so user_update will be accessing a mock object + # instead of the real model. + # It's ckan.logic.__init__.py:get_action() that actually adds the model + # into the context dict, and that module does + # `import ckan.model as model`, so what we actually need to patch is + # 'ckan.logic.model' (the name that the model has when get_action() + # accesses it), not 'ckan.model'. + model_patch = mock.patch('ckan.logic.model') + mock_model = model_patch.start() + + # Patch the validate() function, so validate() won't really be called + # when user_update() calls it. update.py does + # `_validate = ckan.lib.navl.dictization_functions.validate` so we + # actually to patch this new name that's assigned to the function, not + # its original name. + validate_patch = mock.patch('ckan.logic.action.update._validate') + mock_validate = validate_patch.start() + + # user_update() is going to call validate() with some params and it's + # going to use the result that validate() returns. So we need to give + # a mock validate function that will accept the right number of params + # and return the right result. + def mock_validate_function(data_dict, schema, context): + '''Simply return the data_dict as given (without doing any + validation) and an empty error dict.''' + return data_dict, {} + mock_validate.side_effect = mock_validate_function + + # Patch model_save, again update.py does + # `import ckan.logic.action.update.model_save as model_save` so we + # need to patch it by its new name + # 'ckan.logic.action.update.model_save'. + model_save_patch = mock.patch('ckan.logic.action.update.model_save') + model_save_patch.start() + + # Patch model_dictize, again using the new name that update.py imports + # it as. + model_dictize_patch = mock.patch( + 'ckan.logic.action.update.model_dictize') + model_dictize_patch.start() + + # Patch the get_action() function (using the name that update.py + # assigns to it) with a default mock function that does nothing and + # returns None. + get_action_patch = mock.patch('ckan.logic.action.update._get_action') + get_action_patch.start() + + # After all that patching, we can finally call user_update passing + # 'defer_commit': True. The logic code in user_update will be run but + # it'll have no effect because everything it calls is mocked out. + helpers.call_action('user_update', + context={'defer_commit': True}, + id='foobar', name='new_name', + ) + + # Assert that user_update did *not* call our mock model object's + # model.repo.commit() method. + assert not mock_model.repo.commit.called From 482c4969afb32d796c231839d2d794d28daf90ae Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 19 Jul 2013 17:58:27 +0200 Subject: [PATCH 014/269] [#1117] Add draft testing coding standards doc --- ckan/new_tests/logic/action/test_update.py | 41 +-- doc/testing-coding-standards.rst | 340 +++++++++++++++++++++ 2 files changed, 341 insertions(+), 40 deletions(-) create mode 100644 doc/testing-coding-standards.rst diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index 489fa82e15b..9c8e95fcb78 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -1,43 +1,4 @@ -'''Unit tests for ckan/logic/action/update.py. - -Notes: - -- All user_update tests are named test_user_update_* - -- Using the canonical test form from the Pylons guidelines - (i.e. each method tests One Thing and the method name explains what it tests) - -- Not testing auth here, that can be done in ckan.tests.logic.auth.*.py - -- But I am testing validation here, because some action functions do some of - their own validation, not all the validation is done in the schemas - -- The tests for each action function try to cover: - - Testing for success: - - Typical values - - Edge cases - - Multiple parameters in different combinations - - Testing for failure: - - Common mistakes - - Bizarre input - - Unicode - -- Cover the interface of the function (i.e. test all params and features) - -- Not storing anything (e.g. test data) against self in the test class, - instead have each test method call helper functions to get any test data - it needs - - - I think it would be okay to create *read-only* data against self in - setup_class though - -- The tests are not ordered, and each one can be run on its own (db is rebuilt - after each test method) - -- Within reason, keep tests as clear and simple as possible even if it means - they get repetitive - -''' +'''Unit tests for ckan/logic/action/update.py.''' import datetime import nose.tools diff --git a/doc/testing-coding-standards.rst b/doc/testing-coding-standards.rst new file mode 100644 index 00000000000..7356a9edfc3 --- /dev/null +++ b/doc/testing-coding-standards.rst @@ -0,0 +1,340 @@ +======================== +Testing coding standards +======================== + + +------------------------------------------------ +Transitioning from the legacy tests to new tests +------------------------------------------------ + +CKAN is an old code base with a large legacy test suite in +:mod:`ckan.legacy_tests`. The legacy tests are difficult to maintain and +extend, but are too many to be replaced all at once in a single effort. So +we're following this strategy: + +.. todo:: The legacy tests haven't actually been moved to ``ckan.legacy_tests`` + yet. + +#. The legacy test suite has been moved to :mod:`ckan.legacy_tests`. +#. The new test suite has been started in :mod:`ckan.tests`. +#. For now, we'll run both the legacy tests and the new tests before + merging something into the master branch. +#. Whenever we add new code, or change existing code, we'll add new-style tests + for it. +#. If you change the behavior of some code and break some legacy tests, + consider adding new tests for that code and deleting the legacy tests, + rather than updating the legacy tests. +#. Now and then, we'll write a set of new tests to cover part of the code, + and delete the relevant legacy tests. For example if you want to refactor + some code that doesn't have good tests, write a set of new-style tests for + it first, refactor, then delete the relevant legacy tests. + +In this way we can incrementally extend the new tests to cover CKAN one "island +of code" at a time, and eventually we can delete the :mod:`legacy_tests` +directory entirely. + + +-------------------------------------- +Guidelines for writing new-style tests +-------------------------------------- + +.. important:: + + All new code, or changes to existing code, should have new or updated tests + before getting merged into master. + +.. todo:: + + Maybe give a short list of what we want from the new-style tests at the top + here. + +This section gives guidelines and examples to follow when writing or reviewing +new-style tests. The subsections below cover: + +#. `What should be tested?`_ Which parts of CKAN code should have tests, + and which don't have to? +#. `How should tests be organized?`_ When adding tests for some code, or + looking for the tests for some code, how do you know where the tests should + be? +#. `How detailed should tests be?`_ When writing the tests for a function or + method, how many tests should you write and what should the tests cover? +#. `How should tests be written?`_ What's the formula and guidelines for + writing a *good* test? + + +What should be tested? +====================== + +.. note:: + + When we say that *all* functions/methods of a module/class should have + tests, we mean all *public* functions/methods that the module/class exports + for use by other modules/classes in CKAN or by extensions or templates. + + *Private* helper methods (with names beginning with ``_``) never have to + have their own tests, although they can have tests if helpful. + +:mod:`ckan.logic.action` + All action functions should have tests. Note that the tests for an action + function *don't* need to cover authorization, because the authorization + functions have their own tests. But action function tests *do* need to cover + validation, more on that later. + +:mod:`ckan.logic.auth` + All auth functions should have tests. + +:mod:`ckan.logic.converters`, :mod:`ckan.logic.validators` + All converter and validator functions should have unit tests. + Although these functions are tested indirectly by the action function + tests, this may not catch all the converters and validators and all their + options, and converters and validators are not only used by the action + functions but are also available to plugins. Having unit tests will also + help to clarify the intended behavior of each converter and validator. + +:mod:`ckan.logic.schema.py` + We *don't* write tests for each schema. The validation done by the schemas + is instead tested indirectly by the action function tests. The reason for + this is that CKAN actually does validation in multiple places: some + validation is done using schemas, some validation is done in the action + functions themselves, some is done in dictization, and some in the model. + By testing all the different valid and invalid inputs at the action function + level, we catch it all in one place. + +:mod:`ckan.controllers` + All controller methods should have tests. + +:mod:`ckan.model` and :mod:`ckan.lib` + All "non-trivial" model and lib functions and methods should have tests. + + .. todo:: Define "trivial" with an example. + + Some code is used by extensions or templates, for example the template + helpers in :mod:`ckan.lib.helpers`. If a function or method is available to + extensions or templates then it should have tests, even if you think + it's trivial. + +:mod:`ckan.plugins` + The plugin interfaces in :mod:`ckan.plugins.interfaces` are not directly + testable because they don't contain any code, *but*: + + * Each plugin interface should have an example plugin in :mod:`ckan.ckanext` + and the example plugin should have its own functional tests. + + * The tests for the code that calls the plugin interface methods should test + that the methods are called correctly. + + For example :func:`ckan.logic.action.get.package_show` calls + :meth:`ckan.plugins.interfaces.IDatasetForm.read`, so the + :func:`~ckan.logic.action.get.package_show` tests should include tests + that :meth:`~ckan.plugins.interfaces.IDatasetForm.read` is called at the + right times and with the right parameters. + + Everything in :mod:`ckan.plugins.toolkit` should have tests, because these + functions are part of the API for extensions to use. But + :mod:`~ckan.plugins.toolkit` imports most of these functions from elsewhere + in CKAN, so the tests should be elsewhere also, in the test modules for the + modules where the functions are defined. + +:mod:`ckan.migration` + All migration scripts should have tests. + +:mod:`ckan.ckanext` + Within extensions, follow the same guidelines as for CKAN core. For example + if an extension adds an action function then the action function should have + tests, etc. + + +How should tests be organized? +============================== + +The organization of test modules in :mod:`ckan.tests` mirrors the organization +of the source modules in :mod:`ckan`:: + + ckan/ + tests/ + controllers/ + test_package.py <-- Tests for ckan/controllers/package.py + ... + lib/ + test_helpers.py <-- Tests for ckan/lib/helpers.py + ... + logic/ + action/ + test_get.py + ... + auth/ + test_get.py + ... + test_converters.py + test_validators.py + migration/ + versions/ + test_001_add_existing_tables.py + ... + model/ + test_package.py + ... + ... + +There are a few exceptional test modules that don't fit into this structure, +for example PEP8 tests and coding standards tests. These modules can just go in +the top-level ``ckan/tests/`` directory. There shouldn't be too many of these. + + +How detailed should tests be? +============================= + +When you're writing the tests for a function or method, how many tests should +you write and what should the tests cover? Generally, what we're trying to do +is test the *interfaces* between modules in a way that supports modularization: +if you change the code within a function, method, class or module, if you don't +break any of that code's unit tests you should be able to expect that CKAN as a +whole will not be broken. + +As a general guideline, the tests for a function or method should: + +- Test for success: + + - Test the function with typical, valid input values + - Test with valid, edge-case inputs + - If the function has multiple parameters, test them in different + combinations + +- Test for failure: + + - Test that the function fails correctly (e.g. raises the expected type of + exception) when given likely invalid inputs (for example, if the user + passes an invalid user_id as a parameter) + - Test that the function fails correctly when given bizarre input + +- Test that the function behaves correctly when given unicode characters as + input + +- Cover the interface of the function: test all the parameters and features of + the function + +.. todo:: + + What about sanity tests? For example I should be able to convert a value + to an extra using convert_to_extras() and then convert it back again using + convert_from_extras() and get the same value back. + + Private functions and methods that are only used within the module (and + whose names should begin with underscores) *may* have tests, but don't have + to. + + +How should tests be written? +============================ + +In general, follow the `Pylons Unit Testing Guidelines +`_. +We'll give some additional, CKAN-specific guidelines below: + + +Naming test methods +------------------- + +Some modules in CKAN contain large numbers of more-or-less unrelated functions. +For example, :mod:`ckan.logic.action.update` contains all functions for +updating things in CKAN. This means that +:mod:`ckan.tests.logic.action.test_update` is going to contain an even larger +number of test functions. + +So in addition to the name of each test function clearly explaining the intent +of the test, it's important to name the test function after the function it's +testing, for example all the tests for ``user_update`` should be named +``test_user_update_*``: + +* ``test_user_update_name`` +* ``test_user_update_password`` +* ``test_user_update_with_id_that_does_not_exist`` +* etc. + +It's also a good idea to keep all the ``user_update`` tests next to each other +in the file, and to order the tests for each function in the same order as the +functions are ordered in the source file. + +In smaller modules putting the source function name in the test function names +may not be necessary, but for a lot of modules in CKAN it's probably a good +idea. + +:mod:`ckan.tests.helpers` and :mod:`ckan.tests.data` +---------------------------------------------------- + +.. todo:: + + There are some test helper functions here. Explain what they're for and + maybe autodoc them here. + + +:mod:`ckan.tests.logic.action` +------------------------------ + +Tests for action functions should use the +:func:`ckan.tests.helpers.call_action` function to call the action functions. + +One thing :func:`~ckan.tests.helpers.call_action` does is to add +``ignore_auth: True`` into the ``context`` dict that's passed to the action +function. This means CKAN will not call the action function's authorization +function. :mod:`ckan.tests.logic.action` should not test authorization +(e.g. testing that users that should not be authorized cannot call an action, +etc.) because the authorization functions are tested separately in +:mod:`ckan.tests.logic.auth`. + +Action function tests *should* test the logic of the actions themselves, and +*should* test validation (e.g. that various kinds of valid input work as +expected, and invalid inputs raise the expected exceptions). + +.. todo:: + + Insert some examples here. + + +:mod:`ckan.tests.controllers` +----------------------------- + +Tests for controller methods should work by simulating HTTP requests and +testing the HTML that they get back. + +In general the tests for a controller shouldn't need to be too detailed, +because there shouldn't be a lot of complicated logic and code in controller +classes (the logic should be handled in :mod:`ckan.logic` and :mod:`ckan.lib`, +for example). The tests for a controller should: + +* Make sure that the template renders without crashing. + +* Test that the page contents seem basically correct, or test certain important + elements in the page contents (but don't do too much HTML parsing). + +* Test that submitting any forms on the page works without crashing and has + the expected side-effects. + +When asserting side-effects after submitting a form, controller tests should +user the :func:`ckan.tests.helpers.call_action` function. For example after +creating a new user by submitting the new user form, a test could call the +:func:`~ckan.logic.action.get.user_show` action function to verify that the +user was created with the correct values. + +.. warning:: + + Some CKAN controllers *do* contain a lot of complicated logic code. These + controllers should be refactored to move the logic into :mod:`ckan.logic` or + :mod:`ckan.lib` where it can be tested easily. Unfortunately in cases like + this it may be necessary to write a lot of controller tests to get this + code's behavior into a test harness before it can be safely refactored. + +.. todo:: + + How exactly should controller tests work? (e.g. with a webtest testapp and + beautifulsoup?) + + Insert examples here. + + +Mocking +------- + +.. todo:: + + Some examples of how (and when/where) to use the ``mock`` library in CKAN. From dab8d49e6d9a89df4e940cbed4282ab5d7698576 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 13:56:39 +0200 Subject: [PATCH 015/269] [#1117] Add a few new-style tests for user_update auth --- ckan/new_tests/logic/auth/test_update.py | 70 ++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 ckan/new_tests/logic/auth/test_update.py diff --git a/ckan/new_tests/logic/auth/test_update.py b/ckan/new_tests/logic/auth/test_update.py new file mode 100644 index 00000000000..49bf163178f --- /dev/null +++ b/ckan/new_tests/logic/auth/test_update.py @@ -0,0 +1,70 @@ +'''Unit tests for ckan/logic/auth.update.py. + +''' +import ckan.new_tests.helpers as helpers +import ckan.new_tests.data as data + + +class TestUpdate(object): + + @classmethod + def setup_class(cls): + helpers.reset_db() + + def setup(self): + import ckan.model as model + model.repo.rebuild_db() + + # TODO: Probably all auth function tests want this? Move to helpers.py? + def _call_auth(self, auth_name, context=None, **kwargs): + '''Call a ckan.logic.auth function more conveniently for testing. + + ''' + import ckan.model + import ckan.logic.auth.update + if context is None: + context = {} + context.setdefault('user', '127.0.0.1') + + context.setdefault('model', ckan.model) + + # FIXME: Do we want to go through check_access() here? + auth_function = ckan.logic.auth.update.__getattribute__(auth_name) + return auth_function(context=context, data_dict=kwargs) + + def test_user_update_visitor_cannot_update_user(self): + '''Visitors should not be able to update users' accounts.''' + + user = helpers.call_action('user_create', **data.typical_user()) + user['name'] = 'updated' + + # Try to update the user, but without passing any API key. + result = self._call_auth('user_update', **user) + assert result['success'] is False + + # TODO: Assert result['msg'] as well? In this case the message is not + # very sensible: "User 127.0.0.1 is not authorized to edit user foo". + # Also applies to the rest of these tests. + + def test_user_update_user_cannot_update_another_user(self): + '''Users should not be able to update other users' accounts.''' + + fred = helpers.call_action('user_create', **data.typical_user()) + bob = helpers.call_action('user_create', name='bob', + email='bob@bob.com', password='pass') + fred['name'] = 'updated' + + # Make Bob try to update Fred's user account. + context = {'user': bob['name']} + result = self._call_auth('user_update', context=context, **fred) + assert result['success'] is False + + def test_user_update_user_can_update_herself(self): + '''Users should be authorized to update their own accounts.''' + + user = helpers.call_action('user_create', **data.typical_user()) + + context = {'user': user['name']} + user['name'] = 'updated' + result = self._call_auth('user_update', context=context, **user) + assert result['success'] is True From 19b0b016d5acb3162bb7fbaf40fc9c4320f384e8 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 15:25:55 +0200 Subject: [PATCH 016/269] [#1117] Never return reset_keys in user dicts Like the user's password, there's never any reason for a user dict to contain their reset key (even if the user dict is being generated for the user herself or for a sysadmin) --- ckan/lib/dictization/model_dictize.py | 1 + ckan/tests/lib/test_dictization.py | 8 ++++---- ckan/tests/logic/test_action.py | 2 -- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index 3bb20695820..9687c52ae98 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -455,6 +455,7 @@ def user_dictize(user, context): result_dict = d.table_dictize(user, context) del result_dict['password'] + del result_dict['reset_key'] result_dict['display_name'] = user.display_name result_dict['email_hash'] = user.email_hash diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py index 04237766fdb..94059d94476 100644 --- a/ckan/tests/lib/test_dictization.py +++ b/ckan/tests/lib/test_dictization.py @@ -1136,11 +1136,11 @@ def test_22_user_dictize_as_sysadmin(self): # Check sensitive data is available assert 'apikey' in user_dict - assert 'reset_key' in user_dict assert 'email' in user_dict - # Passwords should never be available + # Passwords and reset keys should never be available assert 'password' not in user_dict + assert 'reset_key' not in user_dict def test_23_user_dictize_as_same_user(self): '''User should be able to see their own sensitive data.''' @@ -1160,11 +1160,11 @@ def test_23_user_dictize_as_same_user(self): # Check sensitive data is available assert 'apikey' in user_dict - assert 'reset_key' in user_dict assert 'email' in user_dict - # Passwords should never be available + # Passwords and reset keys should never be available assert 'password' not in user_dict + assert 'reset_key' not in user_dict def test_24_user_dictize_as_other_user(self): '''User should not be able to see other's sensitive data.''' diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index ddf9d3576a3..41e9de47e62 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -370,7 +370,6 @@ def test_05_user_show(self): result = res_obj['result'] assert result['name'] == 'annafan' assert 'apikey' in result - assert 'reset_key' in result # Sysadmin user can see everyone's api key res = self.app.post('/api/action/user_show', params=postparams, @@ -380,7 +379,6 @@ def test_05_user_show(self): result = res_obj['result'] assert result['name'] == 'annafan' assert 'apikey' in result - assert 'reset_key' in result def test_05_user_show_edits(self): postparams = '%s=1' % json.dumps({'id':'tester'}) From a956b1f7de469cf89a36441c2b11d12bb5fadcc7 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 16:11:30 +0200 Subject: [PATCH 017/269] [#1117] Add docstring and unit tests for ignore_missing validator --- ckan/lib/navl/validators.py | 13 +++++ ckan/new_tests/lib/navl/test_validators.py | 65 ++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 ckan/new_tests/lib/navl/test_validators.py diff --git a/ckan/lib/navl/validators.py b/ckan/lib/navl/validators.py index 01cbd87567f..b7ac124e624 100644 --- a/ckan/lib/navl/validators.py +++ b/ckan/lib/navl/validators.py @@ -76,7 +76,20 @@ def callable(key, data, errors, context): return callable def ignore_missing(key, data, errors, context): + '''If the key is missing from the data, ignore the rest of the key's + schema. + By putting ignore_missing at the start of the schema list for a key, + you can allow users to post a dict without the key and the dict will pass + validation. But if they post a dict that does contain the key, then any + validators after ignore_missing in the key's schema list will be applied. + + :raises ckan.lib.navl.dictization_functions.StopOnError: if ``data[key]`` + is :py:data:`ckan.lib.navl.dictization_functions.missing` or ``None`` + + :returns: ``None`` + + ''' value = data.get(key) if value is missing or value is None: diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py new file mode 100644 index 00000000000..a3eaf3a5ee5 --- /dev/null +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -0,0 +1,65 @@ +'''Unit tests for ckan/lib/navl/validators.py. + +''' +import nose.tools + + +class TestValidators(object): + + def test_ignore_missing_with_None(self): + '''If data[key] is None ignore_missing() should raise StopOnError.''' + + import ckan.lib.navl.dictization_functions as df + import ckan.lib.navl.validators as validators + + key = ('foo',) + data = {key: None} + errors = {key: []} + + with nose.tools.assert_raises(df.StopOnError): + validators.ignore_missing( + key=key, + data=data, + errors=errors, + context={}) + + # ignore_missing should remove the item from the dict. + assert key not in data + + def test_ignore_missing_with_missing(self): + '''If data[key] is missing ignore_missing() should raise StopOnError. + + ''' + import ckan.lib.navl.dictization_functions as df + import ckan.lib.navl.validators as validators + + key = ('foo',) + data = {key: df.missing} + errors = {key: []} + + with nose.tools.assert_raises(df.StopOnError): + validators.ignore_missing( + key=key, + data=data, + errors=errors, + context={}) + + # ignore_missing should remove the item from the dict. + assert key not in data + + def test_ignore_missing_with_a_value(self): + '''If data[key] is neither None or missing, ignore_missing() should do + nothing. + + ''' + import ckan.lib.navl.validators as validators + + key = ('foo',) + data = {key: 'bar'} + errors = {key: []} + + result = validators.ignore_missing(key=key, data=data, errors=errors, + context={}) + + assert result is None + assert data.get(key) == 'bar' From ca16b016287d013f41abd033cb95085762ac96ca Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 16:15:50 +0200 Subject: [PATCH 018/269] [#1117] Add another ignore_missing test --- ckan/new_tests/lib/navl/test_validators.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index a3eaf3a5ee5..1d0f526b273 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -47,6 +47,26 @@ def test_ignore_missing_with_missing(self): # ignore_missing should remove the item from the dict. assert key not in data + def test_ignore_missing_without_key(self): + '''If key is not in data ignore_missing() should raise StopOnError.''' + + import ckan.lib.navl.dictization_functions as df + import ckan.lib.navl.validators as validators + + key = ('foo',) + data = {} + errors = {key: []} + + with nose.tools.assert_raises(df.StopOnError): + validators.ignore_missing( + key=key, + data=data, + errors=errors, + context={}) + + # ignore_missing should remove the item from the dict. + assert key not in data + def test_ignore_missing_with_a_value(self): '''If data[key] is neither None or missing, ignore_missing() should do nothing. From 0365a9599e3050256879f982d5edeade1d0f3a83 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 16:17:47 +0200 Subject: [PATCH 019/269] [#1117] Add some asserts to ignore_missing tests --- ckan/new_tests/lib/navl/test_validators.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index 1d0f526b273..1e34b51face 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -26,6 +26,9 @@ def test_ignore_missing_with_None(self): # ignore_missing should remove the item from the dict. assert key not in data + # ignore_missing should not add any errors. + assert errors[key] == [] + def test_ignore_missing_with_missing(self): '''If data[key] is missing ignore_missing() should raise StopOnError. @@ -47,6 +50,9 @@ def test_ignore_missing_with_missing(self): # ignore_missing should remove the item from the dict. assert key not in data + # ignore_missing should not add any errors. + assert errors[key] == [] + def test_ignore_missing_without_key(self): '''If key is not in data ignore_missing() should raise StopOnError.''' @@ -67,6 +73,9 @@ def test_ignore_missing_without_key(self): # ignore_missing should remove the item from the dict. assert key not in data + # ignore_missing should not add any errors. + assert errors[key] == [] + def test_ignore_missing_with_a_value(self): '''If data[key] is neither None or missing, ignore_missing() should do nothing. @@ -82,4 +91,9 @@ def test_ignore_missing_with_a_value(self): context={}) assert result is None + + # ignore_missing shouldn't remove or modify the value. assert data.get(key) == 'bar' + + # ignore_missing shouldn't add any errors. + assert errors[key] == [] From 262adf316d0ac4131341fbd07e4cade16531d36d Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 16:24:10 +0200 Subject: [PATCH 020/269] [#1117] Tweak a unit test --- ckan/new_tests/lib/navl/test_validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index 1e34b51face..0b38b7e3a91 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -70,8 +70,8 @@ def test_ignore_missing_without_key(self): errors=errors, context={}) - # ignore_missing should remove the item from the dict. - assert key not in data + # ignore_missing shouldn't change the data dict. + assert data == {} # ignore_missing should not add any errors. assert errors[key] == [] From 69ea1ad0ca2c0d9a4ee1faf95c25e28b64d4cfe6 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 17:10:00 +0200 Subject: [PATCH 021/269] [#1117] Combine three unit tests into one --- ckan/new_tests/lib/navl/test_validators.py | 88 ++++++---------------- 1 file changed, 23 insertions(+), 65 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index 0b38b7e3a91..9a3dc4e7256 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -6,75 +6,33 @@ class TestValidators(object): - def test_ignore_missing_with_None(self): - '''If data[key] is None ignore_missing() should raise StopOnError.''' + def test_ignore_missing_with_value_missing(self): + '''If data[key] is None or dictization_functions.missing or if + key is not in data, then ignore_missing() should raise StopOnError.''' import ckan.lib.navl.dictization_functions as df import ckan.lib.navl.validators as validators - key = ('foo',) - data = {key: None} - errors = {key: []} - - with nose.tools.assert_raises(df.StopOnError): - validators.ignore_missing( - key=key, - data=data, - errors=errors, - context={}) - - # ignore_missing should remove the item from the dict. - assert key not in data - - # ignore_missing should not add any errors. - assert errors[key] == [] - - def test_ignore_missing_with_missing(self): - '''If data[key] is missing ignore_missing() should raise StopOnError. - - ''' - import ckan.lib.navl.dictization_functions as df - import ckan.lib.navl.validators as validators - - key = ('foo',) - data = {key: df.missing} - errors = {key: []} - - with nose.tools.assert_raises(df.StopOnError): - validators.ignore_missing( - key=key, - data=data, - errors=errors, - context={}) - - # ignore_missing should remove the item from the dict. - assert key not in data - - # ignore_missing should not add any errors. - assert errors[key] == [] - - def test_ignore_missing_without_key(self): - '''If key is not in data ignore_missing() should raise StopOnError.''' - - import ckan.lib.navl.dictization_functions as df - import ckan.lib.navl.validators as validators - - key = ('foo',) - data = {} - errors = {key: []} - - with nose.tools.assert_raises(df.StopOnError): - validators.ignore_missing( - key=key, - data=data, - errors=errors, - context={}) - - # ignore_missing shouldn't change the data dict. - assert data == {} - - # ignore_missing should not add any errors. - assert errors[key] == [] + for value in (None, df.missing, 'skip'): + key = ('foo',) + if value == 'skip': + data = {} + else: + data = {key: value} + errors = {key: []} + + with nose.tools.assert_raises(df.StopOnError): + validators.ignore_missing( + key=key, + data=data, + errors=errors, + context={}) + + # ignore_missing should remove the item from the dict. + assert key not in data + + # ignore_missing should not add any errors. + assert errors[key] == [] def test_ignore_missing_with_a_value(self): '''If data[key] is neither None or missing, ignore_missing() should do From e7d5c9d08ddf3153c08ee3e96ee8e6b956db7523 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 17:26:10 +0200 Subject: [PATCH 022/269] [#1117] Improve a unit test Add some more realistic data and check that it doesn't get modified. --- ckan/new_tests/lib/navl/test_validators.py | 28 ++++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index 9a3dc4e7256..cf1334adbcc 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -14,13 +14,30 @@ def test_ignore_missing_with_value_missing(self): import ckan.lib.navl.validators as validators for value in (None, df.missing, 'skip'): - key = ('foo',) + + # This is the key for the value that is going to be validated. + key = ('key to be validated',) + + # This is another random key that's going to be in the data and + # errors dict just so we can test that ignore_missing() doesn't + # modify it. + other_key = ('other key',) + if value == 'skip': data = {} else: data = {key: value} + + # Add some other random stuff into data, just so we can test that + # ignore_missing() doesn't modify it. + data[other_key] = 'other value' + errors = {key: []} + # Add some other random errors into errors, just so we can test + # that ignore_missing doesn't modify them. + errors[other_key] = ['other error'] + with nose.tools.assert_raises(df.StopOnError): validators.ignore_missing( key=key, @@ -28,11 +45,12 @@ def test_ignore_missing_with_value_missing(self): errors=errors, context={}) - # ignore_missing should remove the item from the dict. - assert key not in data + # ignore_missing should remove the key being validated from the + # dict, but it should not remove other keys or add any keys. + assert data == {other_key: 'other value'} - # ignore_missing should not add any errors. - assert errors[key] == [] + # ignore_missing shouldn't modify the errors dict. + assert errors == {key: [], other_key: ['other error']} def test_ignore_missing_with_a_value(self): '''If data[key] is neither None or missing, ignore_missing() should do From 476b8ff944eeab40f6c98a5334df69b6d6787332 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 17:41:33 +0200 Subject: [PATCH 023/269] [#1117] Move some unit test code into helper functions --- ckan/new_tests/lib/navl/test_validators.py | 65 ++++++++++++++-------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index cf1334adbcc..b8307e54db4 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -1,9 +1,35 @@ '''Unit tests for ckan/lib/navl/validators.py. ''' +import copy + import nose.tools +def _data(): + '''Return a data dict with some random data in it, suitable to be passed + to validators for testing. + + This is a function that returns a dict (rather than just a dict as a + module-level variable) so that if one test method modifies the dict the + next test method gets a new clean copy. + + ''' + return {('other key',): 'other value'} + + +def _errors(): + '''Return an errors dict with some random errors in it, suitable to be + passed to validators for testing. + + This is a function that returns a dict (rather than just a dict as a + module-level variable) so that if one test method modifies the dict the + next test method gets a new clean copy. + + ''' + return {('other key',): ['other error']} + + class TestValidators(object): def test_ignore_missing_with_value_missing(self): @@ -18,25 +44,18 @@ def test_ignore_missing_with_value_missing(self): # This is the key for the value that is going to be validated. key = ('key to be validated',) - # This is another random key that's going to be in the data and - # errors dict just so we can test that ignore_missing() doesn't - # modify it. - other_key = ('other key',) - - if value == 'skip': - data = {} - else: - data = {key: value} - - # Add some other random stuff into data, just so we can test that - # ignore_missing() doesn't modify it. - data[other_key] = 'other value' + # The data to pass to the validator function for validation. + data = _data() + if value != 'skip': + data[key] = value - errors = {key: []} + # The errors dict to pass to the validator function. + errors = _errors() + errors[key] = [] - # Add some other random errors into errors, just so we can test - # that ignore_missing doesn't modify them. - errors[other_key] = ['other error'] + # Make copies of the data and errors dicts for asserting later. + original_data = copy.deepcopy(data) + original_errors = copy.deepcopy(errors) with nose.tools.assert_raises(df.StopOnError): validators.ignore_missing( @@ -45,12 +64,14 @@ def test_ignore_missing_with_value_missing(self): errors=errors, context={}) - # ignore_missing should remove the key being validated from the - # dict, but it should not remove other keys or add any keys. - assert data == {other_key: 'other value'} + # ignore_missing() should remove the key being validated from the + # data dict, but it should not remove other keys or add any keys. + if key in original_data: + del original_data[key] + assert data == original_data - # ignore_missing shouldn't modify the errors dict. - assert errors == {key: [], other_key: ['other error']} + # ignore_missing() shouldn't modify the errors dict. + assert errors == original_errors def test_ignore_missing_with_a_value(self): '''If data[key] is neither None or missing, ignore_missing() should do From 854407ae96b0722a874dcceb14192b28eb7ffe1e Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 17:46:05 +0200 Subject: [PATCH 024/269] [#1117] Tweak a couple of docstrings --- ckan/new_tests/lib/navl/test_validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index b8307e54db4..d97213752f5 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -7,7 +7,7 @@ def _data(): - '''Return a data dict with some random data in it, suitable to be passed + '''Return a data dict with some arbitrary data in it, suitable to be passed to validators for testing. This is a function that returns a dict (rather than just a dict as a @@ -19,7 +19,7 @@ def _data(): def _errors(): - '''Return an errors dict with some random errors in it, suitable to be + '''Return an errors dict with some arbitrary errors in it, suitable to be passed to validators for testing. This is a function that returns a dict (rather than just a dict as a From ed12fa0f61a555baa33ef762085df2b214dd46dc Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 25 Jul 2013 18:24:22 +0200 Subject: [PATCH 025/269] [#1117] Move some test comments into asserts This makes the output when tests fail more useful Unfortunately I don't see how to do this with assert_raises. --- ckan/new_tests/lib/navl/test_validators.py | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index d97213752f5..6428b3a2358 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -58,20 +58,22 @@ def test_ignore_missing_with_value_missing(self): original_errors = copy.deepcopy(errors) with nose.tools.assert_raises(df.StopOnError): - validators.ignore_missing( - key=key, - data=data, - errors=errors, - context={}) - - # ignore_missing() should remove the key being validated from the - # data dict, but it should not remove other keys or add any keys. + validators.ignore_missing(key=key, data=data, errors=errors, + context={}) + + assert key not in data, ('When given a value of {value} ' + 'ignore_missing() should remove the item from the data ' + 'dict'.format(value=value)) + if key in original_data: del original_data[key] - assert data == original_data + assert data == original_data, ('When given a value of {value} ' + 'ignore_missing() should not modify other items in the ' + 'data dict'.format(value=value)) - # ignore_missing() shouldn't modify the errors dict. - assert errors == original_errors + assert errors == original_errors, ('When given a value of {value} ' + 'ignore_missing should not modify the errors dict'.format( + value=value)) def test_ignore_missing_with_a_value(self): '''If data[key] is neither None or missing, ignore_missing() should do From 2f8e74a0e2bb39c2fec1bf9604ab6ada764c1f2b Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 12:15:42 +0200 Subject: [PATCH 026/269] [#1117] Tweak a docstring This is easier to read I think --- ckan/new_tests/lib/navl/test_validators.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index 6428b3a2358..3f00ef62330 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -33,9 +33,13 @@ def _errors(): class TestValidators(object): def test_ignore_missing_with_value_missing(self): - '''If data[key] is None or dictization_functions.missing or if - key is not in data, then ignore_missing() should raise StopOnError.''' + '''ignore_missing() should raise StopOnError if: + - data[key] is None, or + - data[key] is dictization_functions.missing, or + - key is not in data + + ''' import ckan.lib.navl.dictization_functions as df import ckan.lib.navl.validators as validators From e5d37e1d6543522aebd58ec9bcd1ef825179d0d2 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 12:23:20 +0200 Subject: [PATCH 027/269] [#1117] Make a unit test more realistic Make it use the _data() and _errors() helper functions to get more realistic (non-empty) data and errors dicts. This just makes the test a little more thorough and realistic. --- ckan/new_tests/lib/navl/test_validators.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index 3f00ef62330..a6d7d627b65 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -86,17 +86,23 @@ def test_ignore_missing_with_a_value(self): ''' import ckan.lib.navl.validators as validators - key = ('foo',) - data = {key: 'bar'} - errors = {key: []} + key = ('key to be validated',) + data = _data() + data[key] = 'value to be validated' + errors = _errors() + errors[key] = [] + + # Make copies of the data and errors dicts for asserting later. + original_data = copy.deepcopy(data) + original_errors = copy.deepcopy(errors) result = validators.ignore_missing(key=key, data=data, errors=errors, context={}) assert result is None - # ignore_missing shouldn't remove or modify the value. - assert data.get(key) == 'bar' + assert data == original_data, ("ignore_missing() shouldn't modify the " + "data dict") - # ignore_missing shouldn't add any errors. - assert errors[key] == [] + assert errors == original_errors, ("ignore_missing() shouldn't modify " + "the errors dict") From 404f06e778c6b9f2de8dea44c241769a598f2e7a Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 13:57:18 +0200 Subject: [PATCH 028/269] [#1117] Add new tests and docstring for name_validator() There are no existing tests for this that I can see. --- ckan/logic/validators.py | 30 ++++++-- ckan/new_tests/lib/navl/test_validators.py | 80 ++++++++++++++++++++++ 2 files changed, 103 insertions(+), 7 deletions(-) diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 5874f89d81c..67a68d0424b 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -275,23 +275,39 @@ def extras_unicode_convert(extras, context): return extras name_match = re.compile('[a-z0-9_\-]*$') -def name_validator(val, context): - if not isinstance(val, basestring): +def name_validator(value, context): + '''Return the given value if it's a valid name, otherwise raise Invalid. + + If it's a valid name, the given value will be returned unmodified. + + This function applies general validation rules for names of packages, + groups, users, etc. + + Most schemas also have their own custom name validator function to apply + custom validation rules after this function, for example a + ``package_name_validator()`` to check that no package with the given name + already exists. + + :raises ckan.lib.navl.dictization_functions.Invalid: if ``value`` is not + a valid name + + ''' + if not isinstance(value, basestring): raise Invalid(_('Names must be strings')) # check basic textual rules - if val in ['new', 'edit', 'search']: + if value in ['new', 'edit', 'search']: raise Invalid(_('That name cannot be used')) - if len(val) < 2: + if len(value) < 2: raise Invalid(_('Name must be at least %s characters long') % 2) - if len(val) > PACKAGE_NAME_MAX_LENGTH: + if len(value) > PACKAGE_NAME_MAX_LENGTH: raise Invalid(_('Name must be a maximum of %i characters long') % \ PACKAGE_NAME_MAX_LENGTH) - if not name_match.match(val): + if not name_match.match(value): raise Invalid(_('Url must be purely lowercase alphanumeric ' '(ascii) characters and these symbols: -_')) - return val + return value def package_name_validator(key, data, errors, context): model = context["model"] diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index a6d7d627b65..e40d9122c3a 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- '''Unit tests for ckan/lib/navl/validators.py. ''' @@ -106,3 +107,82 @@ def test_ignore_missing_with_a_value(self): assert errors == original_errors, ("ignore_missing() shouldn't modify " "the errors dict") + + def test_name_validator_with_invalid_value(self): + '''If given an invalid value name_validator() should do raise Invalid. + + ''' + import ckan.logic.validators as validators + import ckan.lib.navl.dictization_functions as df + import ckan.model as model + + invalid_values = [ + # Non-string names aren't allowed as names. + 13, + 23.7, + 100L, + 1.0j, + None, + True, + False, + ('a', 2, False), + [13, None, True], + {'foo': 'bar'}, + lambda x: x**2, + + # Certain reserved strings aren't allowed as names. + 'new', + 'edit', + 'search', + + # Strings < 2 characters long aren't allowed as names. + '', + 'a', + '2', + + # Strings > PACKAGE_NAME_MAX_LENGTH long aren't allowed as names. + 'a' * (model.PACKAGE_NAME_MAX_LENGTH + 1), + + # Strings containing non-ascii characters aren't allowed as names. + u"fred_â¤%'\"Ußabc@fred.com", + + # Strings containing upper-case characters aren't allowed as names. + 'seanH', + + # Strings containing spaces aren't allowed as names. + 'sean h', + + # Strings containing punctuation aren't allowed as names. + 'seanh!', + ] + + for invalid_value in invalid_values: + with nose.tools.assert_raises(df.Invalid): + validators.name_validator(invalid_value, context={}) + + def test_name_validator_with_valid_value(self): + '''If given a valid string name_validator() should do nothing and + return the string. + + ''' + import ckan.logic.validators as validators + import ckan.model as model + + valid_names = [ + 'fred', + 'fred-flintstone', + 'fred_flintstone', + 'fred_flintstone-9', + 'f' * model.PACKAGE_NAME_MAX_LENGTH, + '-' * model.PACKAGE_NAME_MAX_LENGTH, + '_' * model.PACKAGE_NAME_MAX_LENGTH, + '9' * model.PACKAGE_NAME_MAX_LENGTH, + '99', + '--', + '__', + ] + + for valid_name in valid_names: + result = validators.name_validator(valid_name, context={}) + assert result == valid_name, ('If given a valid string ' + 'name_validator() should return the string unmodified.') From a0a3d93ca87df5c2af629dde365fe872c4c248d9 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 14:01:30 +0200 Subject: [PATCH 029/269] [#1117] Add a valid unicode name to name_validator() test --- ckan/new_tests/lib/navl/test_validators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index e40d9122c3a..191db6877bb 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -180,6 +180,7 @@ def test_name_validator_with_valid_value(self): '99', '--', '__', + u'fred-flintstone_9', ] for valid_name in valid_names: From f3d0de6208486528f0e08f7edcbba70faea2dd86 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 16:31:16 +0200 Subject: [PATCH 030/269] [#1117] Move some validator tests into the right test module Oops :) --- ckan/new_tests/lib/navl/test_validators.py | 80 -------------------- ckan/new_tests/logic/test_validators.py | 88 ++++++++++++++++++++++ 2 files changed, 88 insertions(+), 80 deletions(-) create mode 100644 ckan/new_tests/logic/test_validators.py diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index 191db6877bb..0405678ef1e 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -107,83 +107,3 @@ def test_ignore_missing_with_a_value(self): assert errors == original_errors, ("ignore_missing() shouldn't modify " "the errors dict") - - def test_name_validator_with_invalid_value(self): - '''If given an invalid value name_validator() should do raise Invalid. - - ''' - import ckan.logic.validators as validators - import ckan.lib.navl.dictization_functions as df - import ckan.model as model - - invalid_values = [ - # Non-string names aren't allowed as names. - 13, - 23.7, - 100L, - 1.0j, - None, - True, - False, - ('a', 2, False), - [13, None, True], - {'foo': 'bar'}, - lambda x: x**2, - - # Certain reserved strings aren't allowed as names. - 'new', - 'edit', - 'search', - - # Strings < 2 characters long aren't allowed as names. - '', - 'a', - '2', - - # Strings > PACKAGE_NAME_MAX_LENGTH long aren't allowed as names. - 'a' * (model.PACKAGE_NAME_MAX_LENGTH + 1), - - # Strings containing non-ascii characters aren't allowed as names. - u"fred_â¤%'\"Ußabc@fred.com", - - # Strings containing upper-case characters aren't allowed as names. - 'seanH', - - # Strings containing spaces aren't allowed as names. - 'sean h', - - # Strings containing punctuation aren't allowed as names. - 'seanh!', - ] - - for invalid_value in invalid_values: - with nose.tools.assert_raises(df.Invalid): - validators.name_validator(invalid_value, context={}) - - def test_name_validator_with_valid_value(self): - '''If given a valid string name_validator() should do nothing and - return the string. - - ''' - import ckan.logic.validators as validators - import ckan.model as model - - valid_names = [ - 'fred', - 'fred-flintstone', - 'fred_flintstone', - 'fred_flintstone-9', - 'f' * model.PACKAGE_NAME_MAX_LENGTH, - '-' * model.PACKAGE_NAME_MAX_LENGTH, - '_' * model.PACKAGE_NAME_MAX_LENGTH, - '9' * model.PACKAGE_NAME_MAX_LENGTH, - '99', - '--', - '__', - u'fred-flintstone_9', - ] - - for valid_name in valid_names: - result = validators.name_validator(valid_name, context={}) - assert result == valid_name, ('If given a valid string ' - 'name_validator() should return the string unmodified.') diff --git a/ckan/new_tests/logic/test_validators.py b/ckan/new_tests/logic/test_validators.py new file mode 100644 index 00000000000..64cee386440 --- /dev/null +++ b/ckan/new_tests/logic/test_validators.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +'''Unit tests for ckan/logic/validators.py. + +''' +import nose.tools + + +class TestValidators(object): + + def test_name_validator_with_invalid_value(self): + '''If given an invalid value name_validator() should do raise Invalid. + + ''' + import ckan.logic.validators as validators + import ckan.lib.navl.dictization_functions as df + import ckan.model as model + + invalid_values = [ + # Non-string names aren't allowed as names. + 13, + 23.7, + 100L, + 1.0j, + None, + True, + False, + ('a', 2, False), + [13, None, True], + {'foo': 'bar'}, + lambda x: x**2, + + # Certain reserved strings aren't allowed as names. + 'new', + 'edit', + 'search', + + # Strings < 2 characters long aren't allowed as names. + '', + 'a', + '2', + + # Strings > PACKAGE_NAME_MAX_LENGTH long aren't allowed as names. + 'a' * (model.PACKAGE_NAME_MAX_LENGTH + 1), + + # Strings containing non-ascii characters aren't allowed as names. + u"fred_â¤%'\"Ußabc@fred.com", + + # Strings containing upper-case characters aren't allowed as names. + 'seanH', + + # Strings containing spaces aren't allowed as names. + 'sean h', + + # Strings containing punctuation aren't allowed as names. + 'seanh!', + ] + + for invalid_value in invalid_values: + with nose.tools.assert_raises(df.Invalid): + validators.name_validator(invalid_value, context={}) + + def test_name_validator_with_valid_value(self): + '''If given a valid string name_validator() should do nothing and + return the string. + + ''' + import ckan.logic.validators as validators + import ckan.model as model + + valid_names = [ + 'fred', + 'fred-flintstone', + 'fred_flintstone', + 'fred_flintstone-9', + 'f' * model.PACKAGE_NAME_MAX_LENGTH, + '-' * model.PACKAGE_NAME_MAX_LENGTH, + '_' * model.PACKAGE_NAME_MAX_LENGTH, + '9' * model.PACKAGE_NAME_MAX_LENGTH, + '99', + '--', + '__', + u'fred-flintstone_9', + ] + + for valid_name in valid_names: + result = validators.name_validator(valid_name, context={}) + assert result == valid_name, ('If given a valid string ' + 'name_validator() should return the string unmodified.') From f234745c5f034ce59fb919dcf5536688a664bb88 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 17:21:20 +0200 Subject: [PATCH 031/269] [#1117] Move some helpers into shared test helper data module --- ckan/new_tests/data.py | 16 +++++++++++ ckan/new_tests/lib/navl/test_validators.py | 32 ++++------------------ 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/ckan/new_tests/data.py b/ckan/new_tests/data.py index e7e9f120e1c..c8b534a2a3c 100644 --- a/ckan/new_tests/data.py +++ b/ckan/new_tests/data.py @@ -12,3 +12,19 @@ def typical_user(): 'email': 'fred@fred.com', 'password': 'wilma', } + + +def validator_data_dict(): + '''Return a data dict with some arbitrary data in it, suitable to be passed + to validator functions for testing. + + ''' + return {('other key',): 'other value'} + + +def validator_errors_dict(): + '''Return an errors dict with some arbitrary errors in it, suitable to be + passed to validator functions for testing. + + ''' + return {('other key',): ['other error']} diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index 0405678ef1e..51df87da648 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -6,29 +6,7 @@ import nose.tools - -def _data(): - '''Return a data dict with some arbitrary data in it, suitable to be passed - to validators for testing. - - This is a function that returns a dict (rather than just a dict as a - module-level variable) so that if one test method modifies the dict the - next test method gets a new clean copy. - - ''' - return {('other key',): 'other value'} - - -def _errors(): - '''Return an errors dict with some arbitrary errors in it, suitable to be - passed to validators for testing. - - This is a function that returns a dict (rather than just a dict as a - module-level variable) so that if one test method modifies the dict the - next test method gets a new clean copy. - - ''' - return {('other key',): ['other error']} +import ckan.new_tests.data as test_data class TestValidators(object): @@ -50,12 +28,12 @@ def test_ignore_missing_with_value_missing(self): key = ('key to be validated',) # The data to pass to the validator function for validation. - data = _data() + data = test_data.validator_data_dict() if value != 'skip': data[key] = value # The errors dict to pass to the validator function. - errors = _errors() + errors = test_data.validator_errors_dict() errors[key] = [] # Make copies of the data and errors dicts for asserting later. @@ -88,9 +66,9 @@ def test_ignore_missing_with_a_value(self): import ckan.lib.navl.validators as validators key = ('key to be validated',) - data = _data() + data = test_data.validator_data_dict() data[key] = 'value to be validated' - errors = _errors() + errors = test_data.validator_errors_dict() errors[key] = [] # Make copies of the data and errors dicts for asserting later. From a3485a1fa4a872abc3d29f382069f34785f72f1e Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 17:51:03 +0200 Subject: [PATCH 032/269] [#1117] Add unit tests for user_name_validator() --- ckan/new_tests/logic/test_validators.py | 142 ++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/ckan/new_tests/logic/test_validators.py b/ckan/new_tests/logic/test_validators.py index 64cee386440..d6556532347 100644 --- a/ckan/new_tests/logic/test_validators.py +++ b/ckan/new_tests/logic/test_validators.py @@ -2,11 +2,27 @@ '''Unit tests for ckan/logic/validators.py. ''' +import copy + import nose.tools +import ckan.new_tests.helpers as helpers +import ckan.new_tests.data as test_data + class TestValidators(object): + @classmethod + def setup_class(cls): + # Initialize the test db (if it isn't already) and clean out any data + # left in it. + helpers.reset_db() + + def setup(self): + import ckan.model as model + # Reset the db before each test method. + model.repo.rebuild_db() + def test_name_validator_with_invalid_value(self): '''If given an invalid value name_validator() should do raise Invalid. @@ -86,3 +102,129 @@ def test_name_validator_with_valid_value(self): result = validators.name_validator(valid_name, context={}) assert result == valid_name, ('If given a valid string ' 'name_validator() should return the string unmodified.') + + def test_user_name_validator_with_non_string_value(self): + '''user_name_validator() should raise Invalid if given a non-string + value. + + ''' + import ckan.logic.validators as validators + import ckan.model as model + import ckan.lib.navl.dictization_functions as df + + non_string_values = [ + 13, + 23.7, + 100L, + 1.0j, + None, + True, + False, + ('a', 2, False), + [13, None, True], + {'foo': 'bar'}, + lambda x: x**2, + ] + + key = ('name',) + context = { + 'model': model, + 'session': model.Session, + 'user': '', + } + + for non_string_value in non_string_values: + data = test_data.validator_data_dict() + data[key] = non_string_value + errors = test_data.validator_errors_dict() + errors[key] = [] + + # Make copies of the data and errors dicts for asserting later. + original_data = copy.deepcopy(data) + original_errors = copy.deepcopy(errors) + + with nose.tools.assert_raises(df.Invalid): + validators.user_name_validator(key, data, errors, context) + + assert data == original_data, ("user_name_validator shouldn't " + 'modify the data dict') + + assert errors == original_errors, ("user_name_validator shouldn't " + 'modify the errors dict') + + def test_user_name_validator_with_a_name_that_already_exists(self): + '''user_name_validator() should add to the errors dict if given a + user name that already exists. + + ''' + import ckan.logic.validators as validators + import ckan.model as model + + existing_user = helpers.call_action('user_create', + **test_data.typical_user()) + + # Try to create another user with the same name as the existing user. + data = test_data.validator_data_dict() + key = ('name',) + data[key] = existing_user['name'] + errors = test_data.validator_errors_dict() + errors[key] = [] + context = { + 'model': model, + 'session': model.Session, + 'user': '', + } + + # Make copies of the data and errors dicts for asserting later. + original_data = copy.deepcopy(data) + original_errors = copy.deepcopy(errors) + + result = validators.user_name_validator(key, data, errors, context) + + assert result is None, ("user_name_validator() shouldn't return " + "anything") + + msg = 'That login name is not available.' + assert errors[key] == [msg], ('user_name_validator() should add to ' + 'the errors dict when given the name of ' + 'a user that already exists') + + errors[key] = [] + assert errors == original_errors, ('user_name_validator() should not ' + 'modify other parts of the errors ' + 'dict') + + assert data == original_data, ('user_name_validator() should not ' + 'modify the data dict') + + def test_user_name_validator_successful(self): + '''user_name_validator() should do nothing if given a valid name.''' + + import ckan.logic.validators as validators + import ckan.model as model + + data = test_data.validator_data_dict() + key = ('name',) + data[key] = 'new_user_name' + errors = test_data.validator_errors_dict() + errors[key] = [] + context = { + 'model': model, + 'session': model.Session, + 'user': '', + } + + # Make copies of the data and errors dicts for asserting later. + original_data = copy.deepcopy(data) + original_errors = copy.deepcopy(errors) + + result = validators.user_name_validator(key, data, errors, context) + + assert result is None, ("user_name_validator() shouldn't return " + 'anything') + + assert data == original_data, ("user_name_validator shouldn't modify " + 'the data dict') + + assert errors == original_errors, ("user_name_validator shouldn't " + 'modify the errors dict') From 807b601f3ade09aac3f0ef0d530e947774e93975 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 19:16:57 +0200 Subject: [PATCH 033/269] [#1117] Fix user_password validator Commit f3165ce (user_update: don't crash on non-string password) unwittingly changed the behavior of user_password_validator(), causing some legacy tests to fail: If the given password is Missing or '' user_password_validator() should *not* raise Invalid. Fiux user_password_validator to not raise Invalid in these cases, so that the tests pass again. --- ckan/logic/validators.py | 4 +++- ckan/new_tests/logic/action/test_update.py | 20 ++++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 67a68d0424b..a18cf783d64 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -524,9 +524,11 @@ def user_password_validator(key, data, errors, context): value = data[key] if isinstance(value, Missing): - errors[('password',)].append(_('Your password must be 4 characters or longer')) + pass elif not isinstance(value, basestring): errors[('password',)].append(_('Passwords must be strings')) + elif value == '': + pass elif len(value) < 4: errors[('password',)].append(_('Your password must be 4 characters or longer')) diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index 9c8e95fcb78..46235fd778f 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -130,11 +130,23 @@ def test_user_update_with_short_password(self): helpers.call_action('user_update', **user) def test_user_update_with_empty_password(self): - user = helpers.call_action('user_create', **data.typical_user()) + '''If an empty password is passed to user_update, nothing should + happen. - user['password'] = '' - with nose.tools.assert_raises(logic.ValidationError) as context: - helpers.call_action('user_update', **user) + No error (e.g. a validation error) is raised, but the password is not + changed either. + + ''' + user_dict = data.typical_user() + original_password = user_dict['password'] + user_dict = helpers.call_action('user_create', **user_dict) + + user_dict['password'] = '' + helpers.call_action('user_update', **user_dict) + + import ckan.model as model + updated_user = model.User.get(user_dict['id']) + assert updated_user.validate_password(original_password) def test_user_update_with_null_password(self): user = helpers.call_action('user_create', **data.typical_user()) From 51cd5d18c4005f469c6052a6bddc3275b6373cde Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 19:29:02 +0200 Subject: [PATCH 034/269] [#1117] Tweak a docstring --- ckan/new_tests/data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/new_tests/data.py b/ckan/new_tests/data.py index c8b534a2a3c..c67058e2c93 100644 --- a/ckan/new_tests/data.py +++ b/ckan/new_tests/data.py @@ -8,6 +8,8 @@ def typical_user(): + '''Return a dictionary representation of a typical, valid CKAN user.''' + return {'name': 'fred', 'email': 'fred@fred.com', 'password': 'wilma', From c6b953496177ca795ac9347eaf7986edb3049ed7 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 19:32:07 +0200 Subject: [PATCH 035/269] [#1117] Refactor and add docstring to user_name_validator - Add docstring - Use model.User.get() to find whether a user exists, instead of a lot of SQLAlchemy in ckan.logic. This should make it easier to unit test user_name_validator() in isolation, because the tests will only have to mock one method ckan.model.User.get() instead of having to mock several things. Also SQLAlchemy should just be in the model anyway, not in the logic. - Refactor and add code comments to clarify the obscure thing that user_name_validator() does with context['user_obj'] on user_update()s. This was completely obscure before, now hopefully it's clearer. (But it's a bad design anyway, user_create and user_update shouldn't be sharing the same user_name_validator function.) I don't *think* I broke anything by refactoring this (tests are still passing). --- ckan/logic/validators.py | 42 ++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index a18cf783d64..a32d3ce3d28 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -493,23 +493,37 @@ def ignore_not_group_admin(key, data, errors, context): data.pop(key) def user_name_validator(key, data, errors, context): - model = context["model"] - session = context["session"] - user = context.get("user_obj") + '''Validate a new user name. + + Append an error message to ``errors[key]`` if a user named ``data[key]`` + already exists. Otherwise, do nothing. + + :raises ckan.lib.navl.dictization_functions.Invalid: if ``data[key]`` is + not a string + :rtype: None + + ''' + model = context['model'] + new_user_name = data[key] - if not isinstance(data[key], basestring): + if not isinstance(new_user_name, basestring): raise Invalid(_('User names must be strings')) - query = session.query(model.User.name).filter_by(name=data[key]) - if user: - user_id = user.id - else: - user_id = data.get(key[:-1] + ("id",)) - if user_id and user_id is not missing: - query = query.filter(model.User.id <> user_id) - result = query.first() - if result: - errors[key].append(_('That login name is not available.')) + user = model.User.get(new_user_name) + if user is not None: + # A user with new_user_name already exists in the database. + + user_obj_from_context = context.get('user_obj') + if user_obj_from_context and user_obj_from_context.id == user.id: + # If there's a user_obj in context with the same id as the user + # found in the db, then we must be doing a user_update and not + # updating the user name, so don't return an error. + return + else: + # Otherwise return an error: there's already another user with that + # name, so you can create a new user with that name or update an + # existing user's name to that name. + errors[key].append(_('That login name is not available.')) def user_both_passwords_entered(key, data, errors, context): From 93d0d819cc68cc3e84b472eefdf4fa3f26f32c07 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 26 Jul 2013 20:21:35 +0200 Subject: [PATCH 036/269] [#1117] Mock ckan.model in user_name_validator() tests Now that user_name_validator() has been refactored to call model.User.get() instead of doing its own SQLAlchemy (commit c6b953), it's really easy to mock ckan.model in the user_name_validator() unit tests by just mocking the single method ckan.model.User.get(). This means the user_name_validator() unit tests no longer touch the disk or db or bring in ckan.model. --- ckan/new_tests/logic/test_validators.py | 56 +++++++++++++------------ 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/ckan/new_tests/logic/test_validators.py b/ckan/new_tests/logic/test_validators.py index d6556532347..712f3787672 100644 --- a/ckan/new_tests/logic/test_validators.py +++ b/ckan/new_tests/logic/test_validators.py @@ -4,6 +4,7 @@ ''' import copy +import mock import nose.tools import ckan.new_tests.helpers as helpers @@ -109,7 +110,6 @@ def test_user_name_validator_with_non_string_value(self): ''' import ckan.logic.validators as validators - import ckan.model as model import ckan.lib.navl.dictization_functions as df non_string_values = [ @@ -126,13 +126,12 @@ def test_user_name_validator_with_non_string_value(self): lambda x: x**2, ] - key = ('name',) - context = { - 'model': model, - 'session': model.Session, - 'user': '', - } + # Mock ckan.model. + mock_model = mock.MagicMock() + # model.User.get(some_user_id) needs to return None for this test. + mock_model.User.get.return_value = None + key = ('name',) for non_string_value in non_string_values: data = test_data.validator_data_dict() data[key] = non_string_value @@ -144,7 +143,8 @@ def test_user_name_validator_with_non_string_value(self): original_errors = copy.deepcopy(errors) with nose.tools.assert_raises(df.Invalid): - validators.user_name_validator(key, data, errors, context) + validators.user_name_validator(key, data, errors, + context={'model': mock_model}) assert data == original_data, ("user_name_validator shouldn't " 'modify the data dict') @@ -158,28 +158,25 @@ def test_user_name_validator_with_a_name_that_already_exists(self): ''' import ckan.logic.validators as validators - import ckan.model as model - existing_user = helpers.call_action('user_create', - **test_data.typical_user()) + # Mock ckan.model. model.User.get('user_name') will return another mock + # object rather than None, which will simulate an existing user with + # the same user name in the database. + mock_model = mock.MagicMock() - # Try to create another user with the same name as the existing user. data = test_data.validator_data_dict() key = ('name',) - data[key] = existing_user['name'] + data[key] = 'user_name' errors = test_data.validator_errors_dict() errors[key] = [] - context = { - 'model': model, - 'session': model.Session, - 'user': '', - } # Make copies of the data and errors dicts for asserting later. original_data = copy.deepcopy(data) original_errors = copy.deepcopy(errors) - result = validators.user_name_validator(key, data, errors, context) + # Try to create another user with the same name as the existing user. + result = validators.user_name_validator(key, data, errors, + context={'model': mock_model}) assert result is None, ("user_name_validator() shouldn't return " "anything") @@ -201,24 +198,25 @@ def test_user_name_validator_successful(self): '''user_name_validator() should do nothing if given a valid name.''' import ckan.logic.validators as validators - import ckan.model as model data = test_data.validator_data_dict() key = ('name',) data[key] = 'new_user_name' errors = test_data.validator_errors_dict() errors[key] = [] - context = { - 'model': model, - 'session': model.Session, - 'user': '', - } + + # Mock ckan.model. + mock_model = mock.MagicMock() + # model.User.get(user_name) should return None, to simulate that no + # user with that name exists in the database. + mock_model.User.get.return_value = None # Make copies of the data and errors dicts for asserting later. original_data = copy.deepcopy(data) original_errors = copy.deepcopy(errors) - result = validators.user_name_validator(key, data, errors, context) + result = validators.user_name_validator(key, data, errors, + context={'model': mock_model}) assert result is None, ("user_name_validator() shouldn't return " 'anything') @@ -227,4 +225,8 @@ def test_user_name_validator_successful(self): 'the data dict') assert errors == original_errors, ("user_name_validator shouldn't " - 'modify the errors dict') + 'modify the errors dict if given a' + 'valid user name') + + # TODO: Test user_name_validator()'s behavior when there's a 'user_obj' in + # the context dict. From eb6fd0a5ab53f60b0a43a797abcd62a8a45d84f0 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 30 Jul 2013 15:54:21 +0200 Subject: [PATCH 037/269] [#1117] Add factory_boy to action tests Add a new module ckan.new_tests.factories with (so far) just a User factory class that uses the factory_boy library to create CKAN users. Update the ckan.new_tests.logic.action.test_update tests to use this factory instead of ckan.new_tests.data. Still need to update the rest of the new tests to use the factory (and also add factories for other classes such as dataset etc.), then data.py can be deleted. Still need to add factory_boy to dev-requirements.txt. --- ckan/new_tests/factories.py | 80 ++++++++++++++++++++++ ckan/new_tests/logic/action/test_update.py | 38 +++++----- 2 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 ckan/new_tests/factories.py diff --git a/ckan/new_tests/factories.py b/ckan/new_tests/factories.py new file mode 100644 index 00000000000..e338c1caa1b --- /dev/null +++ b/ckan/new_tests/factories.py @@ -0,0 +1,80 @@ +'''A collection of factory classes for building CKAN users, datasets, etc. + +These are meant to be used by tests to create any objects or "test fixtures" +that are needed for the tests. They're written using factory_boy: + + http://factoryboy.readthedocs.org/en/latest/ + +These are not meant to be used for the actual testing, e.g. if you're writing a +test for the user_create action function then call user_create, don't test it +via the User factory below. + +Usage: + + # Create a user with the factory's default attributes, and get back a + # user dict: + user_dict = factories.User() + + # You can create a second user the same way. For attributes that can't be + # the same (e.g. you can't have two users with the same name) a new value + # will be generated each time you use the factory: + another_user_dict = factories.User() + + # Create a user and specify your own user name and email (this works + # with any params that CKAN's user_create() accepts): + custom_user_dict = factories.User(name='bob', email='bob@bob.com') + + # Get a user dict containing the attributes (name, email, password, etc.) + # that the factory would use to create a user, but without actually + # creating the user in CKAN: + user_attributes_dict = factories.User.attributes() + + # If you later want to create a user using these attributes, just pass them + # to the factory: + user = factories.User(**user_attributes_dict) + +''' +import factory + +import ckan.model +import ckan.logic +import ckan.new_tests.helpers as helpers + + +class User(factory.Factory): + '''A factory class for creating CKAN users.''' + + # This is the class that UserFactory will create and return instances + # of. + FACTORY_FOR = ckan.model.User + + # These are the default params that will be used to create new users. + fullname = 'Mr. Test User' + password = 'pass' + about = 'Just another test user.' + + # Generate a different user name param for each user that gets created. + name = factory.Sequence( + lambda n: 'test_user_{n}'.format(n=n)) + + # Compute the email param for each user based on the values of the other + # params above. + email = factory.LazyAttribute( + lambda a: '{0}@ckan.org'.format(a.name).lower()) + + # I'm not sure how to support factory_boy's .build() feature in CKAN, + # so I've disabled it here. + @classmethod + def _build(cls, target_class, *args, **kwargs): + raise NotImplementedError(".build() isn't supported in CKAN") + + # To make factory_boy work with CKAN we override _create() and make it call + # a CKAN action function. + # We might also be able to do this by using factory_boy's direct SQLAlchemy + # support: http://factoryboy.readthedocs.org/en/latest/orms.html#sqlalchemy + @classmethod + def _create(cls, target_class, *args, **kwargs): + if args: + assert False, "Positional args aren't supported, use keyword args." + user_dict = helpers.call_action('user_create', **kwargs) + return user_dict diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index 46235fd778f..d94a930ae0f 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -6,7 +6,7 @@ import ckan.logic as logic import ckan.new_tests.helpers as helpers -import ckan.new_tests.data as data +import ckan.new_tests.factories as factories def datetime_from_string(s): @@ -49,7 +49,7 @@ def test_user_update_name(self): # 4. Do absolutely nothing else! # 1. Setup. - user = helpers.call_action('user_create', **data.typical_user()) + user = factories.User() # 2. Call the function that is being tested, once only. # FIXME we have to pass the email address and password to user_update @@ -57,7 +57,7 @@ def test_user_update_name(self): # fails. helpers.call_action('user_update', id=user['name'], email=user['email'], - password=data.typical_user()['password'], + password=factories.User.attributes()['password'], name='updated', ) @@ -70,21 +70,22 @@ def test_user_update_name(self): # 4. Do absolutely nothing else! def test_user_update_with_id_that_does_not_exist(self): - user_dict = data.typical_user() + user_dict = factories.User.attributes() user_dict['id'] = "there's no user with this id" + with nose.tools.assert_raises(logic.NotFound) as context: helpers.call_action('user_update', **user_dict) # TODO: Could assert the actual error message, not just the exception? # (Could also do this with many of the tests below.) def test_user_update_with_no_id(self): - user_dict = data.typical_user() + user_dict = factories.User.attributes() assert 'id' not in user_dict with nose.tools.assert_raises(logic.ValidationError) as context: helpers.call_action('user_update', **user_dict) def test_user_update_with_invalid_name(self): - user = helpers.call_action('user_create', **data.typical_user()) + user = factories.User() invalid_names = ('', 'a', False, 0, -1, 23, 'new', 'edit', 'search', 'a'*200, 'Hi!', ) @@ -94,9 +95,8 @@ def test_user_update_with_invalid_name(self): helpers.call_action('user_update', **user) def test_user_update_to_name_that_already_exists(self): - fred = helpers.call_action('user_create', **data.typical_user()) - bob = helpers.call_action('user_create', name='bob', - email='bob@bob.com', password='pass') + fred = factories.User(name='fred') + bob = factories.User(name='bob') # Try to update fred and change his user name to bob, which is already # bob's user name @@ -107,7 +107,7 @@ def test_user_update_to_name_that_already_exists(self): def test_user_update_password(self): '''Test that updating a user's password works successfully.''' - user = helpers.call_action('user_create', **data.typical_user()) + user = factories.User() # FIXME we have to pass the email address to user_update even though # we're not updating it, otherwise validation fails. @@ -123,7 +123,7 @@ def test_user_update_password(self): assert updated_user.validate_password('new password') def test_user_update_with_short_password(self): - user = helpers.call_action('user_create', **data.typical_user()) + user = factories.User() user['password'] = 'xxx' # This password is too short. with nose.tools.assert_raises(logic.ValidationError) as context: @@ -137,9 +137,9 @@ def test_user_update_with_empty_password(self): changed either. ''' - user_dict = data.typical_user() + user_dict = factories.User.attributes() original_password = user_dict['password'] - user_dict = helpers.call_action('user_create', **user_dict) + user_dict = factories.User(**user_dict) user_dict['password'] = '' helpers.call_action('user_update', **user_dict) @@ -149,14 +149,14 @@ def test_user_update_with_empty_password(self): assert updated_user.validate_password(original_password) def test_user_update_with_null_password(self): - user = helpers.call_action('user_create', **data.typical_user()) + user = factories.User() user['password'] = None with nose.tools.assert_raises(logic.ValidationError) as context: helpers.call_action('user_update', **user) def test_user_update_with_invalid_password(self): - user = helpers.call_action('user_create', **data.typical_user()) + user = factories.User() for password in (False, -1, 23, 30.7): user['password'] = password @@ -168,7 +168,7 @@ def test_user_update_with_invalid_password(self): def test_user_update_activity_stream(self): '''Test that the right activity is emitted when updating a user.''' - user = helpers.call_action('user_create', **data.typical_user()) + user = factories.User() before = datetime.datetime.now() # FIXME we have to pass the email address and password to user_update @@ -176,7 +176,7 @@ def test_user_update_activity_stream(self): # fails. helpers.call_action('user_update', id=user['name'], email=user['email'], - password=data.typical_user()['password'], + password=factories.User.attributes()['password'], name='updated', ) @@ -203,7 +203,7 @@ def test_user_update_with_custom_schema(self): ''' import ckan.logic.schema - user = helpers.call_action('user_create', **data.typical_user()) + user = factories.User() # A mock validator method, it doesn't do anything but it records what # params it gets called with and how many times. @@ -219,7 +219,7 @@ def test_user_update_with_custom_schema(self): # trying to update them, or validation fails. helpers.call_action('user_update', context={'schema': schema}, id=user['name'], email=user['email'], - password=data.typical_user()['password'], + password=factories.User.attributes()['password'], name='updated', ) From c556fabfd5c43b3ff0e775462b0831f1d59cee99 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 30 Jul 2013 16:20:25 +0200 Subject: [PATCH 038/269] [#1117] Use factory in auth/test_update.py Instead of data.py. --- ckan/new_tests/logic/auth/test_update.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ckan/new_tests/logic/auth/test_update.py b/ckan/new_tests/logic/auth/test_update.py index 49bf163178f..be58fe89407 100644 --- a/ckan/new_tests/logic/auth/test_update.py +++ b/ckan/new_tests/logic/auth/test_update.py @@ -2,7 +2,7 @@ ''' import ckan.new_tests.helpers as helpers -import ckan.new_tests.data as data +import ckan.new_tests.factories as factories class TestUpdate(object): @@ -35,7 +35,7 @@ def _call_auth(self, auth_name, context=None, **kwargs): def test_user_update_visitor_cannot_update_user(self): '''Visitors should not be able to update users' accounts.''' - user = helpers.call_action('user_create', **data.typical_user()) + user = factories.User() user['name'] = 'updated' # Try to update the user, but without passing any API key. @@ -49,9 +49,8 @@ def test_user_update_visitor_cannot_update_user(self): def test_user_update_user_cannot_update_another_user(self): '''Users should not be able to update other users' accounts.''' - fred = helpers.call_action('user_create', **data.typical_user()) - bob = helpers.call_action('user_create', name='bob', - email='bob@bob.com', password='pass') + fred = factories.User(name='fred') + bob = factories.User(name='bob') fred['name'] = 'updated' # Make Bob try to update Fred's user account. @@ -62,7 +61,7 @@ def test_user_update_user_cannot_update_another_user(self): def test_user_update_user_can_update_herself(self): '''Users should be authorized to update their own accounts.''' - user = helpers.call_action('user_create', **data.typical_user()) + user = factories.User() context = {'user': user['name']} user['name'] = 'updated' From a5fe9e6e231dbc64f729427323c677d415c60744 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 30 Jul 2013 16:25:15 +0200 Subject: [PATCH 039/269] [#1117] Remove data.py from test_validators.py Move the helper functions it was using into factories.py and make it use that instead. --- ckan/new_tests/factories.py | 16 ++++++++++++++++ ckan/new_tests/logic/test_validators.py | 14 +++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/ckan/new_tests/factories.py b/ckan/new_tests/factories.py index e338c1caa1b..b68fa7789f7 100644 --- a/ckan/new_tests/factories.py +++ b/ckan/new_tests/factories.py @@ -78,3 +78,19 @@ def _create(cls, target_class, *args, **kwargs): assert False, "Positional args aren't supported, use keyword args." user_dict = helpers.call_action('user_create', **kwargs) return user_dict + + +def validator_data_dict(): + '''Return a data dict with some arbitrary data in it, suitable to be passed + to validator functions for testing. + + ''' + return {('other key',): 'other value'} + + +def validator_errors_dict(): + '''Return an errors dict with some arbitrary errors in it, suitable to be + passed to validator functions for testing. + + ''' + return {('other key',): ['other error']} diff --git a/ckan/new_tests/logic/test_validators.py b/ckan/new_tests/logic/test_validators.py index 712f3787672..20347575db0 100644 --- a/ckan/new_tests/logic/test_validators.py +++ b/ckan/new_tests/logic/test_validators.py @@ -8,7 +8,7 @@ import nose.tools import ckan.new_tests.helpers as helpers -import ckan.new_tests.data as test_data +import ckan.new_tests.factories as factories class TestValidators(object): @@ -133,9 +133,9 @@ def test_user_name_validator_with_non_string_value(self): key = ('name',) for non_string_value in non_string_values: - data = test_data.validator_data_dict() + data = factories.validator_data_dict() data[key] = non_string_value - errors = test_data.validator_errors_dict() + errors = factories.validator_errors_dict() errors[key] = [] # Make copies of the data and errors dicts for asserting later. @@ -164,10 +164,10 @@ def test_user_name_validator_with_a_name_that_already_exists(self): # the same user name in the database. mock_model = mock.MagicMock() - data = test_data.validator_data_dict() + data = factories.validator_data_dict() key = ('name',) data[key] = 'user_name' - errors = test_data.validator_errors_dict() + errors = factories.validator_errors_dict() errors[key] = [] # Make copies of the data and errors dicts for asserting later. @@ -199,10 +199,10 @@ def test_user_name_validator_successful(self): import ckan.logic.validators as validators - data = test_data.validator_data_dict() + data = factories.validator_data_dict() key = ('name',) data[key] = 'new_user_name' - errors = test_data.validator_errors_dict() + errors = factories.validator_errors_dict() errors[key] = [] # Mock ckan.model. From fe8eec0f4bde1f4b23d71d49584e434c78148d91 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 30 Jul 2013 16:29:09 +0200 Subject: [PATCH 040/269] [#1117] Use factories instead of data in navl/test_validators.py So data.py is now unused and we can remove it. --- ckan/new_tests/data.py | 32 ---------------------- ckan/new_tests/lib/navl/test_validators.py | 10 +++---- 2 files changed, 5 insertions(+), 37 deletions(-) delete mode 100644 ckan/new_tests/data.py diff --git a/ckan/new_tests/data.py b/ckan/new_tests/data.py deleted file mode 100644 index c67058e2c93..00000000000 --- a/ckan/new_tests/data.py +++ /dev/null @@ -1,32 +0,0 @@ -'''A collection of static data for use in tests. - -These are functions that return objects because the data is intended to be -read-only: if they were simply module-level objects then one test may modify -eg. a dict or list and then break the tests that follow it. - -''' - - -def typical_user(): - '''Return a dictionary representation of a typical, valid CKAN user.''' - - return {'name': 'fred', - 'email': 'fred@fred.com', - 'password': 'wilma', - } - - -def validator_data_dict(): - '''Return a data dict with some arbitrary data in it, suitable to be passed - to validator functions for testing. - - ''' - return {('other key',): 'other value'} - - -def validator_errors_dict(): - '''Return an errors dict with some arbitrary errors in it, suitable to be - passed to validator functions for testing. - - ''' - return {('other key',): ['other error']} diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index 51df87da648..cbebfddd467 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -6,7 +6,7 @@ import nose.tools -import ckan.new_tests.data as test_data +import ckan.new_tests.factories as factories class TestValidators(object): @@ -28,12 +28,12 @@ def test_ignore_missing_with_value_missing(self): key = ('key to be validated',) # The data to pass to the validator function for validation. - data = test_data.validator_data_dict() + data = factories.validator_data_dict() if value != 'skip': data[key] = value # The errors dict to pass to the validator function. - errors = test_data.validator_errors_dict() + errors = factories.validator_errors_dict() errors[key] = [] # Make copies of the data and errors dicts for asserting later. @@ -66,9 +66,9 @@ def test_ignore_missing_with_a_value(self): import ckan.lib.navl.validators as validators key = ('key to be validated',) - data = test_data.validator_data_dict() + data = factories.validator_data_dict() data[key] = 'value to be validated' - errors = test_data.validator_errors_dict() + errors = factories.validator_errors_dict() errors[key] = [] # Make copies of the data and errors dicts for asserting later. From 3e7f6900882804a1a0b9560333654306f4e8cee1 Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 30 Jul 2013 15:30:27 +0100 Subject: [PATCH 041/269] [#1126] First run at new homepage modules core extension --- ckan/public/base/less/homepage.less | 112 +++++++----------- ckan/templates/home/index.html | 93 +-------------- ckanext/homepage/__init__.py | 0 ckanext/homepage/plugin.py | 24 ++++ .../homepage/theme/templates/home/index.html | 36 ++++++ .../home/snippets/featured_group.html | 3 + .../home/snippets/featured_organization.html | 3 + .../templates/home/snippets/promoted.html | 20 ++++ .../theme/templates/home/snippets/search.html | 18 +++ setup.py | 1 + 10 files changed, 150 insertions(+), 160 deletions(-) create mode 100644 ckanext/homepage/__init__.py create mode 100644 ckanext/homepage/plugin.py create mode 100644 ckanext/homepage/theme/templates/home/index.html create mode 100644 ckanext/homepage/theme/templates/home/snippets/featured_group.html create mode 100644 ckanext/homepage/theme/templates/home/snippets/featured_organization.html create mode 100644 ckanext/homepage/theme/templates/home/snippets/promoted.html create mode 100644 ckanext/homepage/theme/templates/home/snippets/search.html diff --git a/ckan/public/base/less/homepage.less b/ckan/public/base/less/homepage.less index 0aaa8a839ab..02574845817 100644 --- a/ckan/public/base/less/homepage.less +++ b/ckan/public/base/less/homepage.less @@ -1,26 +1,24 @@ -.hero { - background: url("@{imagePath}/background-tile.png"); - padding: 20px 0; - min-height: 0; - > .container { - position: relative; - padding-bottom: 0; - } - .search-giant { - margin-bottom: 10px; - input { - border-color: darken(@mastheadBackgroundColorEnd, 5); - } +.homepage { + + [role=main] { + padding: 20px 0; } - .page-heading { - font-size: 18px; - margin-bottom: 0; + + .row { + position: relative; } - .module-dark { + + .module-search { padding: 5px; margin-bottom: 0; color: @mastheadTextColor; background: @layoutTrimBackgroundColor; + .search-giant { + margin-bottom: 10px; + input { + border-color: darken(@mastheadBackgroundColorEnd, 5); + } + } .module-content { .border-radius(3px 3px 0 0); background-color: @mastheadBackgroundColor; @@ -32,66 +30,38 @@ line-height: 40px; } } - } - .tags { - .clearfix(); - padding: 5px 10px 10px 10px; - background-color: darken(@mastheadBackgroundColor, 10%); - .border-radius(0 0 3px 3px); - h3, - .tag { - display: block; - float: left; - margin: 5px 10px 0 0; - } - h3 { - font-size: @baseFontSize; - line-height: @baseLineHeight; - padding: 2px 8px; + .tags { + .clearfix(); + padding: 5px 10px 10px 10px; + background-color: darken(@mastheadBackgroundColor, 10%); + .border-radius(0 0 3px 3px); + h3, + .tag { + display: block; + float: left; + margin: 5px 10px 0 0; + } + h3 { + font-size: @baseFontSize; + line-height: @baseLineHeight; + padding: 2px 8px; + } } } -} -.hero-primary, -.hero-secondary { - .makeColumn(6); -} - -.hero-primary { - margin-left: 0; // Remove grid margin. - margin-bottom: 0; -} + .group-list { + margin: 0; + } -.hero-secondary { - position: absolute; - bottom: 0; - right: 0; - .hero-secondary-inner { + .slot2 { + position: absolute; bottom: 0; - left: 0; right: 0; - } -} - -.main.homepage { - padding-top: 20px; - padding-bottom: 20px; - border-top: 1px solid @layoutTrimBorderColor; - .module-heading .media-image { - margin-right: 15px; - max-height: 53px; - .box-shadow(0 0 0 2px rgba(255, 255, 255, 0.5)); - } - .group-listing .box { - min-height: 275px; - } - .group-list { - margin-bottom: 0; - .dataset-content { - min-height: 70px; + .module-search { + bottom: 0; + left: 0; + right: 0; } } - .box .module { - margin-top: 0; - } + } diff --git a/ckan/templates/home/index.html b/ckan/templates/home/index.html index 2508f8b8508..ee1d1f4c8ae 100644 --- a/ckan/templates/home/index.html +++ b/ckan/templates/home/index.html @@ -3,98 +3,13 @@ {% block subtitle %}{{ _("Welcome") }}{% endblock %} {% block maintag %}{% endblock %} +{% block toolbar %}{% endblock %} {% block content %} -
-
- {{ self.flash() }} - {{ self.primary_content() }} -
-
-
+
- {{ self.secondary_content() }} -
-
-{% endblock %} - -{% block primary_content %} -
-
- {% block home_primary %} -
- {% if g.site_intro_text %} - {{ h.render_markdown(g.site_intro_text) }} - {% else %} - {% block home_primary_content %} -

{% block home_primary_heading %}{{ _("Welcome to CKAN") }}{% endblock %}

-

- {% block home_primary_text %} - {% trans %}This is a nice introductory paragraph about CKAN or the site - in general. We don't have any copy to go here yet but soon we will - {% endtrans %} - {% endblock %} -

- {% endblock %} - {% endif %} -
- {% endblock %} - - {% block home_image %} -
-

{% block home_image_caption %}{{ _("This is a featured section") }}{% endblock %}

- {% block home_image_content %} - - Placeholder - - {% endblock %} -
- {% endblock %} -
-
-
-
- {% block home_secondary_content %} -
- {% block home_search %} -
-

{{ _("Search Your Data") }}

-
- - -
-
- {% endblock %} - {% block home_tags %} -
-

{{ _('Popular Tags') }}

- {% set tags = h.get_facet_items_dict('tags', limit=3) %} - {% for tag in tags %} - {{ h.truncate(tag.display_name, 22) }} - {% endfor %} -
- {% endblock %} -
- {% endblock %} + {{ self.flash() }}
+ {% block primary_content %}{% endblock %}
{% endblock %} - -{% block secondary_content %} -
- {% for group in c.group_package_stuff %} -
-
- {% snippet 'snippets/group_item.html', group=group.group_dict, truncate=50, truncate_title=35 %} -
-
- {% endfor %} -
-{% endblock %} - -{# Remove the toolbar. #} - -{% block toolbar %}{% endblock %} diff --git a/ckanext/homepage/__init__.py b/ckanext/homepage/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/homepage/plugin.py b/ckanext/homepage/plugin.py new file mode 100644 index 00000000000..ef429f0457d --- /dev/null +++ b/ckanext/homepage/plugin.py @@ -0,0 +1,24 @@ +import logging + +import ckan.plugins as p + +log = logging.getLogger(__name__) + +class HomepagePlugin(p.SingletonPlugin): + p.implements(p.IConfigurer, inherit=True) + p.implements(p.IConfigurable, inherit=True) + + def update_config(self, config): + p.toolkit.add_template_directory(config, 'theme/templates') + + def get_featured_organization(self): + return + + def get_featured_group(self): + return + + def get_helpers(self): + return { + 'get_featured_organization': self.get_featured_organization, + 'get_featured_group': self.get_featured_group, + } diff --git a/ckanext/homepage/theme/templates/home/index.html b/ckanext/homepage/theme/templates/home/index.html new file mode 100644 index 00000000000..0a9d0de727b --- /dev/null +++ b/ckanext/homepage/theme/templates/home/index.html @@ -0,0 +1,36 @@ +{% ckan_extends %} + +{% block primary_content %} +
+
+
+
{{ self.homepage_slot_1() }}
+
{{ self.homepage_slot_2() }}
+
+
+
+
+
+
+
{{ self.homepage_slot_3() }}
+
{{ self.homepage_slot_4() }}
+
+
+
+{% endblock %} + +{% block homepage_slot_1 %} + {% snippet 'home/snippets/promoted.html', intro=g.site_intro_text %} +{% endblock %} + +{% block homepage_slot_2 %} + {% snippet 'home/snippets/search.html', query=c.q, tags=h.get_facet_items_dict('tags', limit=3), placeholder=_('eg. Gold Prices') %} +{% endblock %} + +{% block homepage_slot_3 %} + {% snippet 'home/snippets/featured_group.html', group=h.get_featured_group() %} +{% endblock %} + +{% block homepage_slot_4 %} + {% snippet 'home/snippets/featured_organization.html', organization=h.get_featured_organization() %} +{% endblock %} diff --git a/ckanext/homepage/theme/templates/home/snippets/featured_group.html b/ckanext/homepage/theme/templates/home/snippets/featured_group.html new file mode 100644 index 00000000000..bdd20fafdbe --- /dev/null +++ b/ckanext/homepage/theme/templates/home/snippets/featured_group.html @@ -0,0 +1,3 @@ +
+ {% snippet 'snippets/group_item.html', group=group, truncate=50, truncate_title=35 %} +
diff --git a/ckanext/homepage/theme/templates/home/snippets/featured_organization.html b/ckanext/homepage/theme/templates/home/snippets/featured_organization.html new file mode 100644 index 00000000000..760f8b5e479 --- /dev/null +++ b/ckanext/homepage/theme/templates/home/snippets/featured_organization.html @@ -0,0 +1,3 @@ +
+ {% snippet 'snippets/organization_item.html', organization=organization, truncate=50, truncate_title=35 %} +
diff --git a/ckanext/homepage/theme/templates/home/snippets/promoted.html b/ckanext/homepage/theme/templates/home/snippets/promoted.html new file mode 100644 index 00000000000..287083a51b0 --- /dev/null +++ b/ckanext/homepage/theme/templates/home/snippets/promoted.html @@ -0,0 +1,20 @@ +
+
+ {% if intro %} + {{ h.render_markdown(intro) }} + {% else %} +

{{ _("Welcome to CKAN") }}

+

+ {% trans %}This is a nice introductory paragraph about CKAN or the site + in general. We don't have any copy to go here yet but soon we will + {% endtrans %} +

+ {% endif %} +
+
+

{{ _("This is a featured section") }}

+ + Placeholder + +
+
diff --git a/ckanext/homepage/theme/templates/home/snippets/search.html b/ckanext/homepage/theme/templates/home/snippets/search.html new file mode 100644 index 00000000000..586ae5dba6a --- /dev/null +++ b/ckanext/homepage/theme/templates/home/snippets/search.html @@ -0,0 +1,18 @@ +
+
+

{{ _("Search Your Data") }}

+
+ + +
+
+
+

{{ _('Popular Tags') }}

+ {% for tag in tags %} + {{ h.truncate(tag.display_name, 22) }} + {% endfor %} +
+
diff --git a/setup.py b/setup.py index f4e97942047..958db3cf788 100644 --- a/setup.py +++ b/setup.py @@ -173,6 +173,7 @@ [ckan.system_plugins] domain_object_mods = ckan.model.modification:DomainObjectModificationExtension + homepage = ckanext.homepage.plugin:HomepagePlugin [ckan.test_plugins] routes_plugin=tests.ckantestplugins:RoutesPlugin From 7910f78ce23a049eb5cdd53efa1f2f117f65b0b9 Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 31 Jul 2013 10:31:39 +0100 Subject: [PATCH 042/269] [#1126] Added config for sysadmin area --- ckan/templates/admin/config.html | 6 +++++- .../theme/templates/admin/config.html | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 ckanext/homepage/theme/templates/admin/config.html diff --git a/ckan/templates/admin/config.html b/ckan/templates/admin/config.html index 0ec27d1e687..64fd8558b28 100644 --- a/ckan/templates/admin/config.html +++ b/ckan/templates/admin/config.html @@ -4,7 +4,9 @@ {% block primary_content_inner %}
- {{ autoform.generate(form_items, data, errors) }} + {% block admin_form %} + {{ autoform.generate(form_items, data, errors) }} + {% endblock %}
{% set locale = h.dump_json({'content': _('Are you sure you want to reset the config?')}) %} {{ _('Reset') }} @@ -20,6 +22,7 @@

{{ _('CKAN config options') }}

+ {% block admin_form_help %} {% set about_url = h.url_for(controller='home', action='about') %} {% set home_url = h.url_for(controller='home', action='index') %} {% set docs_url = "http://docs.ckan.org/en/{0}/theming.html".format(g.ckan_doc_version) %} @@ -39,6 +42,7 @@

the templates more fully we recommend reading the documentation.

{% endtrans %} + {% endblock %}

{% endblock %} diff --git a/ckanext/homepage/theme/templates/admin/config.html b/ckanext/homepage/theme/templates/admin/config.html new file mode 100644 index 00000000000..86e85b47714 --- /dev/null +++ b/ckanext/homepage/theme/templates/admin/config.html @@ -0,0 +1,19 @@ +{% ckan_extends %} + +{% import 'macros/form.html' as form %} + +{% block admin_form %} + {{ super() }} + {{ form.select('homepage_layout', label=_('Homepage'), options=[ + {'value': 1, 'text': 'Introductory area, search, featured group and featured organization'}, + {'value': 2, 'text': 'Search, featured dataset, stats and featured organization'} + ], selected=1, error=false) }} +{% endblock %} + +{% block admin_form_help %} + {{ super() }} + {% trans %} +

Homepage: This is for choosing a predefined layout for + the modules that appear on your homepage.

+ {% endtrans %} +{% endblock %} From a957533216a07e6e03daabb4a78cdd0a17f6695f Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 31 Jul 2013 10:42:28 +0100 Subject: [PATCH 043/269] [#1126] Simplified snippets and added temporary homepage layour variable --- ckan/public/base/less/homepage.less | 6 ++- ckan/templates/home/index.html | 2 +- .../homepage/theme/templates/home/index.html | 44 +++++++++++++------ .../home/snippets/featured_group.html | 2 + .../home/snippets/featured_organization.html | 2 + .../templates/home/snippets/promoted.html | 2 + .../theme/templates/home/snippets/search.html | 5 ++- 7 files changed, 45 insertions(+), 18 deletions(-) diff --git a/ckan/public/base/less/homepage.less b/ckan/public/base/less/homepage.less index 02574845817..fc185cb3098 100644 --- a/ckan/public/base/less/homepage.less +++ b/ckan/public/base/less/homepage.less @@ -53,7 +53,10 @@ margin: 0; } - .slot2 { +} + +.homepage-1 { + .row1 .col2 { position: absolute; bottom: 0; right: 0; @@ -63,5 +66,4 @@ right: 0; } } - } diff --git a/ckan/templates/home/index.html b/ckan/templates/home/index.html index ee1d1f4c8ae..d9e4abfb17b 100644 --- a/ckan/templates/home/index.html +++ b/ckan/templates/home/index.html @@ -6,7 +6,7 @@ {% block toolbar %}{% endblock %} {% block content %} -
+
{{ self.flash() }}
diff --git a/ckanext/homepage/theme/templates/home/index.html b/ckanext/homepage/theme/templates/home/index.html index 0a9d0de727b..bf9addfcb4b 100644 --- a/ckanext/homepage/theme/templates/home/index.html +++ b/ckanext/homepage/theme/templates/home/index.html @@ -1,36 +1,52 @@ {% ckan_extends %} +{% set homepage_layout = 1 %} + +{% block homepage_layout_class %} homepage-{{ homepage_layout }}{% endblock %} + {% block primary_content %}
-
-
{{ self.homepage_slot_1() }}
-
{{ self.homepage_slot_2() }}
+
+
{{ self.row1_col1() }}
+
{{ self.row1_col2() }}
-
-
{{ self.homepage_slot_3() }}
-
{{ self.homepage_slot_4() }}
+
+
{{ self.row2_col1() }}
+
{{ self.row2_col2() }}
{% endblock %} -{% block homepage_slot_1 %} - {% snippet 'home/snippets/promoted.html', intro=g.site_intro_text %} +{% block row1_col1 %} + {% if homepage_layout == 1 %} + {% snippet 'home/snippets/promoted.html' %} + {% elif homepage_layout == 2 %} + {% snippet 'home/snippets/search.html' %} + {% endif %} {% endblock %} -{% block homepage_slot_2 %} - {% snippet 'home/snippets/search.html', query=c.q, tags=h.get_facet_items_dict('tags', limit=3), placeholder=_('eg. Gold Prices') %} +{% block row1_col2 %} + {% if homepage_layout == 1 %} + {% snippet 'home/snippets/search.html' %} + {% elif homepage_layout == 2 %} + {% snippet 'home/snippets/featured_dataset.html' %} + {% endif %} {% endblock %} -{% block homepage_slot_3 %} - {% snippet 'home/snippets/featured_group.html', group=h.get_featured_group() %} +{% block row2_col1 %} + {% if homepage_layout == 1 %} + {% snippet 'home/snippets/featured_group.html' %} + {% elif homepage_layout == 2 %} + {% snippet 'home/snippets/stats.html' %} + {% endif %} {% endblock %} -{% block homepage_slot_4 %} - {% snippet 'home/snippets/featured_organization.html', organization=h.get_featured_organization() %} +{% block row2_col2 %} + {% snippet 'home/snippets/featured_organization.html' %} {% endblock %} diff --git a/ckanext/homepage/theme/templates/home/snippets/featured_group.html b/ckanext/homepage/theme/templates/home/snippets/featured_group.html index bdd20fafdbe..fb5ff22a640 100644 --- a/ckanext/homepage/theme/templates/home/snippets/featured_group.html +++ b/ckanext/homepage/theme/templates/home/snippets/featured_group.html @@ -1,3 +1,5 @@ +{% set group = h.get_featured_group() %} +
{% snippet 'snippets/group_item.html', group=group, truncate=50, truncate_title=35 %}
diff --git a/ckanext/homepage/theme/templates/home/snippets/featured_organization.html b/ckanext/homepage/theme/templates/home/snippets/featured_organization.html index 760f8b5e479..7e43e6c4bcb 100644 --- a/ckanext/homepage/theme/templates/home/snippets/featured_organization.html +++ b/ckanext/homepage/theme/templates/home/snippets/featured_organization.html @@ -1,3 +1,5 @@ +{% set organization = h.get_featured_organization() %} +
{% snippet 'snippets/organization_item.html', organization=organization, truncate=50, truncate_title=35 %}
diff --git a/ckanext/homepage/theme/templates/home/snippets/promoted.html b/ckanext/homepage/theme/templates/home/snippets/promoted.html index 287083a51b0..fc09d9a6300 100644 --- a/ckanext/homepage/theme/templates/home/snippets/promoted.html +++ b/ckanext/homepage/theme/templates/home/snippets/promoted.html @@ -1,3 +1,5 @@ +{% set intro = g.site_intro_text %} +
{% if intro %} diff --git a/ckanext/homepage/theme/templates/home/snippets/search.html b/ckanext/homepage/theme/templates/home/snippets/search.html index 586ae5dba6a..e4a72c7e074 100644 --- a/ckanext/homepage/theme/templates/home/snippets/search.html +++ b/ckanext/homepage/theme/templates/home/snippets/search.html @@ -1,8 +1,11 @@ +{% set tags = h.get_facet_items_dict('tags', limit=3) %} +{% set placeholder = _('eg. Gold Prices') %} +

{{ _("Search Your Data") }}

- +
diff --git a/ckan/templates/user/read_base.html b/ckan/templates/user/read_base.html index aa27a2b4119..f2b94e20782 100644 --- a/ckan/templates/user/read_base.html +++ b/ckan/templates/user/read_base.html @@ -76,6 +76,10 @@

{{ user.display_name }}

{{ _('Member Since') }}
{{ h.render_datetime(user.created) }}
+
+
{{ _('State') }}
+
{{ user.state }}
+
{% if c.is_myself %}
{{ _('API Key') }} {{ _('Private') }}
diff --git a/ckan/tests/functional/test_user.py b/ckan/tests/functional/test_user.py index 0ef528fd9ad..16795561f46 100644 --- a/ckan/tests/functional/test_user.py +++ b/ckan/tests/functional/test_user.py @@ -64,6 +64,25 @@ def test_user_read(self): 'rel="nofollow"') assert 'Edit Profile' not in main_res, main_res + def test_user_delete_redirects_to_user_index(self): + user = CreateTestData.create_user('a_user') + url = url_for(controller='user', action='delete', id=user.id) + extra_environ = {'REMOTE_USER': 'testsysadmin'} + + redirect_url = url_for(controller='user', action='index', + qualified=True) + res = self.app.get(url, status=302, extra_environ=extra_environ) + + assert user.is_deleted(), user + assert res.header('Location').startswith(redirect_url), res.header('Location') + + def test_user_delete_by_unauthorized_user(self): + user = model.User.by_name(u'annafan') + url = url_for(controller='user', action='delete', id=user.id) + extra_environ = {'REMOTE_USER': 'an_unauthorized_user'} + + self.app.get(url, status=401, extra_environ=extra_environ) + def test_user_read_without_id(self): offset = '/user/' res = self.app.get(offset, status=302) diff --git a/ckan/tests/lib/test_authenticator.py b/ckan/tests/lib/test_authenticator.py new file mode 100644 index 00000000000..c7c026112d2 --- /dev/null +++ b/ckan/tests/lib/test_authenticator.py @@ -0,0 +1,106 @@ +import ckan + +import ckan.lib.create_test_data as ctd +import ckan.lib.authenticator as authenticator + +CreateTestData = ctd.CreateTestData + + +class TestUsernamePasswordAuthenticator(object): + @classmethod + def setup_class(cls): + auth = authenticator.UsernamePasswordAuthenticator() + cls.authenticate = auth.authenticate + + @classmethod + def teardown(cls): + ckan.model.repo.rebuild_db() + + def test_authenticate_succeeds_if_login_and_password_are_correct(self): + environ = {} + password = 'somepass' + user = CreateTestData.create_user('a_user', **{'password': password}) + identity = {'login': user.name, 'password': password} + + username = self.authenticate(environ, identity) + assert username == user.name, username + + def test_authenticate_fails_if_user_is_deleted(self): + environ = {} + password = 'somepass' + user = CreateTestData.create_user('a_user', **{'password': password}) + identity = {'login': user.name, 'password': password} + user.delete() + + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_password_is_wrong(self): + environ = {} + user = CreateTestData.create_user('a_user') + identity = {'login': user.name, 'password': 'wrong-password'} + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_received_no_login_or_pass(self): + environ = {} + identity = {} + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_received_just_login(self): + environ = {} + identity = {'login': 'some-user'} + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_received_just_password(self): + environ = {} + identity = {'password': 'some-password'} + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_user_doesnt_exist(self): + environ = {} + identity = {'login': 'inexistent-user'} + assert self.authenticate(environ, identity) is None + + +class TestOpenIDAuthenticator(object): + @classmethod + def setup_class(cls): + auth = authenticator.OpenIDAuthenticator() + cls.authenticate = auth.authenticate + + @classmethod + def teardown(cls): + ckan.model.repo.rebuild_db() + + def test_authenticate_succeeds_if_openid_is_correct(self): + environ = {} + openid = 'some-openid-key' + user = CreateTestData.create_user('a_user', **{'openid': openid}) + identity = {'login': user.name, + 'repoze.who.plugins.openid.userid': openid} + + username = self.authenticate(environ, identity) + assert username == user.name, username + + def test_authenticate_fails_if_openid_is_incorrect(self): + environ = {} + openid = 'wrong-openid-key' + user = CreateTestData.create_user('a_user') + identity = {'login': user.name, + 'repoze.who.plugins.openid.userid': openid} + + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_user_is_deleted(self): + environ = {} + openid = 'some-openid-key' + user = CreateTestData.create_user('a_user', **{'openid': openid}) + user.delete() + identity = {'login': user.name, + 'repoze.who.plugins.openid.userid': openid} + + assert self.authenticate(environ, identity) is None + + def test_authenticate_fails_if_user_have_no_openid(self): + environ = {} + identity = {} + assert self.authenticate(environ, identity) is None diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py index fb8bb49e628..72ca6fa31e5 100644 --- a/ckan/tests/lib/test_dictization.py +++ b/ckan/tests/lib/test_dictization.py @@ -939,6 +939,7 @@ def test_16_group_dictized(self): 'users': [{'about': u'I love reading Annakarenina. My site: http://anna.com', 'display_name': u'annafan', 'capacity' : 'public', + 'state': 'active', 'sysadmin': False, 'email_hash': 'd41d8cd98f00b204e9800998ecf8427e', 'fullname': None, diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 73a3b838f0c..fecbdf96b53 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -7,6 +7,7 @@ from pylons import config import datetime +import vdm.sqlalchemy import ckan from ckan.lib.create_test_data import CreateTestData from ckan.lib.dictization.model_dictize import resource_dictize @@ -334,6 +335,11 @@ def test_42_create_resource_with_error(self): def test_04_user_list(self): + # Create deleted user to make sure he won't appear in the user_list + deleted_user = CreateTestData.create_user('deleted_user') + deleted_user.delete() + model.repo.commit() + postparams = '%s=1' % json.dumps({}) res = self.app.post('/api/action/user_list', params=postparams) res_obj = json.loads(res.body) @@ -555,6 +561,33 @@ def test_12_user_update_errors(self): for expected_message in test_call['messages']: assert expected_message[1] in ''.join(res_obj['error'][expected_message[0]]) + def test_user_delete(self): + name = 'normal_user' + CreateTestData.create_user(name) + user = model.User.get(name) + user_dict = {'id': user.id} + postparams = '%s=1' % json.dumps(user_dict) + + res = self.app.post('/api/action/user_delete', params=postparams, + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}) + + res_obj = json.loads(res.body) + deleted_user = model.User.get(name) + assert res_obj['success'] is True + assert deleted_user.is_deleted(), deleted_user + + def test_user_delete_requires_data_dict_with_key_id(self): + user_dict = {'name': 'normal_user'} + postparams = '%s=1' % json.dumps(user_dict) + + res = self.app.post('/api/action/user_delete', params=postparams, + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + status=StatusCodes.STATUS_409_CONFLICT) + + res_obj = json.loads(res.body) + assert res_obj['success'] is False + assert res_obj['error']['id'] == ['Missing value'] + def test_13_group_list(self): postparams = '%s=1' % json.dumps({}) res = self.app.post('/api/action/group_list', params=postparams) @@ -633,6 +666,11 @@ def test_14_group_show(self): assert res_obj['success'] is False def test_16_user_autocomplete(self): + # Create deleted user to make sure he won't appear in the user_list + deleted_user = CreateTestData.create_user('joe') + deleted_user.delete() + model.repo.commit() + #Empty query postparams = '%s=1' % json.dumps({}) res = self.app.post( diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 3ea9f2db39f..0f0a63372b2 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -48,8 +48,33 @@ def create_user(self, name): self.apikeys[name] = str(json.loads(res.body)['result']['apikey']) -class TestAuthOrgs(TestAuth): +class TestAuthUsers(TestAuth): + def test_only_sysadmins_can_delete_users(self): + username = 'username' + user = {'id': username} + self.create_user(username) + + self._call_api('user_delete', user, username, 403) + self._call_api('user_delete', user, 'sysadmin', 200) + + def test_auth_deleted_users_are_always_unauthorized(self): + always_success = lambda x,y: {'success': True} + new_authz._AuthFunctions._build() + new_authz._AuthFunctions._functions['always_success'] = always_success + # We can't reuse the username with the other tests because we can't + # rebuild_db(), because in the setup_class we get the sysadmin. If we + # rebuild the DB, we would delete the sysadmin as well. + username = 'other_username' + self.create_user(username) + user = model.User.get(username) + user.delete() + + assert not new_authz.is_authorized_boolean('always_success', {'user': username}) + del new_authz._AuthFunctions._functions['always_success'] + + +class TestAuthOrgs(TestAuth): def test_01_create_users(self): # actual roles assigned later self.create_user('org_admin') diff --git a/ckan/tests/models/test_follower.py b/ckan/tests/models/test_follower.py new file mode 100644 index 00000000000..9f4eb51ea3f --- /dev/null +++ b/ckan/tests/models/test_follower.py @@ -0,0 +1,124 @@ +import ckan.model as model +import ckan.lib.create_test_data as ctd + +CreateTestData = ctd.CreateTestData + + +class FollowerClassesTests(object): + @classmethod + def teardown_class(cls): + model.repo.rebuild_db() + + def test_get(self): + following = self.FOLLOWER_CLASS.get(self.follower.id, self.followee.id) + assert following.follower_id == self.follower.id, following + assert following.object_id == self.followee.id, following + + def test_get_returns_none_if_couldnt_find_users(self): + following = self.FOLLOWER_CLASS.get('some-id', 'other-id') + assert following is None, following + + def test_is_following(self): + assert self.FOLLOWER_CLASS.is_following(self.follower.id, + self.followee.id) + + def test_is_following_returns_false_if_user_isnt_following(self): + assert not self.FOLLOWER_CLASS.is_following(self.followee.id, + self.follower.id) + + def test_followee_count(self): + count = self.FOLLOWER_CLASS.followee_count(self.follower.id) + assert count == 1, count + + def test_followee_list(self): + followees = self.FOLLOWER_CLASS.followee_list(self.follower.id) + object_ids = [f.object_id for f in followees] + assert object_ids == [self.followee.id], object_ids + + def test_follower_count(self): + count = self.FOLLOWER_CLASS.follower_count(self.followee.id) + assert count == 1, count + + def test_follower_list(self): + followers = self.FOLLOWER_CLASS.follower_list(self.followee.id) + follower_ids = [f.follower_id for f in followers] + assert follower_ids == [self.follower.id], follower_ids + + +class TestUserFollowingUser(FollowerClassesTests): + FOLLOWER_CLASS = model.UserFollowingUser + + @classmethod + def setup_class(cls): + model.repo.rebuild_db() + cls.follower = CreateTestData.create_user('follower') + cls.followee = CreateTestData.create_user('followee') + cls.FOLLOWER_CLASS(cls.follower.id, cls.followee.id).save() + cls._create_deleted_models() + + @classmethod + def _create_deleted_models(cls): + deleted_user = CreateTestData.create_user('deleted_user') + cls.FOLLOWER_CLASS(deleted_user.id, cls.followee.id).save() + cls.FOLLOWER_CLASS(cls.follower.id, deleted_user.id).save() + deleted_user.delete() + deleted_user.save() + + +class TestUserFollowingDataset(FollowerClassesTests): + FOLLOWER_CLASS = model.UserFollowingDataset + + @classmethod + def setup_class(cls): + model.repo.rebuild_db() + cls.follower = CreateTestData.create_user('follower') + cls.followee = cls._create_dataset('followee') + cls.FOLLOWER_CLASS(cls.follower.id, cls.followee.id).save() + cls._create_deleted_models() + + @classmethod + def _create_deleted_models(cls): + deleted_user = CreateTestData.create_user('deleted_user') + cls.FOLLOWER_CLASS(deleted_user.id, cls.followee.id).save() + deleted_user.delete() + deleted_user.save() + deleted_dataset = cls._create_dataset('deleted_dataset') + cls.FOLLOWER_CLASS(cls.follower.id, deleted_dataset.id).save() + deleted_dataset.delete() + deleted_dataset.save() + + @classmethod + def _create_dataset(self, name): + CreateTestData.create_arbitrary({'name': name}) + return model.Package.get(name) + + +class TestUserFollowingGroup(FollowerClassesTests): + FOLLOWER_CLASS = model.UserFollowingGroup + + @classmethod + def setup_class(cls): + model.repo.rebuild_db() + model.repo.new_revision() + cls.follower = CreateTestData.create_user('follower') + cls.followee = cls._create_group('followee') + cls.FOLLOWER_CLASS(cls.follower.id, cls.followee.id).save() + cls._create_deleted_models() + model.repo.commit_and_remove() + + @classmethod + def _create_deleted_models(cls): + deleted_user = CreateTestData.create_user('deleted_user') + cls.FOLLOWER_CLASS(deleted_user.id, cls.followee.id).save() + deleted_user.delete() + deleted_user.save() + deleted_group = cls._create_group('deleted_group') + cls.FOLLOWER_CLASS(cls.follower.id, deleted_group.id).save() + deleted_group.delete() + deleted_group.save() + + @classmethod + def _create_group(self, name): + group = model.Group(name) + group.save() + return group diff --git a/ckan/tests/models/test_user.py b/ckan/tests/models/test_user.py index b9ef7f0624f..c4b356ae854 100644 --- a/ckan/tests/models/test_user.py +++ b/ckan/tests/models/test_user.py @@ -56,6 +56,13 @@ def test_4_get_openid_missing_slash(self): assert out assert out.fullname == u'Sandra' + def test_is_deleted(self): + user = CreateTestData._create_user_without_commit('a_user') + assert not user.is_deleted(), user + user.delete() + assert user.is_deleted(), user + + def to_names(domain_obj_list): '''Takes a list of domain objects and returns a corresponding list of their names.''' From ad2de00a4e2c2772d1e59faee26ca818731168c9 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 9 Aug 2013 16:07:29 -0300 Subject: [PATCH 086/269] [#1163] Use ckan.model.State instead of vdm.sqlalchemy.State This makes us a bit less tied to vdm. --- ckan/logic/action/get.py | 5 ++--- .../migration/versions/070_add_state_column_to_user_table.py | 4 ++-- ckan/model/follower.py | 4 ++-- ckan/model/group.py | 4 ++-- ckan/model/user.py | 4 +++- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a6cec306c91..9ede04d7a39 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -7,7 +7,6 @@ from pylons import config import sqlalchemy -import vdm.sqlalchemy import ckan.lib.dictization import ckan.logic as logic @@ -666,7 +665,7 @@ def user_list(context, data_dict): ) # Filter deleted users - query = query.filter(model.User.state != vdm.sqlalchemy.State.DELETED) + query = query.filter(model.User.state != model.State.DELETED) ## hack for pagination if context.get('return_query'): @@ -1178,7 +1177,7 @@ def user_autocomplete(context, data_dict): limit = data_dict.get('limit', 20) query = model.User.search(q) - query = query.filter(model.User.state != vdm.sqlalchemy.State.DELETED) + query = query.filter(model.User.state != model.State.DELETED) query = query.limit(limit) user_list = [] diff --git a/ckan/migration/versions/070_add_state_column_to_user_table.py b/ckan/migration/versions/070_add_state_column_to_user_table.py index 1dfadbe7d32..46c5c3d4cd5 100644 --- a/ckan/migration/versions/070_add_state_column_to_user_table.py +++ b/ckan/migration/versions/070_add_state_column_to_user_table.py @@ -1,11 +1,11 @@ -import vdm.sqlalchemy +import ckan.model def upgrade(migrate_engine): migrate_engine.execute( ''' ALTER TABLE "user" ADD COLUMN "state" text NOT NULL DEFAULT '%s' - ''' % vdm.sqlalchemy.State.ACTIVE + ''' % ckan.model.State.ACTIVE ) diff --git a/ckan/model/follower.py b/ckan/model/follower.py index d14ffe667ae..66ac1da435d 100644 --- a/ckan/model/follower.py +++ b/ckan/model/follower.py @@ -80,8 +80,8 @@ def _get(cls, follower_id=None, object_id=None): .filter(sqlalchemy.and_(follower_alias.id == follower_id,\ cls.follower_id == follower_alias.id,\ cls.object_id == object_alias.id,\ - follower_alias.state != vdm.sqlalchemy.State.DELETED,\ - object_alias.state != vdm.sqlalchemy.State.DELETED,\ + follower_alias.state != ckan.model.State.DELETED,\ + object_alias.state != ckan.model.State.DELETED,\ object_alias.id == object_id)) return query diff --git a/ckan/model/group.py b/ckan/model/group.py index c16424affee..d5fadab6c2c 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -195,8 +195,8 @@ def packages(self, with_private=False, limit=None, query = meta.Session.query(_package.Package).\ filter( - or_(_package.Package.state == vdm.sqlalchemy.State.ACTIVE, - _package.Package.state == vdm.sqlalchemy.State.PENDING)). \ + or_(_package.Package.state == model.State.ACTIVE, + _package.Package.state == model.State.PENDING)). \ filter(group_table.c.id == self.id).\ filter(member_table.c.state == 'active') diff --git a/ckan/model/user.py b/ckan/model/user.py index d36dd5343cc..82e1acd99b8 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -167,7 +167,9 @@ def number_administered_packages(self): return q.count() def is_deleted(self): - return self.state == vdm.sqlalchemy.State.DELETED + # have to import here to avoid circular imports + import ckan.model as model + return self.state == model.State.DELETED def is_in_group(self, group): return group in self.get_group_ids() From d6fde48e5cd73a2d0754b3ceb9fe6319deedaa20 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 9 Aug 2013 18:40:36 -0300 Subject: [PATCH 087/269] [#1163] Use ckan.model.core.State instead of ckan.model.State With this change, we're able to avoid having to load ckan.model inside methods, to avoid circular dependencies. --- ckan/logic/action/get.py | 5 +++-- .../migration/versions/070_add_state_column_to_user_table.py | 4 ++-- ckan/model/follower.py | 5 +++-- ckan/model/group.py | 4 ++-- ckan/model/user.py | 5 ++--- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 9ede04d7a39..176096f4481 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -14,6 +14,7 @@ import ckan.logic.schema import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.navl.dictization_functions +import ckan.model.core as core import ckan.model.misc as misc import ckan.plugins as plugins import ckan.lib.search as search @@ -665,7 +666,7 @@ def user_list(context, data_dict): ) # Filter deleted users - query = query.filter(model.User.state != model.State.DELETED) + query = query.filter(model.User.state != core.State.DELETED) ## hack for pagination if context.get('return_query'): @@ -1177,7 +1178,7 @@ def user_autocomplete(context, data_dict): limit = data_dict.get('limit', 20) query = model.User.search(q) - query = query.filter(model.User.state != model.State.DELETED) + query = query.filter(model.User.state != core.State.DELETED) query = query.limit(limit) user_list = [] diff --git a/ckan/migration/versions/070_add_state_column_to_user_table.py b/ckan/migration/versions/070_add_state_column_to_user_table.py index 46c5c3d4cd5..367e3a48c32 100644 --- a/ckan/migration/versions/070_add_state_column_to_user_table.py +++ b/ckan/migration/versions/070_add_state_column_to_user_table.py @@ -1,11 +1,11 @@ -import ckan.model +import ckan.model.core def upgrade(migrate_engine): migrate_engine.execute( ''' ALTER TABLE "user" ADD COLUMN "state" text NOT NULL DEFAULT '%s' - ''' % ckan.model.State.ACTIVE + ''' % ckan.model.core.State.ACTIVE ) diff --git a/ckan/model/follower.py b/ckan/model/follower.py index 66ac1da435d..a8afd07706f 100644 --- a/ckan/model/follower.py +++ b/ckan/model/follower.py @@ -3,6 +3,7 @@ import sqlalchemy import vdm.sqlalchemy +import core import ckan.model import domain_object @@ -80,8 +81,8 @@ def _get(cls, follower_id=None, object_id=None): .filter(sqlalchemy.and_(follower_alias.id == follower_id,\ cls.follower_id == follower_alias.id,\ cls.object_id == object_alias.id,\ - follower_alias.state != ckan.model.State.DELETED,\ - object_alias.state != ckan.model.State.DELETED,\ + follower_alias.state != core.State.DELETED,\ + object_alias.state != core.State.DELETED,\ object_alias.id == object_id)) return query diff --git a/ckan/model/group.py b/ckan/model/group.py index d5fadab6c2c..86963983e9a 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -195,8 +195,8 @@ def packages(self, with_private=False, limit=None, query = meta.Session.query(_package.Package).\ filter( - or_(_package.Package.state == model.State.ACTIVE, - _package.Package.state == model.State.PENDING)). \ + or_(_package.Package.state == core.State.ACTIVE, + _package.Package.state == core.State.PENDING)). \ filter(group_table.c.id == self.id).\ filter(member_table.c.state == 'active') diff --git a/ckan/model/user.py b/ckan/model/user.py index 82e1acd99b8..ce882ca2b8f 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -9,6 +9,7 @@ import vdm.sqlalchemy import meta +import core import types as _types import domain_object @@ -167,9 +168,7 @@ def number_administered_packages(self): return q.count() def is_deleted(self): - # have to import here to avoid circular imports - import ckan.model as model - return self.state == model.State.DELETED + return self.state == core.State.DELETED def is_in_group(self, group): return group in self.get_group_ids() From 89a38101afe7b1415d431cdffc013e45bf90bf67 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Mon, 12 Aug 2013 17:04:00 -0300 Subject: [PATCH 088/269] [#1178] Add PENDING state to User --- ckan/lib/authenticator.py | 6 +++--- ckan/lib/base.py | 2 +- ckan/model/user.py | 11 +++++++++++ ckan/new_authz.py | 27 ++++++++++++--------------- ckan/tests/lib/test_authenticator.py | 19 +++++++++++++++++++ ckan/tests/logic/test_auth.py | 18 +++++++++++++++++- ckan/tests/models/test_user.py | 26 ++++++++++++++++++++++++++ 7 files changed, 89 insertions(+), 20 deletions(-) diff --git a/ckan/lib/authenticator.py b/ckan/lib/authenticator.py index f64db4f5b8d..cf6ed016694 100644 --- a/ckan/lib/authenticator.py +++ b/ckan/lib/authenticator.py @@ -14,7 +14,7 @@ def authenticate(self, environ, identity): if 'repoze.who.plugins.openid.userid' in identity: openid = identity['repoze.who.plugins.openid.userid'] user = User.by_openid(openid) - if user is None or user.is_deleted(): + if user is None or not user.is_active(): return None else: return user.name @@ -33,8 +33,8 @@ def authenticate(self, environ, identity): if user is None: log.debug('Login failed - username %r not found', login) - elif user.is_deleted(): - log.debug('Login as %r failed - user is deleted', login) + elif not user.is_active(): + log.debug('Login as %r failed - user isn\'t active', login) elif not user.validate_password(identity['password']): log.debug('Login as %r failed - password not valid', login) else: diff --git a/ckan/lib/base.py b/ckan/lib/base.py index bfffcaf2a5d..eabd9200b1d 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -290,7 +290,7 @@ def _identify_user_default(self): if c.user: c.user = c.user.decode('utf8') c.userobj = model.User.by_name(c.user) - if c.userobj is None or c.userobj.is_deleted(): + if c.userobj is None or not c.userobj.is_active(): # This occurs when a user that was still logged in is deleted, # or when you are logged in, clean db # and then restart (or when you change your username) diff --git a/ckan/model/user.py b/ckan/model/user.py index ce882ca2b8f..a6d6c44b612 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -167,9 +167,20 @@ def number_administered_packages(self): q = q.filter_by(user=self, role=model.Role.ADMIN) return q.count() + def activate(self): + ''' Activate the user ''' + self.state = core.State.ACTIVE + + def set_pending(self): + ''' Set the user as pending ''' + self.state = core.State.PENDING + def is_deleted(self): return self.state == core.State.DELETED + def is_pending(self): + return self.state == core.State.PENDING + def is_in_group(self, group): return group in self.get_group_ids() diff --git a/ckan/new_authz.py b/ckan/new_authz.py index bf1d0830d47..b98434a9b45 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -106,12 +106,6 @@ def is_sysadmin(username): return user and user.sysadmin -def is_deleted(username): - ''' Returns True if username is deleted ''' - user = _get_user(username) - return user and user.is_deleted() - - def _get_user(username): ''' Try to get the user from c, if possible, and fallback to using the DB ''' if not username: @@ -154,15 +148,18 @@ def is_authorized(action, context, data_dict=None): auth_function = _AuthFunctions.get(action) if auth_function: username = context.get('user') - # deleted users are always unauthorized - if is_deleted(username): - return {'success': False} - # sysadmins can do anything unless the auth_sysadmins_check - # decorator was used in which case they are treated like all other - # users. - elif is_sysadmin(username): - if not getattr(auth_function, 'auth_sysadmins_check', False): - return {'success': True} + user = _get_user(username) + + if user: + # inactive users are always unauthorized + if not user.is_active(): + return {'success': False} + # sysadmins can do anything unless the auth_sysadmins_check + # decorator was used in which case they are treated like all other + # users. + elif user.sysadmin: + if not getattr(auth_function, 'auth_sysadmins_check', False): + return {'success': True} return auth_function(context, data_dict) else: diff --git a/ckan/tests/lib/test_authenticator.py b/ckan/tests/lib/test_authenticator.py index c7c026112d2..8e7254d94c7 100644 --- a/ckan/tests/lib/test_authenticator.py +++ b/ckan/tests/lib/test_authenticator.py @@ -34,6 +34,15 @@ def test_authenticate_fails_if_user_is_deleted(self): assert self.authenticate(environ, identity) is None + def test_authenticate_fails_if_user_is_pending(self): + environ = {} + password = 'somepass' + user = CreateTestData.create_user('a_user', **{'password': password}) + identity = {'login': user.name, 'password': password} + user.set_pending() + + assert self.authenticate(environ, identity) is None + def test_authenticate_fails_if_password_is_wrong(self): environ = {} user = CreateTestData.create_user('a_user') @@ -100,6 +109,16 @@ def test_authenticate_fails_if_user_is_deleted(self): assert self.authenticate(environ, identity) is None + def test_authenticate_fails_if_user_is_deleted(self): + environ = {} + openid = 'some-openid-key' + user = CreateTestData.create_user('a_user', **{'openid': openid}) + user.set_pending() + identity = {'login': user.name, + 'repoze.who.plugins.openid.userid': openid} + + assert self.authenticate(environ, identity) is None + def test_authenticate_fails_if_user_have_no_openid(self): environ = {} identity = {} diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 0f0a63372b2..2b5204e878a 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -64,7 +64,7 @@ def test_auth_deleted_users_are_always_unauthorized(self): # We can't reuse the username with the other tests because we can't # rebuild_db(), because in the setup_class we get the sysadmin. If we # rebuild the DB, we would delete the sysadmin as well. - username = 'other_username' + username = 'deleted_user' self.create_user(username) user = model.User.get(username) user.delete() @@ -73,6 +73,22 @@ def test_auth_deleted_users_are_always_unauthorized(self): del new_authz._AuthFunctions._functions['always_success'] + def test_auth_pending_users_are_always_unauthorized(self): + always_success = lambda x,y: {'success': True} + new_authz._AuthFunctions._build() + new_authz._AuthFunctions._functions['always_success'] = always_success + # We can't reuse the username with the other tests because we can't + # rebuild_db(), because in the setup_class we get the sysadmin. If we + # rebuild the DB, we would delete the sysadmin as well. + username = 'pending_user' + self.create_user(username) + user = model.User.get(username) + user.state = model.State.PENDING + + assert not new_authz.is_authorized_boolean('always_success', {'user': username}) + + del new_authz._AuthFunctions._functions['always_success'] + class TestAuthOrgs(TestAuth): def test_01_create_users(self): diff --git a/ckan/tests/models/test_user.py b/ckan/tests/models/test_user.py index c4b356ae854..bd34641b936 100644 --- a/ckan/tests/models/test_user.py +++ b/ckan/tests/models/test_user.py @@ -58,10 +58,36 @@ def test_4_get_openid_missing_slash(self): def test_is_deleted(self): user = CreateTestData._create_user_without_commit('a_user') + user.state = 'some-state' assert not user.is_deleted(), user user.delete() assert user.is_deleted(), user + def test_user_is_active_by_default(self): + user = CreateTestData._create_user_without_commit('a_user') + assert user.is_active(), user + + def test_activate(self): + user = CreateTestData._create_user_without_commit('a_user') + user.state = 'some-state' + assert not user.is_active(), user + user.activate() + assert user.is_active(), user + + def test_activate(self): + user = CreateTestData._create_user_without_commit('a_user') + user.state = 'some-state' + assert not user.is_active(), user + user.activate() + assert user.is_active(), user + + def test_is_pending(self): + user = CreateTestData._create_user_without_commit('a_user') + user.state = 'some-state' + assert not user.is_pending(), user + user.set_pending() + assert user.is_pending(), user + def to_names(domain_obj_list): '''Takes a list of domain objects and returns a corresponding list From c0c6803b574d48016af6185fd0a2a71605567a77 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Wed, 14 Aug 2013 18:13:36 -0300 Subject: [PATCH 089/269] [#1178] The user is activated when it performs the reset password --- ckan/controllers/user.py | 1 + ckan/tests/functional/test_user.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index e0b1ba6c762..4e76b098d37 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -479,6 +479,7 @@ def perform_reset(self, id): new_password = self._get_form_password() user_dict['password'] = new_password user_dict['reset_key'] = c.reset_key + user_dict['state'] = model.State.ACTIVE user = get_action('user_update')(context, user_dict) h.flash_success(_("Your password has been reset.")) diff --git a/ckan/tests/functional/test_user.py b/ckan/tests/functional/test_user.py index 16795561f46..f0ff38cfdac 100644 --- a/ckan/tests/functional/test_user.py +++ b/ckan/tests/functional/test_user.py @@ -955,3 +955,21 @@ def test_perform_reset_user_password_link_user_incorrect(self): id='randomness', # i.e. incorrect key='randomness') res = self.app.get(offset, status=404) + + def test_perform_reset_activates_pending_user(self): + password = 'password' + params = { 'password1': password, 'password2': password } + user = CreateTestData.create_user(name='username', + email='user@email.com') + user.set_pending() + create_reset_key(user) + assert user.is_pending(), user.state + + offset = url_for(controller='user', + action='perform_reset', + id=user.id, + key=user.reset_key) + res = self.app.post(offset, params=params, status=302) + + user = model.User.get(user.id) + assert user.is_active(), user From 74f649c9e3eb0690f5e48f939b4546b415b0777b Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Wed, 14 Aug 2013 21:42:22 -0300 Subject: [PATCH 090/269] [#1178] Admins can invite users The invited user starts in pending state, with the password reset key set. We still have to send an email to the user telling him/her to change the password and log in. I had to change authorization code to only automatically unauthorize deleted users, not pending. This was because the users needs to be able to perform the password reset when pending, to be able to become active. --- ckan/controllers/user.py | 2 +- ckan/logic/action/create.py | 37 ++++++++++++++++++++++ ckan/logic/auth/create.py | 2 ++ ckan/logic/schema.py | 1 + ckan/new_authz.py | 4 +-- ckan/tests/functional/test_user.py | 20 +++++++++++- ckan/tests/logic/test_action.py | 50 ++++++++++++++++++++++++++++++ ckan/tests/logic/test_auth.py | 22 ++++--------- dev-requirements.txt | 1 + 9 files changed, 119 insertions(+), 20 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 4e76b098d37..000f3db5a0c 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -447,7 +447,7 @@ def perform_reset(self, id): # FIXME We should reset the reset key when it is used to prevent # reuse of the url context = {'model': model, 'session': model.Session, - 'user': c.user, + 'user': c.user or id, 'keep_sensitive_data': True} data_dict = {'id': id} diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 1fc0fb6e058..346fd30e680 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -1,6 +1,8 @@ '''API functions for adding data to CKAN.''' import logging +import random +import re from pylons import config import paste.deploy.converters @@ -15,6 +17,7 @@ import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.dictization.model_save as model_save import ckan.lib.navl.dictization_functions +import ckan.lib.navl.validators as validators from ckan.common import _ @@ -836,6 +839,40 @@ def user_create(context, data_dict): log.debug('Created user {name}'.format(name=user.name)) return user_dict +def user_invite(context, data_dict): + '''docstring''' + _check_access('user_invite', context, data_dict) + + user_invite_schema = { + 'email': [validators.not_empty, unicode] + } + _, errors = _validate(data_dict, user_invite_schema, context) + if errors: + raise ValidationError(errors) + + while True: + try: + import ckan.lib.mailer + name = _get_random_username_from_email(data_dict['email']) + password = str(random.SystemRandom().random()) + data_dict['name'] = name + data_dict['password'] = password + data_dict['state'] = ckan.model.State.PENDING + user_dict = _get_action('user_create')(context, data_dict) + user = ckan.model.User.get(user_dict['id']) + ckan.lib.mailer.create_reset_key(user) + return model_dictize.user_dictize(user, context) + except ValidationError as e: + if 'name' not in e.error_dict: + raise e + +def _get_random_username_from_email(email): + localpart = email.split('@')[0] + cleaned_localpart = re.sub(r'[^\w]', '', localpart) + random_number = random.SystemRandom().random() * 10000 + name = '%s-%d' % (cleaned_localpart, random_number) + return name + ## Modifications for rest api def package_create_rest(context, data_dict): diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index bf9c3d17ea3..fafbc5a2e4c 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -112,6 +112,8 @@ def user_create(context, data_dict=None): else: return {'success': True} +def user_invite(context, data_dict=None): + return {'success': False} def _check_group_auth(context, data_dict): # FIXME This code is shared amoung other logic.auth files and should be diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f8fe8df7fbd..a8cd4ce6779 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -395,6 +395,7 @@ def default_user_schema(): 'apikey': [ignore], 'reset_key': [ignore], 'activity_streams_email_notifications': [ignore_missing], + 'state': [ignore_missing], } return schema diff --git a/ckan/new_authz.py b/ckan/new_authz.py index b98434a9b45..62a4e88a978 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -151,8 +151,8 @@ def is_authorized(action, context, data_dict=None): user = _get_user(username) if user: - # inactive users are always unauthorized - if not user.is_active(): + # deleted users are always unauthorized + if user.is_deleted(): return {'success': False} # sysadmins can do anything unless the auth_sysadmins_check # decorator was used in which case they are treated like all other diff --git a/ckan/tests/functional/test_user.py b/ckan/tests/functional/test_user.py index f0ff38cfdac..fe30f53a505 100644 --- a/ckan/tests/functional/test_user.py +++ b/ckan/tests/functional/test_user.py @@ -959,7 +959,7 @@ def test_perform_reset_user_password_link_user_incorrect(self): def test_perform_reset_activates_pending_user(self): password = 'password' params = { 'password1': password, 'password2': password } - user = CreateTestData.create_user(name='username', + user = CreateTestData.create_user(name='pending_user', email='user@email.com') user.set_pending() create_reset_key(user) @@ -973,3 +973,21 @@ def test_perform_reset_activates_pending_user(self): user = model.User.get(user.id) assert user.is_active(), user + + def test_perform_reset_doesnt_activate_deleted_user(self): + password = 'password' + params = { 'password1': password, 'password2': password } + user = CreateTestData.create_user(name='deleted_user', + email='user@email.com') + user.delete() + create_reset_key(user) + assert user.is_deleted(), user.state + + offset = url_for(controller='user', + action='perform_reset', + id=user.id, + key=user.reset_key) + res = self.app.post(offset, params=params, status=302) + + user = model.User.get(user.id) + assert user.is_deleted(), user diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index fecbdf96b53..7bf17bd094e 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -6,6 +6,7 @@ from nose.plugins.skip import SkipTest from pylons import config import datetime +import mock import vdm.sqlalchemy import ckan @@ -561,6 +562,55 @@ def test_12_user_update_errors(self): for expected_message in test_call['messages']: assert expected_message[1] in ''.join(res_obj['error'][expected_message[0]]) + def test_user_invite(self): + email_username = 'invited_user$ckan' + email = '%s@email.com' % email_username + user_dict = {'email': email} + postparams = '%s=1' % json.dumps(user_dict) + extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} + + res = self.app.post('/api/action/user_invite', params=postparams, + extra_environ=extra_environ) + + res_obj = json.loads(res.body) + user = model.User.get(res_obj['result']['id']) + expected_username = email_username.replace('$', '') + assert res_obj['success'] is True, res_obj + assert user.email == email, (user.email, email) + assert user.name.startswith(expected_username), (user.name, expected_username) + assert user.is_pending(), user + assert user.reset_key is not None, user + + def test_user_invite_without_email_raises_error(self): + user_dict = {} + postparams = '%s=1' % json.dumps(user_dict) + extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} + + res = self.app.post('/api/action/user_invite', params=postparams, + extra_environ=extra_environ, + status=StatusCodes.STATUS_409_CONFLICT) + + res_obj = json.loads(res.body) + assert res_obj['success'] is False, res_obj + assert 'email' in res_obj['error'], res_obj + + @mock.patch('ckan.logic.action.create._get_random_username_from_email') + def test_user_invite_should_work_even_if_tried_username_already_exists(self, random_username_mock): + email = 'invited_user@email.com' + user_dict = {'email': email} + postparams = '%s=1' % json.dumps(user_dict) + extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} + + usernames = ['first', 'first', 'second'] + random_username_mock.side_effect = lambda email: usernames.pop(0) + + for _ in range(2): + res = self.app.post('/api/action/user_invite', params=postparams, + extra_environ=extra_environ) + + res_obj = json.loads(res.body) + assert res_obj['success'] is True, res_obj + def test_user_delete(self): name = 'normal_user' CreateTestData.create_user(name) diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 2b5204e878a..13e7b38f2a5 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -49,6 +49,12 @@ def create_user(self, name): class TestAuthUsers(TestAuth): + def test_only_sysadmins_can_invite_users(self): + username = 'normal_user' + self.create_user(username) + + assert not new_authz.is_authorized_boolean('user_invite', {'user': username}) + def test_only_sysadmins_can_delete_users(self): username = 'username' user = {'id': username} @@ -73,22 +79,6 @@ def test_auth_deleted_users_are_always_unauthorized(self): del new_authz._AuthFunctions._functions['always_success'] - def test_auth_pending_users_are_always_unauthorized(self): - always_success = lambda x,y: {'success': True} - new_authz._AuthFunctions._build() - new_authz._AuthFunctions._functions['always_success'] = always_success - # We can't reuse the username with the other tests because we can't - # rebuild_db(), because in the setup_class we get the sysadmin. If we - # rebuild the DB, we would delete the sysadmin as well. - username = 'pending_user' - self.create_user(username) - user = model.User.get(username) - user.state = model.State.PENDING - - assert not new_authz.is_authorized_boolean('always_success', {'user': username}) - - del new_authz._AuthFunctions._functions['always_success'] - class TestAuthOrgs(TestAuth): def test_01_create_users(self): diff --git a/dev-requirements.txt b/dev-requirements.txt index a445712d239..a781573b0b8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,3 +7,4 @@ nose==1.3.0 pep8==1.4.6 Sphinx==1.2b1 polib==1.0.3 +mock==1.0.1 From 31523cd27a35958c2f0660a2762bcfd0274c8006 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Thu, 15 Aug 2013 11:49:03 -0300 Subject: [PATCH 091/269] [#1178] perform_reset uses the received id as the context's user When performing a password reset, the user is probably (always?) not logged in. So c.user is an empty string. So, the auth functions have no way to tell which user is trying to reset his/her password. This worked fine before, because everyone was able to reset the password. But now that we've got users in DELETED state, it's not the case anymore. --- ckan/controllers/user.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 000f3db5a0c..98033d9a0ab 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -447,17 +447,16 @@ def perform_reset(self, id): # FIXME We should reset the reset key when it is used to prevent # reuse of the url context = {'model': model, 'session': model.Session, - 'user': c.user or id, + 'user': id, 'keep_sensitive_data': True} - data_dict = {'id': id} - try: check_access('user_reset', context) except NotAuthorized: abort(401, _('Unauthorized to reset password.')) try: + data_dict = {'id': id} user_dict = get_action('user_show')(context, data_dict) # Be a little paranoid, and get rid of sensitive data that's From e5890740164c38ca0ebf10bf25a297b0b634e204 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 16 Aug 2013 10:39:25 -0300 Subject: [PATCH 092/269] [#1178] Send email to the invited user I removed the time.sleep(0.1) on TestMailer. Looking through the code, I couldn't find anywhere where a timer looked needed. And I ran these tests a hundred times without the timer to see if I could make them fail, but no. So, I guess they're not needed anymore. I also had to move the RESET_LINK_MESSAGE inside get_reset_link_body(). This was because, when importing ckan.lib.mailer in ckan.logic.action.create.py, I got: TypeError: No object (name: translator) has been registered for this thread This seems to be because we were using _() before pylons had a change to set up the translator. Moving it inside the method solves this. --- ckan/lib/mailer.py | 59 +++++++++++++++++++++++---------- ckan/logic/action/create.py | 18 +++++++--- ckan/tests/lib/test_mailer.py | 37 ++++++++++++++------- ckan/tests/logic/test_action.py | 28 ++++++++++++++-- 4 files changed, 105 insertions(+), 37 deletions(-) diff --git a/ckan/lib/mailer.py b/ckan/lib/mailer.py index f0e1978ac2c..72ebe33e2e0 100644 --- a/ckan/lib/mailer.py +++ b/ckan/lib/mailer.py @@ -100,21 +100,37 @@ def mail_user(recipient, subject, body, headers={}): mail_recipient(recipient.display_name, recipient.email, subject, body, headers=headers) +def get_reset_link_body(user): + reset_link_message = _( + '''You have requested your password on %(site_title)s to be reset. -RESET_LINK_MESSAGE = _( -'''You have requested your password on %(site_title)s to be reset. + Please click the following link to confirm this request: -Please click the following link to confirm this request: + %(reset_link)s + ''') - %(reset_link)s -''') + d = { + 'reset_link': get_reset_link(user), + 'site_title': g.site_title + } + return reset_link_message % d -def make_key(): - return uuid.uuid4().hex[:10] +def get_invite_body(user): + invite_message = _( + '''You have been invited to %(site_title)s. A user has already been created to + you with the username %(user_name)s. You can change it later. -def create_reset_key(user): - user.reset_key = unicode(make_key()) - model.repo.commit_and_remove() + To accept this invite, please reset your password at: + + %(reset_link)s + ''') + + d = { + 'reset_link': get_reset_link(user), + 'site_title': g.site_title, + 'user_name': user.name, + } + return invite_message % d def get_reset_link(user): return urljoin(g.site_url, @@ -123,17 +139,24 @@ def get_reset_link(user): id=user.id, key=user.reset_key)) -def get_reset_link_body(user): - d = { - 'reset_link': get_reset_link(user), - 'site_title': g.site_title - } - return RESET_LINK_MESSAGE % d - def send_reset_link(user): create_reset_key(user) body = get_reset_link_body(user) - mail_user(user, _('Reset your password'), body) + subject = _('Reset your password') + mail_user(user, subject, body) + +def send_invite(user): + create_reset_key(user) + body = get_invite_body(user) + subject = _('Invite for {site_title}'.format(site_title=g.site_title)) + mail_user(user, subject, body) + +def create_reset_key(user): + user.reset_key = unicode(make_key()) + model.repo.commit_and_remove() + +def make_key(): + return uuid.uuid4().hex[:10] def verify_reset_link(user, key): if not key: diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 346fd30e680..edef0a95f30 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -18,6 +18,7 @@ import ckan.lib.dictization.model_save as model_save import ckan.lib.navl.dictization_functions import ckan.lib.navl.validators as validators +import ckan.lib.mailer as mailer from ckan.common import _ @@ -840,7 +841,17 @@ def user_create(context, data_dict): return user_dict def user_invite(context, data_dict): - '''docstring''' + '''Invite a new user. + + You must be authorized to invite users. + + :param email: the email address for the new user + :type email: string + + :returns: the newly created yser + :rtype: dictionary + + ''' _check_access('user_invite', context, data_dict) user_invite_schema = { @@ -852,7 +863,6 @@ def user_invite(context, data_dict): while True: try: - import ckan.lib.mailer name = _get_random_username_from_email(data_dict['email']) password = str(random.SystemRandom().random()) data_dict['name'] = name @@ -860,7 +870,7 @@ def user_invite(context, data_dict): data_dict['state'] = ckan.model.State.PENDING user_dict = _get_action('user_create')(context, data_dict) user = ckan.model.User.get(user_dict['id']) - ckan.lib.mailer.create_reset_key(user) + mailer.send_invite(user) return model_dictize.user_dictize(user, context) except ValidationError as e: if 'name' not in e.error_dict: @@ -868,7 +878,7 @@ def user_invite(context, data_dict): def _get_random_username_from_email(email): localpart = email.split('@')[0] - cleaned_localpart = re.sub(r'[^\w]', '', localpart) + cleaned_localpart = re.sub(r'[^\w]', '-', localpart) random_number = random.SystemRandom().random() * 10000 name = '%s-%d' % (cleaned_localpart, random_number) return name diff --git a/ckan/tests/lib/test_mailer.py b/ckan/tests/lib/test_mailer.py index b95d4d290e3..22b3985930b 100644 --- a/ckan/tests/lib/test_mailer.py +++ b/ckan/tests/lib/test_mailer.py @@ -1,13 +1,12 @@ -import time from nose.tools import assert_equal, assert_raises from pylons import config from email.mime.text import MIMEText import hashlib -from ckan import model +import ckan.model as model +import ckan.lib.mailer as mailer from ckan.tests.pylons_controller import PylonsTestCase from ckan.tests.mock_mail_server import SmtpServerHarness -from ckan.lib.mailer import mail_recipient, mail_user, send_reset_link, add_msg_niceties, MailerException, get_reset_link_body, get_reset_link from ckan.lib.create_test_data import CreateTestData from ckan.lib.base import g @@ -35,7 +34,7 @@ def setup(self): def mime_encode(self, msg, recipient_name): sender_name = g.site_title sender_url = g.site_url - body = add_msg_niceties(recipient_name, msg, sender_name, sender_url) + body = mailer.add_msg_niceties(recipient_name, msg, sender_name, sender_url) encoded_body = MIMEText(body.encode('utf-8'), 'plain', 'utf-8').get_payload().strip() return encoded_body @@ -49,8 +48,7 @@ def test_mail_recipient(self): 'subject': 'Meeting', 'body': 'The meeting is cancelled.', 'headers': {'header1': 'value1'}} - mail_recipient(**test_email) - time.sleep(0.1) + mailer.mail_recipient(**test_email) # check it went to the mock smtp server msgs = self.get_smtp_messages() @@ -74,8 +72,7 @@ def test_mail_user(self): 'subject': 'Meeting', 'body': 'The meeting is cancelled.', 'headers': {'header1': 'value1'}} - mail_user(**test_email) - time.sleep(0.1) + mailer.mail_user(**test_email) # check it went to the mock smtp server msgs = self.get_smtp_messages() @@ -96,12 +93,11 @@ def test_mail_user_without_email(self): 'subject': 'Meeting', 'body': 'The meeting is cancelled.', 'headers': {'header1': 'value1'}} - assert_raises(MailerException, mail_user, **test_email) + assert_raises(mailer.MailerException, mailer.mail_user, **test_email) def test_send_reset_email(self): # send email - send_reset_link(model.User.by_name(u'bob')) - time.sleep(0.1) + mailer.send_reset_link(model.User.by_name(u'bob')) # check it went to the mock smtp server msgs = self.get_smtp_messages() @@ -110,7 +106,24 @@ def test_send_reset_email(self): assert_equal(msg[1], config['smtp.mail_from']) assert_equal(msg[2], [model.User.by_name(u'bob').email]) assert 'Reset' in msg[3], msg[3] - test_msg = get_reset_link_body(model.User.by_name(u'bob')) + test_msg = mailer.get_reset_link_body(model.User.by_name(u'bob')) + expected_body = self.mime_encode(test_msg, + u'bob') + assert expected_body in msg[3], '%r not in %r' % (expected_body, msg[3]) + + # reset link tested in user functional test + + def test_send_invite_email(self): + # send email + mailer.send_invite(model.User.by_name(u'bob')) + + # check it went to the mock smtp server + msgs = self.get_smtp_messages() + assert_equal(len(msgs), 1) + msg = msgs[0] + assert_equal(msg[1], config['smtp.mail_from']) + assert_equal(msg[2], [model.User.by_name(u'bob').email]) + test_msg = mailer.get_invite_body(model.User.by_name(u'bob')) expected_body = self.mime_encode(test_msg, u'bob') assert expected_body in msg[3], '%r not in %r' % (expected_body, msg[3]) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 7bf17bd094e..b6361803f0e 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -562,7 +562,8 @@ def test_12_user_update_errors(self): for expected_message in test_call['messages']: assert expected_message[1] in ''.join(res_obj['error'][expected_message[0]]) - def test_user_invite(self): + @mock.patch('ckan.lib.mailer.mail_user') + def test_user_invite(self, mail_user): email_username = 'invited_user$ckan' email = '%s@email.com' % email_username user_dict = {'email': email} @@ -574,14 +575,32 @@ def test_user_invite(self): res_obj = json.loads(res.body) user = model.User.get(res_obj['result']['id']) - expected_username = email_username.replace('$', '') + expected_username = email_username.replace('$', '-') assert res_obj['success'] is True, res_obj assert user.email == email, (user.email, email) assert user.name.startswith(expected_username), (user.name, expected_username) assert user.is_pending(), user assert user.reset_key is not None, user - def test_user_invite_without_email_raises_error(self): + @mock.patch('ckan.lib.mailer.send_invite') + def test_user_invite_sends_email(self, send_invite): + email_username = 'invited_user' + email = '%s@email.com' % email_username + user_dict = {'email': email} + postparams = '%s=1' % json.dumps(user_dict) + extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} + + res = self.app.post('/api/action/user_invite', params=postparams, + extra_environ=extra_environ) + + res_obj = json.loads(res.body) + user = model.User.get(res_obj['result']['id']) + assert res_obj['success'] is True, res_obj + assert send_invite.called + assert send_invite.call_args[0][0].id == res_obj['result']['id'] + + @mock.patch('ckan.lib.mailer.mail_user') + def test_user_invite_without_email_raises_error(self, mail_user): user_dict = {} postparams = '%s=1' % json.dumps(user_dict) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} @@ -596,6 +615,8 @@ def test_user_invite_without_email_raises_error(self): @mock.patch('ckan.logic.action.create._get_random_username_from_email') def test_user_invite_should_work_even_if_tried_username_already_exists(self, random_username_mock): + patcher = mock.patch('ckan.lib.mailer.mail_user') + patcher.start() email = 'invited_user@email.com' user_dict = {'email': email} postparams = '%s=1' % json.dumps(user_dict) @@ -610,6 +631,7 @@ def test_user_invite_should_work_even_if_tried_username_already_exists(self, ran res_obj = json.loads(res.body) assert res_obj['success'] is True, res_obj + patcher.stop() def test_user_delete(self): name = 'normal_user' From 85c50e18c43cccc69446aa977b082de0411a0560 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 20 Aug 2013 11:49:31 +0200 Subject: [PATCH 093/269] [#938] Validate datapusher_submit with a schema --- ckanext/datapusher/logic/action.py | 10 +++++++--- ckanext/datapusher/logic/schema.py | 25 +++++++++++++++++++++++++ ckanext/datapusher/plugin.py | 3 ++- ckanext/datastore/logic/action.py | 2 +- 4 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 ckanext/datapusher/logic/schema.py diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 5957887d44e..47a255778f7 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -9,6 +9,7 @@ import ckan.lib.navl.dictization_functions import ckan.logic as logic import ckan.plugins as p +import ckanext.datapusher.logic.schema as dpschema log = logging.getLogger(__name__) _get_or_bust = logic.get_or_bust @@ -40,9 +41,12 @@ def datapusher_submit(context, data_dict): :rtype: boolean ''' - if 'id' in data_dict: - data_dict['resource_id'] = data_dict['id'] - res_id = _get_or_bust(data_dict, 'resource_id') + schema = context.get('schema', dpschema.datapusher_submit_schema()) + data_dict, errors = _validate(data_dict, schema, context) + if errors: + raise p.toolkit.ValidationError(errors) + + res_id = data_dict['resource_id'] p.toolkit.check_access('datapusher_submit', context, data_dict) diff --git a/ckanext/datapusher/logic/schema.py b/ckanext/datapusher/logic/schema.py new file mode 100644 index 00000000000..07e8a36aec8 --- /dev/null +++ b/ckanext/datapusher/logic/schema.py @@ -0,0 +1,25 @@ +import ckan.plugins as p +import ckanext.datastore.logic.schema as dsschema + +get_validator = p.toolkit.get_validator + +not_missing = get_validator('not_missing') +not_empty = get_validator('not_empty') +resource_id_exists = get_validator('resource_id_exists') +package_id_exists = get_validator('package_id_exists') +ignore_missing = get_validator('ignore_missing') +empty = get_validator('empty') +boolean_validator = get_validator('boolean_validator') +int_validator = get_validator('int_validator') +OneOf = get_validator('OneOf') + + +def datapusher_submit_schema(): + schema = { + 'resource_id': [not_missing, not_empty, unicode], + 'id': [ignore_missing], + 'set_url_type': [ignore_missing, boolean_validator], + '__junk': [empty], + '__before': [dsschema.rename('id', 'resource_id')] + } + return schema diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index 17f9022fd22..650282ea9e1 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -47,7 +47,8 @@ def notify(self, entity, operation=None): 'id': entity.get_package_id() }) if (not package['private'] and - entity.format in self.datapusher_formats): + entity.format in self.datapusher_formats and + entity.url_type != 'datapusher'): p.toolkit.get_action('datapusher_submit')(context, { 'resource_id': entity.id }) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 9e93deed315..23dae4dc5b0 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -99,7 +99,7 @@ def datastore_create(context, data_dict): if has_url: p.toolkit.get_action('datapusher_submit')(context, { 'resource_id': res['id'], - 'set_url_to_dump': True + 'set_url_type': True }) # create empty resource else: From 09bde044490fe73c776064cad2fbb26e41652617 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 20 Aug 2013 16:02:02 +0200 Subject: [PATCH 094/269] [#1200] Imports and comments clean up --- ckanext/datapusher/logic/action.py | 16 ++++++++-------- ckanext/datastore/logic/action.py | 9 +++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 47a255778f7..1fe1d5abb8b 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -17,7 +17,7 @@ def datapusher_enabled(context, data_dict): - """ Returns True if the DataPusher is configured """ + ''' Returns True if the DataPusher is configured ''' datapusher_url = pylons.config.get('datapusher.url') return True if datapusher_url else False @@ -30,15 +30,15 @@ def datapusher_submit(context, data_dict): :param resource_id: The resource id of the resource that the data should be imported in. The resource's URL will be used to get the data. :type resource_id: string - :param set_url_type: If set to true, the ``url_type`` of the resource will + :param set_url_type: If set to True, the ``url_type`` of the resource will be set to ``datastore`` and the resource URL will automatically point to the :ref:`datastore dump ` URL. (optional, default: False) - :type set_url_type: boolean + :type set_url_type: bool Returns ``True`` if the job has been submitted and ``False`` if the job has not been submitted, i.e. when the datapusher is not configured. - :rtype: boolean + :rtype: bool ''' schema = context.get('schema', dpschema.datapusher_submit_schema()) @@ -113,11 +113,11 @@ def datapusher_submit(context, data_dict): def datapusher_hook(context, data_dict): - """ Update datapusher task. This action is typically called by the + ''' Update datapusher task. This action is typically called by the datapusher whenever the status of a job changes. Expects a job with ``status`` and ``metadata`` with a ``resource_id``. - """ + ''' # TODO: use a schema to validate @@ -147,12 +147,12 @@ def datapusher_hook(context, data_dict): def datapusher_status(context, data_dict): - """ Get the status of a datapusher job for a certain resource. + ''' Get the status of a datapusher job for a certain resource. :param resource_id: The resource id of the resource that you want the datapusher status for. :type resource_id: string - """ + ''' p.toolkit.check_access('datapusher_status', context, data_dict) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 23dae4dc5b0..489616f49dc 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -1,10 +1,6 @@ import logging -import json -import urlparse -import datetime import pylons -import requests import sqlalchemy import ckan.lib.navl.dictization_functions @@ -101,11 +97,16 @@ def datastore_create(context, data_dict): 'resource_id': res['id'], 'set_url_type': True }) + # since we'll overwrite the datastore resource anyway, we + # don't need to create it here + return # create empty resource else: # no need to set the full url because it will be set in before_show res['url_type'] = 'datastore' p.toolkit.get_action('resource_update')(context, res) + else: + _check_read_only(context, data_dict) data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] From d8a9f37bab6d1db3401117690200c9bf7476437a Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 20 Aug 2013 16:03:13 +0200 Subject: [PATCH 095/269] [#1200] Do not allow writing if url_type != datastore unless force=true is passed --- ckanext/datastore/logic/action.py | 35 ++++++++++++++++++++++---- ckanext/datastore/logic/schema.py | 3 +++ ckanext/datastore/tests/helpers.py | 11 ++++++++ ckanext/datastore/tests/test_create.py | 4 ++- ckanext/datastore/tests/test_delete.py | 4 ++- ckanext/datastore/tests/test_dump.py | 1 + ckanext/datastore/tests/test_search.py | 13 +++++++--- ckanext/datastore/tests/test_upsert.py | 8 +++++- 8 files changed, 68 insertions(+), 11 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 489616f49dc..ee2a44e9e4a 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -35,6 +35,8 @@ def datastore_create(context, data_dict): :param resource_id: resource id that the data is going to be stored against. :type resource_id: string + :param force: set to True to edit a read-only resource + :type force: bool (optional, default: False) :param resource: resource dictionary that is passed to :meth:`~ckan.logic.action.create.resource_create`. Use instead of ``resource_id`` (optional) @@ -154,6 +156,8 @@ def datastore_upsert(context, data_dict): :param resource_id: resource id that the data is going to be stored under. :type resource_id: string + :param force: set to True to edit a read-only resource + :type force: bool (optional, default: False) :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a","b"]}] (optional) :type records: list of dictionaries :param method: the method to use to put the data into the datastore. @@ -174,6 +178,10 @@ def datastore_upsert(context, data_dict): if errors: raise p.toolkit.ValidationError(errors) + p.toolkit.check_access('datastore_upsert', context, data_dict) + + _check_read_only(context, data_dict) + data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] @@ -187,8 +195,6 @@ def datastore_upsert(context, data_dict): u'Resource "{0}" was not found.'.format(res_id) )) - p.toolkit.check_access('datastore_upsert', context, data_dict) - result = db.upsert(context, data_dict) result.pop('id', None) result.pop('connection_url') @@ -200,6 +206,8 @@ def datastore_delete(context, data_dict): :param resource_id: resource id that the data will be deleted from. (optional) :type resource_id: string + :param force: set to True to edit a read-only resource + :type force: bool (optional, default: False) :param filters: filters to apply before deleting (eg {"name": "fred"}). If missing delete whole table and all dependent views. (optional) :type filters: dictionary @@ -218,6 +226,10 @@ def datastore_delete(context, data_dict): if errors: raise p.toolkit.ValidationError(errors) + p.toolkit.check_access('datastore_delete', context, data_dict) + + _check_read_only(context, data_dict) + data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] @@ -231,8 +243,6 @@ def datastore_delete(context, data_dict): u'Resource "{0}" was not found.'.format(res_id) )) - p.toolkit.check_access('datastore_delete', context, data_dict) - result = db.delete(context, data_dict) result.pop('id', None) result.pop('connection_url') @@ -431,7 +441,7 @@ def datastore_make_public(context, data_dict): def _resource_exists(context, data_dict): - # Returns true if the resource exists in CKAN and in the datastore + ''' Returns true if the resource exists in CKAN and in the datastore ''' model = _get_or_bust(context, 'model') res_id = _get_or_bust(data_dict, 'resource_id') if not model.Resource.get(res_id): @@ -441,3 +451,18 @@ def _resource_exists(context, data_dict): WHERE name = :id AND alias_of IS NULL''') results = db._get_engine(data_dict).execute(resources_sql, id=res_id) return results.rowcount > 0 + + +def _check_read_only(context, data_dict): + ''' Raises exception if the resource is read-only. + Make sure the resource id is in resource_id + ''' + if data_dict.get('force'): + return + res = p.toolkit.get_action('resource_show')( + context, {'id': data_dict['resource_id']}) + if res.get('url_type') != 'datastore': + raise p.toolkit.ValidationError({ + 'read-only': ['Cannot edit read-only resource. Either pass' + '"force=True" or change url-type to "datastore"'] + }) diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index 018eb249d79..6f0a74723ca 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -68,6 +68,7 @@ def json_validator(value, context): def datastore_create_schema(): schema = { 'resource_id': [ignore_missing, unicode, resource_id_exists], + 'force': [ignore_missing, boolean_validator], 'id': [ignore_missing], 'aliases': [ignore_missing, list_of_strings_or_string], 'fields': { @@ -85,6 +86,7 @@ def datastore_create_schema(): def datastore_upsert_schema(): schema = { 'resource_id': [not_missing, not_empty, unicode], + 'force': [ignore_missing, boolean_validator], 'id': [ignore_missing], 'method': [ignore_missing, unicode, OneOf( ['upsert', 'insert', 'update'])], @@ -97,6 +99,7 @@ def datastore_upsert_schema(): def datastore_delete_schema(): schema = { 'resource_id': [not_missing, not_empty, unicode], + 'force': [ignore_missing, boolean_validator], 'id': [ignore_missing], '__junk': [empty], '__before': [rename('id', 'resource_id')] diff --git a/ckanext/datastore/tests/helpers.py b/ckanext/datastore/tests/helpers.py index cf83f57378a..3ee89cdda20 100644 --- a/ckanext/datastore/tests/helpers.py +++ b/ckanext/datastore/tests/helpers.py @@ -1,6 +1,8 @@ import ckan.model as model import ckan.lib.cli as cli +import ckan.plugins as p + def extract(d, keys): return dict((k, d[k]) for k in keys if k in d) @@ -29,3 +31,12 @@ def rebuild_all_dbs(Session): model.repo.tables_created_and_initialised = False clear_db(Session) model.repo.rebuild_db() + + +def set_url_type(resources, user): + context = {'user': user.name} + for resource in resources: + resource = p.toolkit.get_action('resource_show')( + context, {'id': resource.id}) + resource['url_type'] = 'datastore' + p.toolkit.get_action('resource_update')(context, resource) diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 7a6933008e1..f95e8044951 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -17,7 +17,7 @@ import ckan.config.middleware as middleware import ckanext.datastore.db as db -from ckanext.datastore.tests.helpers import rebuild_all_dbs +from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type # avoid hanging tests https://github.com/gabrielfalcao/HTTPretty/issues/34 @@ -45,6 +45,8 @@ def setup_class(cls): engine = db._get_engine( {'connection_url': pylons.config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) @classmethod def teardown_class(cls): diff --git a/ckanext/datastore/tests/test_delete.py b/ckanext/datastore/tests/test_delete.py index ce1b02efdb1..bb7db39217a 100644 --- a/ckanext/datastore/tests/test_delete.py +++ b/ckanext/datastore/tests/test_delete.py @@ -11,7 +11,7 @@ import ckan.tests as tests import ckanext.datastore.db as db -from ckanext.datastore.tests.helpers import rebuild_all_dbs +from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type class TestDatastoreDelete(tests.WsgiAppCase): @@ -43,6 +43,8 @@ def setup_class(cls): engine = db._get_engine( {'connection_url': pylons.config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) @classmethod def teardown_class(cls): diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 041105af986..6061535f4ba 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -32,6 +32,7 @@ def setup_class(cls): resource = model.Package.get('annakarenina').resources[0] cls.data = { 'resource_id': resource.id, + 'force': True, 'aliases': 'books', 'fields': [{'id': u'b\xfck', 'type': 'text'}, {'id': 'author', 'type': 'text'}, diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py index 21cb19de4e2..1b7c9486da7 100644 --- a/ckanext/datastore/tests/test_search.py +++ b/ckanext/datastore/tests/test_search.py @@ -30,6 +30,7 @@ def setup_class(cls): cls.resource = cls.dataset.resources[0] cls.data = { 'resource_id': cls.resource.id, + 'force': True, 'aliases': 'books3', 'fields': [{'id': u'b\xfck', 'type': 'text'}, {'id': 'author', 'type': 'text'}, @@ -116,7 +117,7 @@ def test_search_private_dataset(self): context, {'name': 'privatedataset', 'private': True, - 'owner_org' : self.organization['id'], + 'owner_org': self.organization['id'], 'groups': [{ 'id': group.id }]}) @@ -128,6 +129,7 @@ def test_search_private_dataset(self): postparams = '%s=1' % json.dumps({ 'resource_id': resource['id'], + 'force': True }) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_create', params=postparams, @@ -425,6 +427,7 @@ def setup_class(cls): resource = model.Package.get('annakarenina').resources[0] cls.data = dict( resource_id=resource.id, + force=True, fields=[ {'id': 'id'}, {'id': 'date', 'type':'date'}, @@ -499,6 +502,7 @@ def setup_class(cls): resource = cls.dataset.resources[0] cls.data = { 'resource_id': resource.id, + 'force': True, 'aliases': 'books4', 'fields': [{'id': u'b\xfck', 'type': 'text'}, {'id': 'author', 'type': 'text'}, @@ -517,7 +521,7 @@ def setup_class(cls): extra_environ=auth) res_dict = json.loads(res.body) assert res_dict['success'] is True - + # Make an organization, because private datasets must belong to one. cls.organization = tests.call_action_api( cls.app, 'organization_create', @@ -669,6 +673,7 @@ def test_new_datastore_table_from_private_resource(self): postparams = '%s=1' % json.dumps({ 'resource_id': resource['id'], + 'force': True }) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_create', params=postparams, @@ -708,7 +713,9 @@ def test_making_resource_private_makes_datastore_private(self): 'package_id': package['id']}) postparams = '%s=1' % json.dumps({ - 'resource_id': resource['id']}) + 'resource_id': resource['id'], + 'force': True + }) auth = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/datastore_create', params=postparams, extra_environ=auth) diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py index e27f8b9c523..9d65700198a 100644 --- a/ckanext/datastore/tests/test_upsert.py +++ b/ckanext/datastore/tests/test_upsert.py @@ -11,7 +11,7 @@ import ckan.tests as tests import ckanext.datastore.db as db -from ckanext.datastore.tests.helpers import rebuild_all_dbs +from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type class TestDatastoreUpsert(tests.WsgiAppCase): @@ -26,6 +26,8 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) resource = model.Package.get('annakarenina').resources[0] cls.data = { 'resource_id': resource.id, @@ -249,6 +251,8 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) resource = model.Package.get('annakarenina').resources[0] cls.data = { 'resource_id': resource.id, @@ -349,6 +353,8 @@ def setup_class(cls): ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) resource = model.Package.get('annakarenina').resources[0] hhguide = u"hitchhiker's guide to the galaxy" cls.data = { From f2f9095926e143ef7a1fe46f613c88bbf1096478 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Tue, 20 Aug 2013 19:27:49 -0300 Subject: [PATCH 096/269] [#1178] Adds user to organization when inviting, and only org admins can invite --- ckan/logic/action/create.py | 22 +++++++++++---- ckan/logic/auth/create.py | 2 +- ckan/model/user.py | 13 ++++----- ckan/tests/lib/test_mailer.py | 5 +++- ckan/tests/logic/test_action.py | 47 ++++++++++++++++----------------- ckan/tests/logic/test_auth.py | 11 ++++---- 6 files changed, 58 insertions(+), 42 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index edef0a95f30..5e1e2a477e0 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -843,19 +843,25 @@ def user_create(context, data_dict): def user_invite(context, data_dict): '''Invite a new user. - You must be authorized to invite users. + You must be authorized to create organization members. - :param email: the email address for the new user + :param email: the email of the user to be invited to the organization :type email: string + :param organization_id: the id or name of the organization + :type organization_id: string + :param role: role of the user in the group. One of ``member``, ``editor``, + or ``admin`` + :type role: string :returns: the newly created yser :rtype: dictionary - ''' _check_access('user_invite', context, data_dict) user_invite_schema = { - 'email': [validators.not_empty, unicode] + 'email': [validators.not_empty, unicode], + 'organization_id': [validators.not_empty], + 'role': [validators.not_empty], } _, errors = _validate(data_dict, user_invite_schema, context) if errors: @@ -870,6 +876,12 @@ def user_invite(context, data_dict): data_dict['state'] = ckan.model.State.PENDING user_dict = _get_action('user_create')(context, data_dict) user = ckan.model.User.get(user_dict['id']) + member_dict = { + 'username': user.id, + 'id': data_dict['organization_id'], + 'role': data_dict['role'] + } + _get_action('organization_member_create')(context, member_dict) mailer.send_invite(user) return model_dictize.user_dictize(user, context) except ValidationError as e: @@ -1187,7 +1199,7 @@ def _group_or_org_member_create(context, data_dict, is_org=False): role = data_dict.get('role') group_id = data_dict.get('id') group = model.Group.get(group_id) - result = session.query(model.User).filter_by(name=username).first() + result = model.User.get(username) if result: user_id = result.id else: diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index fafbc5a2e4c..55ec2d7df6f 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -113,7 +113,7 @@ def user_create(context, data_dict=None): return {'success': True} def user_invite(context, data_dict=None): - return {'success': False} + return organization_member_create(context, data_dict) def _check_group_auth(context, data_dict): # FIXME This code is shared amoung other logic.auth files and should be diff --git a/ckan/model/user.py b/ckan/model/user.py index a6d6c44b612..4f87175edf4 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -181,20 +181,21 @@ def is_deleted(self): def is_pending(self): return self.state == core.State.PENDING - def is_in_group(self, group): - return group in self.get_group_ids() + def is_in_group(self, group_id): + return group_id in self.get_group_ids() - def is_in_groups(self, groupids): + def is_in_groups(self, group_ids): ''' Given a list of group ids, returns True if this user is in any of those groups ''' guser = set(self.get_group_ids()) - gids = set(groupids) + gids = set(group_ids) return len(guser.intersection(gids)) > 0 - def get_group_ids(self, group_type=None): + def get_group_ids(self, group_type=None, capacity=None): ''' Returns a list of group ids that the current user belongs to ''' - return [g.id for g in self.get_groups(group_type=group_type)] + return [g.id for g in + self.get_groups(group_type=group_type, capacity=capacity)] def get_groups(self, group_type=None, capacity=None): import ckan.model as model diff --git a/ckan/tests/lib/test_mailer.py b/ckan/tests/lib/test_mailer.py index 22b3985930b..83a35c3b92c 100644 --- a/ckan/tests/lib/test_mailer.py +++ b/ckan/tests/lib/test_mailer.py @@ -114,8 +114,10 @@ def test_send_reset_email(self): # reset link tested in user functional test def test_send_invite_email(self): + user = model.User.by_name(u'bob') + assert user.reset_key is None, user # send email - mailer.send_invite(model.User.by_name(u'bob')) + mailer.send_invite(user) # check it went to the mock smtp server msgs = self.get_smtp_messages() @@ -127,5 +129,6 @@ def test_send_invite_email(self): expected_body = self.mime_encode(test_msg, u'bob') assert expected_body in msg[3], '%r not in %r' % (expected_body, msg[3]) + assert user.reset_key is not None, user # reset link tested in user functional test diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index b6361803f0e..f2ef0df9f7d 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -562,12 +562,18 @@ def test_12_user_update_errors(self): for expected_message in test_call['messages']: assert expected_message[1] in ''.join(res_obj['error'][expected_message[0]]) - @mock.patch('ckan.lib.mailer.mail_user') - def test_user_invite(self, mail_user): + @mock.patch('ckan.lib.mailer.send_invite') + def test_user_invite(self, send_invite): email_username = 'invited_user$ckan' email = '%s@email.com' % email_username - user_dict = {'email': email} - postparams = '%s=1' % json.dumps(user_dict) + organization_name = 'an_organization' + CreateTestData.create_groups([{'name': organization_name}]) + role = 'member' + organization = model.Group.get(organization_name) + params = {'email': email, + 'organization_id': organization.id, + 'role': role} + postparams = '%s=1' % json.dumps(params) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} res = self.app.post('/api/action/user_invite', params=postparams, @@ -575,27 +581,14 @@ def test_user_invite(self, mail_user): res_obj = json.loads(res.body) user = model.User.get(res_obj['result']['id']) - expected_username = email_username.replace('$', '-') assert res_obj['success'] is True, res_obj assert user.email == email, (user.email, email) - assert user.name.startswith(expected_username), (user.name, expected_username) assert user.is_pending(), user - assert user.reset_key is not None, user - - @mock.patch('ckan.lib.mailer.send_invite') - def test_user_invite_sends_email(self, send_invite): - email_username = 'invited_user' - email = '%s@email.com' % email_username - user_dict = {'email': email} - postparams = '%s=1' % json.dumps(user_dict) - extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} - - res = self.app.post('/api/action/user_invite', params=postparams, - extra_environ=extra_environ) - - res_obj = json.loads(res.body) - user = model.User.get(res_obj['result']['id']) - assert res_obj['success'] is True, res_obj + expected_username = email_username.replace('$', '-') + assert user.name.startswith(expected_username), (user.name, + expected_username) + group_ids = user.get_group_ids(capacity=role) + assert organization.id in group_ids, (group_ids, organization.id) assert send_invite.called assert send_invite.call_args[0][0].id == res_obj['result']['id'] @@ -618,8 +611,14 @@ def test_user_invite_should_work_even_if_tried_username_already_exists(self, ran patcher = mock.patch('ckan.lib.mailer.mail_user') patcher.start() email = 'invited_user@email.com' - user_dict = {'email': email} - postparams = '%s=1' % json.dumps(user_dict) + organization_name = 'an_organization' + CreateTestData.create_groups([{'name': organization_name}]) + role = 'member' + organization = model.Group.get(organization_name) + params = {'email': email, + 'organization_id': organization.id, + 'role': role} + postparams = '%s=1' % json.dumps(params) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} usernames = ['first', 'first', 'second'] diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 13e7b38f2a5..9ae69f32081 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -1,3 +1,5 @@ +import mock + import ckan.tests as tests from ckan.logic import get_action import ckan.model as model @@ -49,11 +51,10 @@ def create_user(self, name): class TestAuthUsers(TestAuth): - def test_only_sysadmins_can_invite_users(self): - username = 'normal_user' - self.create_user(username) - - assert not new_authz.is_authorized_boolean('user_invite', {'user': username}) + @mock.patch('ckan.logic.auth.create.organization_member_create') + def test_invite_user_delegates_to_organization_member_create(self, organization_member_create): + new_authz.is_authorized_boolean('user_invite', {}) + organization_member_create.assert_called() def test_only_sysadmins_can_delete_users(self): username = 'username' From 34b4466358dca5f9a6f27dd693cc32e28a3aa644 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Sun, 25 Aug 2013 10:56:41 +0200 Subject: [PATCH 097/269] [#938] Prepend config option with ckan. --- ckan/config/deployment.ini_tmpl | 4 ++-- ckanext/datapusher/logic/action.py | 4 ++-- doc/configuration.rst | 12 ++++++------ test-core.ini | 2 ++ 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index 0729c606f3c..665cc8ec08e 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -138,8 +138,8 @@ ckan.feeds.author_link = # Make sure you have set up the DataStore -datapusher.formats = csv -datapusher.url = http://datapusher.ckan.org/ +ckan.datapusher.formats = csv +ckan.datapusher.url = http://datapusher.ckan.org/ ## Activity Streams Settings diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 1fe1d5abb8b..c7161a9ccc5 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -53,7 +53,7 @@ def datapusher_submit(context, data_dict): if not p.toolkit.get_action('datapusher_enabled')(context): return False - datapusher_url = pylons.config.get('datapusher.url') + datapusher_url = pylons.config.get('ckan.datapusher.url') callback_url = p.toolkit.url_for( controller='api', action='action', logic_function='datapusher_hook', @@ -172,7 +172,7 @@ def datapusher_status(context, data_dict): 'key': 'job_key' }) - datapusher_url = pylons.config.get('datapusher.url') + datapusher_url = pylons.config.get('ckan.datapusher.url') if not datapusher_url: raise p.toolkit.ValidationError( {'configuration': ['DataPusher not configured.']}) diff --git a/doc/configuration.rst b/doc/configuration.rst index 284dbbfde76..88794f2d05a 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1065,23 +1065,23 @@ Only used with the Google storage backend. DataPusher Settings ------------------- -.. _datapusher.formats: +.. _ckan.datapusher.formats: -datapusher.formats +ckan.datapusher.formats ^^^^^^^^^^^^^^^^^^ Example:: - datapusher.formats = csv xls xlsx + ckan.datapusher.formats = csv xls xlsx .. todo:: Expand -.. _datapusher.url +.. _ckan.datapusher.url -datapusher.url +ckan.datapusher.url ^^^^^^^^^^^^^^ Example:: - datapusher.url = http://datapusher.ckan.org/ + ckan.datapusher.url = http://datapusher.ckan.org/ .. todo:: Expand diff --git a/test-core.ini b/test-core.ini index 21dc08cfae9..da5c1a0814a 100644 --- a/test-core.ini +++ b/test-core.ini @@ -25,6 +25,8 @@ sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_test ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_test ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_test +ckan.datapusher.url = http://datapusher.ckan.org/ + ## Solr support solr_url = http://127.0.0.1:8983/solr From 91f6a38ec63f6fe5d31598230eb32a346a39ca9f Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Sun, 25 Aug 2013 10:57:36 +0200 Subject: [PATCH 098/269] [#1196] Move datapusher tests --- ckanext/datapusher/tests/__init__.py | 0 ckanext/datapusher/tests/test.py | 192 +++++++++++++++++++++++++ ckanext/datastore/tests/test_create.py | 140 ------------------ 3 files changed, 192 insertions(+), 140 deletions(-) create mode 100644 ckanext/datapusher/tests/__init__.py create mode 100644 ckanext/datapusher/tests/test.py diff --git a/ckanext/datapusher/tests/__init__.py b/ckanext/datapusher/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py new file mode 100644 index 00000000000..f867274d50d --- /dev/null +++ b/ckanext/datapusher/tests/test.py @@ -0,0 +1,192 @@ +import json +import httpretty +import nose +import sys +import datetime + +import pylons +from pylons import config +import sqlalchemy.orm as orm +import paste.fixture + +import ckan.plugins as p +import ckan.lib.create_test_data as ctd +import ckan.model as model +import ckan.tests as tests +import ckan.config.middleware as middleware + +import ckanext.datastore.db as db +from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type + + +# avoid hanging tests https://github.com/gabrielfalcao/HTTPretty/issues/34 +if sys.version_info < (2, 7, 0): + import socket + socket.setdefaulttimeout(1) + + +class TestDatastoreCreate(tests.WsgiAppCase): + sysadmin_user = None + normal_user = None + + @classmethod + def setup_class(cls): + + wsgiapp = middleware.make_app(config['global_conf'], **config) + cls.app = paste.fixture.TestApp(wsgiapp) + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") + p.load('datastore') + p.load('datapusher') + ctd.CreateTestData.create() + cls.sysadmin_user = model.User.get('testsysadmin') + cls.normal_user = model.User.get('annafan') + engine = db._get_engine( + {'connection_url': pylons.config['ckan.datastore.write_url']}) + cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) + set_url_type( + model.Package.get('annakarenina').resources, cls.sysadmin_user) + + @classmethod + def teardown_class(cls): + rebuild_all_dbs(cls.Session) + p.unload('datastore') + p.unload('datapusher') + + def test_create_ckan_resource_in_package(self): + package = model.Package.get('annakarenina') + data = { + 'resource': {'package_id': package.id} + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=200) + res_dict = json.loads(res.body) + + assert 'resource_id' in res_dict['result'] + assert len(model.Package.get('annakarenina').resources) == 3 + + res = tests.call_action_api( + self.app, 'resource_show', id=res_dict['result']['resource_id']) + assert res['url'] == '/datastore/dump/' + res['id'], res + + @httpretty.activate + def test_providing_res_with_url_calls_datapusher_correctly(self): + pylons.config['datapusher.url'] = 'http://datapusher.ckan.org' + httpretty.HTTPretty.register_uri( + httpretty.HTTPretty.POST, + 'http://datapusher.ckan.org/job', + content_type='application/json', + body=json.dumps({'job_id': 'foo', 'job_key': 'bar'})) + + package = model.Package.get('annakarenina') + + tests.call_action_api( + self.app, 'datastore_create', apikey=self.sysadmin_user.apikey, + resource=dict(package_id=package.id, url='demo.ckan.org')) + + assert len(package.resources) == 4, len(package.resources) + resource = package.resources[3] + data = json.loads(httpretty.last_request().body) + assert data['metadata']['resource_id'] == resource.id, data + assert data['result_url'].endswith('/action/datapusher_hook'), data + assert data['result_url'].startswith('http://'), data + + def test_cant_provide_resource_and_resource_id(self): + package = model.Package.get('annakarenina') + resource = package.resources[0] + data = { + 'resource_id': resource.id, + 'resource': {'package_id': package.id} + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datastore_create', params=postparams, + extra_environ=auth, status=409) + res_dict = json.loads(res.body) + + assert res_dict['error']['__type'] == 'Validation Error' + + @httpretty.activate + def test_send_datapusher_creates_task(self): + httpretty.HTTPretty.register_uri( + httpretty.HTTPretty.POST, + 'http://datapusher.ckan.org/job', + content_type='application/json', + body=json.dumps({'job_id': 'foo', 'job_key': 'bar'})) + + package = model.Package.get('annakarenina') + resource = package.resources[0] + + context = { + 'ignore_auth': True, + 'user': self.sysadmin_user.name + } + + p.toolkit.get_action('datapusher_submit')(context, { + 'resource_id': resource.id + }) + + task = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': resource.id, + 'task_type': 'datapusher', + 'key': 'job_id' + }) + + assert task['state'] == 'pending', task + + def test_datapusher_hook(self): + package = model.Package.get('annakarenina') + resource = package.resources[0] + + context = { + 'user': self.sysadmin_user.name + } + + p.toolkit.get_action('task_status_update')(context, { + 'entity_id': resource.id, + 'entity_type': 'resource', + 'task_type': 'datapusher', + 'key': 'job_id', + 'value': 'my_id', + 'last_updated': str(datetime.datetime.now()), + 'state': 'pending' + }) + + p.toolkit.get_action('task_status_update')(context, { + 'entity_id': resource.id, + 'entity_type': 'resource', + 'task_type': 'datapusher', + 'key': 'job_key', + 'value': 'my_key', + 'last_updated': str(datetime.datetime.now()), + 'state': 'pending' + }) + + data = { + 'status': 'success', + 'metadata': { + 'resource_id': resource.id + } + } + postparams = '%s=1' % json.dumps(data) + auth = {'Authorization': str(self.sysadmin_user.apikey)} + res = self.app.post('/api/action/datapusher_hook', params=postparams, + extra_environ=auth, status=200) + print res.body + res_dict = json.loads(res.body) + + assert res_dict['success'] is True + + task = tests.call_action_api( + self.app, 'task_status_show', entity_id=resource.id, + task_type='datapusher', key='job_id') + + assert task['state'] == 'success', task + + task = tests.call_action_api( + self.app, 'task_status_show', entity_id=resource.id, + task_type='datapusher', key='job_key') + + assert task['state'] == 'success', task \ No newline at end of file diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index f95e8044951..f2db232af8a 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -38,7 +38,6 @@ def setup_class(cls): if not tests.is_datastore_supported(): raise nose.SkipTest("Datastore not supported") p.load('datastore') - p.load('datapusher') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') @@ -52,7 +51,6 @@ def setup_class(cls): def teardown_class(cls): rebuild_all_dbs(cls.Session) p.unload('datastore') - p.unload('datapusher') def test_create_requires_auth(self): resource = model.Package.get('annakarenina').resources[0] @@ -540,144 +538,6 @@ def test_create_basic(self): assert res_dict['success'] is True, res_dict - def test_create_ckan_resource_in_package(self): - package = model.Package.get('annakarenina') - data = { - 'resource': {'package_id': package.id} - } - postparams = '%s=1' % json.dumps(data) - auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=200) - res_dict = json.loads(res.body) - - assert 'resource_id' in res_dict['result'] - assert len(model.Package.get('annakarenina').resources) == 3 - - res = tests.call_action_api( - self.app, 'resource_show', id=res_dict['result']['resource_id']) - assert res['url'] == '/datastore/dump/' + res['id'], res - - @httpretty.activate - def test_providing_res_with_url_calls_datapusher_correctly(self): - pylons.config['datapusher.url'] = 'http://datapusher.ckan.org' - httpretty.HTTPretty.register_uri( - httpretty.HTTPretty.POST, - 'http://datapusher.ckan.org/job', - content_type='application/json', - body=json.dumps({'job_id': 'foo', 'job_key': 'bar'})) - - package = model.Package.get('annakarenina') - - tests.call_action_api( - self.app, 'datastore_create', apikey=self.sysadmin_user.apikey, - resource=dict(package_id=package.id, url='demo.ckan.org')) - - assert len(package.resources) == 4, len(package.resources) - resource = package.resources[3] - data = json.loads(httpretty.last_request().body) - assert data['metadata']['resource_id'] == resource.id, data - assert data['result_url'].endswith('/action/datapusher_hook'), data - assert data['result_url'].startswith('http://'), data - - def test_cant_provide_resource_and_resource_id(self): - package = model.Package.get('annakarenina') - resource = package.resources[0] - data = { - 'resource_id': resource.id, - 'resource': {'package_id': package.id} - } - postparams = '%s=1' % json.dumps(data) - auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datastore_create', params=postparams, - extra_environ=auth, status=409) - res_dict = json.loads(res.body) - - assert res_dict['error']['__type'] == 'Validation Error' - - @httpretty.activate - def test_send_datapusher_creates_task(self): - httpretty.HTTPretty.register_uri( - httpretty.HTTPretty.POST, - 'http://datapusher.ckan.org/job', - content_type='application/json', - body=json.dumps({'job_id': 'foo', 'job_key': 'bar'})) - - package = model.Package.get('annakarenina') - resource = package.resources[0] - - context = { - 'ignore_auth': True, - 'user': self.sysadmin_user.name - } - - p.toolkit.get_action('datapusher_submit')(context, { - 'resource_id': resource.id - }) - - task = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': resource.id, - 'task_type': 'datapusher', - 'key': 'job_id' - }) - - assert task['state'] == 'pending', task - - def test_datapusher_hook(self): - package = model.Package.get('annakarenina') - resource = package.resources[0] - - context = { - 'user': self.sysadmin_user.name - } - - p.toolkit.get_action('task_status_update')(context, { - 'entity_id': resource.id, - 'entity_type': 'resource', - 'task_type': 'datapusher', - 'key': 'job_id', - 'value': 'my_id', - 'last_updated': str(datetime.datetime.now()), - 'state': 'pending' - }) - - p.toolkit.get_action('task_status_update')(context, { - 'entity_id': resource.id, - 'entity_type': 'resource', - 'task_type': 'datapusher', - 'key': 'job_key', - 'value': 'my_key', - 'last_updated': str(datetime.datetime.now()), - 'state': 'pending' - }) - - data = { - 'status': 'success', - 'metadata': { - 'resource_id': resource.id - } - } - postparams = '%s=1' % json.dumps(data) - auth = {'Authorization': str(self.sysadmin_user.apikey)} - res = self.app.post('/api/action/datapusher_hook', params=postparams, - extra_environ=auth, status=200) - print res.body - res_dict = json.loads(res.body) - - assert res_dict['success'] is True - - task = tests.call_action_api( - self.app, 'task_status_show', entity_id=resource.id, - task_type='datapusher', key='job_id') - - assert task['state'] == 'success', task - - task = tests.call_action_api( - self.app, 'task_status_show', entity_id=resource.id, - task_type='datapusher', key='job_key') - - assert task['state'] == 'success', task - def test_guess_types(self): resource = model.Package.get('annakarenina').resources[1] From ef7b3e94c3bc09d60086bbc4ae8b320ad8c61b94 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Sun, 25 Aug 2013 11:13:21 +0200 Subject: [PATCH 099/269] [#1183] Remove datapusher_enabled action and move before_show to datstore. Instead of using the action or the template helper, check whether the plugin is enabled. Moved the action so that changing the url works even if the datapusher is disabled. --- ckan/templates/package/resource_read.html | 4 +++- .../templates/snippets/datapusher_status.html | 14 +++++------ ckanext/datapusher/helpers.py | 4 ---- ckanext/datapusher/logic/action.py | 10 -------- ckanext/datapusher/plugin.py | 24 +++++++------------ ckanext/datastore/logic/action.py | 17 +++++++++---- ckanext/datastore/plugin.py | 13 +++++++++- 7 files changed, 41 insertions(+), 45 deletions(-) diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index d4433d4a4a8..f914780d179 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -27,7 +27,9 @@
    {% block resource_actions_inner %} {% if h.check_access('package_update', {'id':pkg.id }) %} -
  • {% snippet 'snippets/datapusher_status.html', resource=res %}
    • + {% if 'datapusher' in g.plugins %} +
  • {% snippet 'snippets/datapusher_status.html', resource=res %}
    • + {% endif %}
  • {% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}
    • {% endif %} {% if res.url %} diff --git a/ckan/templates/snippets/datapusher_status.html b/ckan/templates/snippets/datapusher_status.html index 863faeee491..3aa8908aef8 100644 --- a/ckan/templates/snippets/datapusher_status.html +++ b/ckan/templates/snippets/datapusher_status.html @@ -4,13 +4,11 @@ #} {% if resource.datastore_active %} - {% if h.datapusher_enabled() %} - {% set job = h.datapusher_status(resource.id) %} - {% set title = _('Datapusher status: {status}.').format(status=job.status) %} - {% if job.status == 'unknown' %} - - {% else %} - - {% endif %} + {% set job = h.datapusher_status(resource.id) %} + {% set title = _('Datapusher status: {status}.').format(status=job.status) %} + {% if job.status == 'unknown' %} + + {% else %} + {% endif %} {% endif %} diff --git a/ckanext/datapusher/helpers.py b/ckanext/datapusher/helpers.py index bf06b2e063a..d5291425b28 100644 --- a/ckanext/datapusher/helpers.py +++ b/ckanext/datapusher/helpers.py @@ -9,7 +9,3 @@ def datapusher_status(resource_id): return { 'status': 'unknown' } - - -def datapusher_enabled(): - return toolkit.get_action('datapusher_enabled')({}) diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index c7161a9ccc5..41d3e063284 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -16,13 +16,6 @@ _validate = ckan.lib.navl.dictization_functions.validate -def datapusher_enabled(context, data_dict): - ''' Returns True if the DataPusher is configured ''' - - datapusher_url = pylons.config.get('datapusher.url') - return True if datapusher_url else False - - def datapusher_submit(context, data_dict): ''' Submit a job to the datapusher. The datapusher is a service that imports tabular data into the datastore. @@ -50,9 +43,6 @@ def datapusher_submit(context, data_dict): p.toolkit.check_access('datapusher_submit', context, data_dict) - if not p.toolkit.get_action('datapusher_enabled')(context): - return False - datapusher_url = pylons.config.get('ckan.datapusher.url') callback_url = p.toolkit.url_for( diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index 650282ea9e1..b20f464d789 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -23,7 +23,6 @@ class DatapusherPlugin(p.SingletonPlugin): p.implements(p.IAuthFunctions) p.implements(p.IResourceUrlChange) p.implements(p.IDomainObjectModification, inherit=True) - p.implements(p.IResourceController, inherit=True) p.implements(p.ITemplateHelpers) legacy_mode = False @@ -32,9 +31,14 @@ class DatapusherPlugin(p.SingletonPlugin): def configure(self, config): self.config = config - datapusher_formats = config.get('datapusher.formats', '').split() + datapusher_formats = config.get('ckan.datapusher.formats', '').split() self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS + datapusher_url = config.get('ckan.datapusher.url') + if not datapusher_url: + raise Exception( + 'Config option `ckan.datapusher.url` has to be set.') + def notify(self, entity, operation=None): if isinstance(entity, model.Resource): if (operation == model.domain_object.DomainObjectOperation.new @@ -54,8 +58,7 @@ def notify(self, entity, operation=None): }) def get_actions(self): - return {'datapusher_enabled': action.datapusher_enabled, - 'datapusher_submit': action.datapusher_submit, + return {'datapusher_submit': action.datapusher_submit, 'datapusher_hook': action.datapusher_hook, 'datapusher_status': action.datapusher_status} @@ -63,17 +66,6 @@ def get_auth_functions(self): return {'datapusher_submit': auth.datapusher_submit, 'datapusher_status': auth.datapusher_status} - def before_show(self, resource_dict): - ''' Modify the resource url of datastore resources so that - they link to the datastore dumps. - ''' - if resource_dict['url_type'] == 'datastore': - resource_dict['url'] = p.toolkit.url_for( - controller='ckanext.datastore.controller:DatastoreController', - action='dump', resource_id=resource_dict['id']) - return resource_dict - def get_helpers(self): return { - 'datapusher_status': helpers.datapusher_status, - 'datapusher_enabled': helpers.datapusher_enabled} + 'datapusher_status': helpers.datapusher_status} diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index ee2a44e9e4a..742462ba8ae 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -88,20 +88,27 @@ def datastore_create(context, data_dict): if 'resource' in data_dict: has_url = 'url' in data_dict['resource'] - data_dict['resource'].setdefault('url', '_tmp') + # A datastore only resource does not have a url in the db + data_dict['resource'].setdefault('url', '_datastore_only_resource') res = p.toolkit.get_action('resource_create')(context, data_dict['resource']) data_dict['resource_id'] = res['id'] # create resource from file if has_url: - p.toolkit.get_action('datapusher_submit')(context, { - 'resource_id': res['id'], - 'set_url_type': True - }) + try: + p.toolkit.get_action('datapusher_submit')(context, { + 'resource_id': res['id'], + 'set_url_type': True + }) + except KeyError: + raise p.toolkit.ValidationError({'resource': [ + 'The datapusher has to be enabled.']}) + # since we'll overwrite the datastore resource anyway, we # don't need to create it here return + # create empty resource else: # no need to set the full url because it will be set in before_show diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 8eb753f8e46..61cdd783d81 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -25,6 +25,7 @@ class DatastorePlugin(p.SingletonPlugin): p.implements(p.IResourceUrlChange) p.implements(p.IDomainObjectModification, inherit=True) p.implements(p.IRoutes, inherit=True) + p.implements(p.IResourceController, inherit=True) legacy_mode = False resource_show_action = None @@ -250,4 +251,14 @@ def before_map(self, m): m.connect('/datastore/dump/{resource_id}', controller='ckanext.datastore.controller:DatastoreController', action='dump') - return m \ No newline at end of file + return m + + def before_show(self, resource_dict): + ''' Modify the resource url of datastore resources so that + they link to the datastore dumps. + ''' + if resource_dict['url_type'] == 'datastore': + resource_dict['url'] = p.toolkit.url_for( + controller='ckanext.datastore.controller:DatastoreController', + action='dump', resource_id=resource_dict['id']) + return resource_dict From 4e16bc1abc7578ebcfd564482355f7a5d671a493 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Sun, 25 Aug 2013 11:29:54 +0200 Subject: [PATCH 100/269] PEP8 --- ckanext/datapusher/tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index f867274d50d..f47f256cef6 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -189,4 +189,4 @@ def test_datapusher_hook(self): self.app, 'task_status_show', entity_id=resource.id, task_type='datapusher', key='job_key') - assert task['state'] == 'success', task \ No newline at end of file + assert task['state'] == 'success', task From 98508d1a67b0ec0a86a1786988f3195d6535f2be Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 26 Aug 2013 16:13:48 +0200 Subject: [PATCH 101/269] [#1011] Fix a broken import in the package controller An exception was not getting caught because of the way this import was done. --- ckan/controllers/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index c9940cd57ac..e23c6e39fa1 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -19,7 +19,7 @@ import ckan.lib.datapreview as datapreview import ckan.lib.plugins import ckan.plugins as p -import lib.render +import ckan.lib.render from ckan.common import OrderedDict, _, json, request, c, g, response from home import CACHE_PARAMETERS @@ -366,7 +366,7 @@ def read(self, id, format='html'): try: return render(template, loader_class=loader) - except lib.render.TemplateNotFound: + except ckan.lib.render.TemplateNotFound: msg = _("Viewing {package_type} datasets in {format} format is " "not supported (template file {file} not found).".format( package_type=package_type, format=format, file=template)) From 2d3bcdd7a04b27be135045cf388ef9c983a3c16a Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 26 Aug 2013 16:14:47 +0200 Subject: [PATCH 102/269] [#1011] base.py: Explicitly re-raise TemplateNotFound Functions should not implicitly raise exceptions by failing to catch exceptions raised by functions that they call, they should explicitly catch and re-raise the exceptions instead. --- ckan/lib/base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index a91a1170135..cf5cf031f76 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -104,7 +104,10 @@ def render_template(): # we remove it so any bad templates crash and burn del globs['url'] - template_path, template_type = render_.template_info(template_name) + try: + template_path, template_type = render_.template_info(template_name) + except render_.TemplateNotFound: + raise # snippets should not pass the context # but allow for legacy genshi templates @@ -202,6 +205,8 @@ def render_template(): raise ckan.exceptions.CkanUrlException( '\nAn Exception has been raised for template %s\n%s' % (template_name, e.message)) + except render_.TemplateNotFound: + raise class ValidationException(Exception): From c136aac9c65fcde7c03a2e1c7a9e88e4b5b2f595 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 26 Aug 2013 16:16:03 +0200 Subject: [PATCH 103/269] [#1011] Update some exception handling in template.py Catch the new ckan.lib.render.TemplateNotFound exception that render() now raises, not the Genshi exception it used to. --- ckan/controllers/template.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ckan/controllers/template.py b/ckan/controllers/template.py index 1ebc700345b..0755b1a2cf5 100644 --- a/ckan/controllers/template.py +++ b/ckan/controllers/template.py @@ -1,6 +1,5 @@ -from genshi.template.loader import TemplateNotFound - import ckan.lib.base as base +import ckan.lib.render class TemplateController(base.BaseController): @@ -29,11 +28,11 @@ def view(self, url): """ try: return base.render(url) - except TemplateNotFound: + except ckan.lib.render.TemplateNotFound: if url.endswith('.html'): base.abort(404) url += '.html' try: return base.render(url) - except TemplateNotFound: + except ckan.lib.render.TemplateNotFound: base.abort(404) From 6b1b6f082a402a43815565a82ba407caeb07edcd Mon Sep 17 00:00:00 2001 From: John Martin Date: Thu, 29 Aug 2013 16:05:50 +0100 Subject: [PATCH 104/269] [#1126] Brings featured groups/orgs from from be06121 into this branch It's just a slightly edited version of https://github.com/okfn/ckan/blob/be06121/ckanext/featured/plugin.py as now the helpers are returning data and not data+html. --- ckanext/homepage/plugin.py | 67 +++++++++++++++++-- .../home/snippets/featured_group.html | 10 +-- .../home/snippets/featured_organization.html | 10 +-- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/ckanext/homepage/plugin.py b/ckanext/homepage/plugin.py index ef429f0457d..e59541a16d1 100644 --- a/ckanext/homepage/plugin.py +++ b/ckanext/homepage/plugin.py @@ -6,19 +6,74 @@ class HomepagePlugin(p.SingletonPlugin): p.implements(p.IConfigurer, inherit=True) + p.implements(p.ITemplateHelpers, inherit=True) p.implements(p.IConfigurable, inherit=True) + featured_groups_cache = None + featured_orgs_cache = None + + def configure(self, config): + groups = config.get('ckan.featured_groups', '') + if groups: + log.warning('Config setting `ckan.featured_groups` is deprecated ' + 'please use `ckanext.homepage.groups`') + self.groups = config.get('ckanext.homepage.groups', groups).split() + self.orgs = config.get('ckanext.homepage.orgs', '').split() + def update_config(self, config): p.toolkit.add_template_directory(config, 'theme/templates') - def get_featured_organization(self): - return + def get_featured_organizations(self, count=1): + orgs = self.featured_group_org(get_action='organization_show', + list_action='organization_list', + count=count, + items=self.orgs) + self.featured_orgs_cache = orgs + return self.featured_orgs_cache + + def get_featured_groups(self, count=1): + groups = self.featured_group_org(get_action='group_show', + list_action='group_list', + count=count, + items=self.groups) + self.featured_groups_cache = groups + return self.featured_groups_cache + + def featured_group_org(self, items, get_action, list_action, count): + def get_group(id): + context = {'ignore_auth': True, + 'limits': {'packages': 2}, + 'for_view': True} + data_dict = {'id': id} + + try: + out = p.toolkit.get_action(get_action)(context, data_dict) + except p.toolkit.ObjectNotFound: + return None + return out + + groups_data = [] + + extras = p.toolkit.get_action(list_action)({}, {}) + + # list of found ids to prevent duplicates + found = [] + for group_name in items + extras: + group = get_group(group_name) + if not group: + continue + # ckeck if duplicate + if group['id'] in found: + continue + found.append(group['id']) + groups_data.append(group) + if len(groups_data) == count: + break - def get_featured_group(self): - return + return groups_data def get_helpers(self): return { - 'get_featured_organization': self.get_featured_organization, - 'get_featured_group': self.get_featured_group, + 'get_featured_organizations': self.get_featured_organizations, + 'get_featured_groups': self.get_featured_groups, } diff --git a/ckanext/homepage/theme/templates/home/snippets/featured_group.html b/ckanext/homepage/theme/templates/home/snippets/featured_group.html index fb5ff22a640..d4069faf618 100644 --- a/ckanext/homepage/theme/templates/home/snippets/featured_group.html +++ b/ckanext/homepage/theme/templates/home/snippets/featured_group.html @@ -1,5 +1,7 @@ -{% set group = h.get_featured_group() %} +{% set groups = h.get_featured_groups() %} -
    - {% snippet 'snippets/group_item.html', group=group, truncate=50, truncate_title=35 %} -
    +{% for group in groups %} +
    + {% snippet 'snippets/group_item.html', group=group, truncate=50, truncate_title=35 %} +
    +{% endfor %} diff --git a/ckanext/homepage/theme/templates/home/snippets/featured_organization.html b/ckanext/homepage/theme/templates/home/snippets/featured_organization.html index 7e43e6c4bcb..7cf9ddb7482 100644 --- a/ckanext/homepage/theme/templates/home/snippets/featured_organization.html +++ b/ckanext/homepage/theme/templates/home/snippets/featured_organization.html @@ -1,5 +1,7 @@ -{% set organization = h.get_featured_organization() %} +{% set organizations = h.get_featured_organizations() %} -
    - {% snippet 'snippets/organization_item.html', organization=organization, truncate=50, truncate_title=35 %} -
    +{% for organization in organizations %} +
    + {% snippet 'snippets/organization_item.html', organization=organization, truncate=50, truncate_title=35 %} +
    +{% endfor %} From b75524ee9b74166428555c6c5f76db7f58c67f14 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Thu, 29 Aug 2013 21:33:52 -0300 Subject: [PATCH 105/269] [#1178] Invite user on add member to organization/group I've added an "Email" field to the "Add Member" to Organization/Group page. If you add an e-mail there, we'll create a new user and invite her (doesn't matter if there's already a user with the same email). I still need to change the template and add help texts. --- ckan/controllers/group.py | 13 +++++++++ ckan/logic/action/create.py | 14 +++++----- ckan/logic/auth/create.py | 3 ++- ckan/model/user.py | 4 +++ ckan/templates/group/member_new.html | 1 + ckan/templates/organization/member_new.html | 1 + ckan/tests/functional/test_group.py | 30 +++++++++++++++++++++ ckan/tests/logic/test_action.py | 4 +-- ckan/tests/logic/test_auth.py | 13 ++++++--- 9 files changed, 69 insertions(+), 14 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index ccef46bfe78..c8775168c8f 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -612,6 +612,19 @@ def member_new(self, id): data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) data_dict['id'] = id + + email = data_dict.get('email') + if email and email != '': + user_data_dict = { + 'email': email, + 'group_id': data_dict['id'], + 'role': data_dict['role'] + } + del data_dict['email'] + user_dict = self._action('user_invite')(context, + user_data_dict) + data_dict['username'] = user_dict['name'] + c.group_dict = self._action('group_member_create')(context, data_dict) self._redirect_to(controller='group', action='members', id=id) else: diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 5e1e2a477e0..7bc750fa10e 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -843,12 +843,12 @@ def user_create(context, data_dict): def user_invite(context, data_dict): '''Invite a new user. - You must be authorized to create organization members. + You must be authorized to create group members. - :param email: the email of the user to be invited to the organization + :param email: the email of the user to be invited to the group :type email: string - :param organization_id: the id or name of the organization - :type organization_id: string + :param group_id: the id or name of the group + :type group_id: string :param role: role of the user in the group. One of ``member``, ``editor``, or ``admin`` :type role: string @@ -860,7 +860,7 @@ def user_invite(context, data_dict): user_invite_schema = { 'email': [validators.not_empty, unicode], - 'organization_id': [validators.not_empty], + 'group_id': [validators.not_empty], 'role': [validators.not_empty], } _, errors = _validate(data_dict, user_invite_schema, context) @@ -878,10 +878,10 @@ def user_invite(context, data_dict): user = ckan.model.User.get(user_dict['id']) member_dict = { 'username': user.id, - 'id': data_dict['organization_id'], + 'id': data_dict['group_id'], 'role': data_dict['role'] } - _get_action('organization_member_create')(context, member_dict) + _get_action('group_member_create')(context, member_dict) mailer.send_invite(user) return model_dictize.user_dictize(user, context) except ValidationError as e: diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 55ec2d7df6f..25e9f18988c 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -113,7 +113,8 @@ def user_create(context, data_dict=None): return {'success': True} def user_invite(context, data_dict=None): - return organization_member_create(context, data_dict) + context['id'] = context.get('group_id') + return group_member_create(context, data_dict) def _check_group_auth(context, data_dict): # FIXME This code is shared amoung other logic.auth files and should be diff --git a/ckan/model/user.py b/ckan/model/user.py index 4f87175edf4..da92a57c11a 100644 --- a/ckan/model/user.py +++ b/ckan/model/user.py @@ -44,6 +44,10 @@ def by_openid(cls, openid): obj = meta.Session.query(cls).autoflush(False) return obj.filter_by(openid=openid).first() + @classmethod + def by_email(cls, email): + return meta.Session.query(cls).filter_by(email=email).all() + @classmethod def get(cls, user_reference): # double slashes in an openid often get turned into single slashes diff --git a/ckan/templates/group/member_new.html b/ckan/templates/group/member_new.html index 87c6dbb60c2..2385a1957b9 100644 --- a/ckan/templates/group/member_new.html +++ b/ckan/templates/group/member_new.html @@ -19,6 +19,7 @@

    {% set format_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/user/autocomplete?q=?'} %} {{ form.input('username', id='field-username', label=_('User'), placeholder=_('Username'), value='', error='', classes=['control-medium'], attrs=format_attrs) }} {% endif %} + {{ form.input('email', label=_('Email'), value=user.email, classes=['control-medium']) }} {% set format_attrs = {'data-module': 'autocomplete'} %} {{ form.select('role', label=_('Role'), options=c.roles, selected=c.user_role, error='', attrs=format_attrs) }}
    diff --git a/ckan/templates/organization/member_new.html b/ckan/templates/organization/member_new.html index dd2c4e50bfa..74487100199 100644 --- a/ckan/templates/organization/member_new.html +++ b/ckan/templates/organization/member_new.html @@ -21,6 +21,7 @@

    {% set format_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/user/autocomplete?q=?'} %} {{ form.input('username', id='field-username', label=_('User'), placeholder=_('Username'), value='', error='', classes=['control-medium'], attrs=format_attrs) }} {% endif %} + {{ form.input('email', label=_('Email'), value=user.email, classes=['control-medium']) }} {% set format_attrs = {'data-module': 'autocomplete'} %} {{ form.select('role', label=_('Role'), options=c.roles, selected=c.user_role, error='', attrs=format_attrs) }}
    diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index b07bbe7ea6f..c5dc63b1097 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -1,6 +1,7 @@ import re from nose.tools import assert_equal +import mock import ckan.tests.test_plugins as test_plugins import ckan.model as model @@ -660,3 +661,32 @@ def test_2_atom_feed(self): assert '' in res, res + +class TestMemberInvite(FunctionalTestCase): + @classmethod + def setup_class(self): + model.Session.remove() + model.repo.rebuild_db() + + def teardown(self): + model.repo.rebuild_db() + + @mock.patch('ckan.lib.mailer.mail_user') + def test_member_new_invites_user_if_received_email(self, mail_user): + user = CreateTestData.create_user('a_user', sysadmin=True) + group_name = 'a_group' + CreateTestData.create_groups([{'name': group_name}], user.name) + group = model.Group.get(group_name) + url = url_for(controller='group', action='member_new', id=group.id) + email = 'invited_user@mailinator.com' + role = 'member' + + params = {'email': email, 'role': role} + res = self.app.post(url, params, + extra_environ={'REMOTE_USER': str(user.name)}) + + users = model.User.by_email(email) + assert len(users) == 1, users + user = users[0] + assert user.email == email, user + assert group.id in user.get_group_ids(capacity=role) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index f2ef0df9f7d..8c1d3e0586d 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -571,7 +571,7 @@ def test_user_invite(self, send_invite): role = 'member' organization = model.Group.get(organization_name) params = {'email': email, - 'organization_id': organization.id, + 'group_id': organization.id, 'role': role} postparams = '%s=1' % json.dumps(params) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} @@ -616,7 +616,7 @@ def test_user_invite_should_work_even_if_tried_username_already_exists(self, ran role = 'member' organization = model.Group.get(organization_name) params = {'email': email, - 'organization_id': organization.id, + 'group_id': organization.id, 'role': role} postparams = '%s=1' % json.dumps(params) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 9ae69f32081..ff8a18ccdf3 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -51,10 +51,15 @@ def create_user(self, name): class TestAuthUsers(TestAuth): - @mock.patch('ckan.logic.auth.create.organization_member_create') - def test_invite_user_delegates_to_organization_member_create(self, organization_member_create): - new_authz.is_authorized_boolean('user_invite', {}) - organization_member_create.assert_called() + @mock.patch('ckan.logic.auth.create.group_member_create') + def test_invite_user_prepares_context_and_delegates_to_group_member_create(self, group_member_create): + context = {'group_id': 42} + group_member_create_context = context + group_member_create_context['id'] = context['group_id'] + + new_authz.is_authorized_boolean('user_invite', context) + + group_member_create.assert_called(group_member_create_context, None) def test_only_sysadmins_can_delete_users(self): username = 'username' From 89db2ed23ca5cb0136a44d2829f2e713f2b2558d Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Sat, 31 Aug 2013 11:26:31 -0300 Subject: [PATCH 106/269] [#1178] Move migration from 070 to 071, as we have a new 070 --- ...umn_to_user_table.py => 071_add_state_column_to_user_table.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ckan/migration/versions/{070_add_state_column_to_user_table.py => 071_add_state_column_to_user_table.py} (100%) diff --git a/ckan/migration/versions/070_add_state_column_to_user_table.py b/ckan/migration/versions/071_add_state_column_to_user_table.py similarity index 100% rename from ckan/migration/versions/070_add_state_column_to_user_table.py rename to ckan/migration/versions/071_add_state_column_to_user_table.py From 52c93774c83b13cbfa98107cba87aed6ad9d1b13 Mon Sep 17 00:00:00 2001 From: joetsoi Date: Wed, 4 Sep 2013 17:00:05 +0100 Subject: [PATCH 107/269] [#1224] propogate the NotAuthorized exception extra messages to api --- ckan/controllers/api.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index 22ea7d4f5c3..795488ee02a 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -113,8 +113,10 @@ def _finish_ok(self, response_data=None, return self._finish(status_int, response_data, content_type) - def _finish_not_authz(self): + def _finish_not_authz(self, extra_msg=None): response_data = _('Access denied') + if extra_msg: + response_data = '%s - %s' % (response_data, extra_msg) return self._finish(403, response_data, 'json') def _finish_not_found(self, extra_msg=None): @@ -193,10 +195,14 @@ def action(self, logic_function, ver=None): 'data': request_data} return_dict['success'] = False return self._finish(400, return_dict, content_type='json') - except NotAuthorized: + except NotAuthorized, e: return_dict['error'] = {'__type': 'Authorization Error', 'message': _('Access denied')} return_dict['success'] = False + + if e.extra_msg: + return_dict['error']['message'] += ': %s' % e.extra_msg + return self._finish(403, return_dict, content_type='json') except NotFound, e: return_dict['error'] = {'__type': 'Not Found Error', @@ -275,8 +281,9 @@ def list(self, ver=None, register=None, subregister=None, id=None): except NotFound, e: extra_msg = e.extra_msg return self._finish_not_found(extra_msg) - except NotAuthorized: - return self._finish_not_authz() + except NotAuthorized, e: + extra_msg = e.extra_msg + return self._finish_not_authz(extra_msg) def show(self, ver=None, register=None, subregister=None, id=None, id2=None): @@ -306,8 +313,9 @@ def show(self, ver=None, register=None, subregister=None, except NotFound, e: extra_msg = e.extra_msg return self._finish_not_found(extra_msg) - except NotAuthorized: - return self._finish_not_authz() + except NotAuthorized, e: + extra_msg = e.extra_msg + return self._finish_not_authz(extra_msg) def _represent_package(self, package): return package.as_dict(ref_package_by=self.ref_package_by, @@ -352,8 +360,9 @@ def create(self, ver=None, register=None, subregister=None, data_dict.get("id"))) return self._finish_ok(response_data, resource_location=location) - except NotAuthorized: - return self._finish_not_authz() + except NotAuthorized, e: + extra_msg = e.extra_msg + return self._finish_not_authz(extra_msg) except NotFound, e: extra_msg = e.extra_msg return self._finish_not_found(extra_msg) @@ -408,8 +417,9 @@ def update(self, ver=None, register=None, subregister=None, try: response_data = action(context, data_dict) return self._finish_ok(response_data) - except NotAuthorized: - return self._finish_not_authz() + except NotAuthorized, e: + extra_msg = e.extra_msg + return self._finish_not_authz(extra_msg) except NotFound, e: extra_msg = e.extra_msg return self._finish_not_found(extra_msg) @@ -456,8 +466,9 @@ def delete(self, ver=None, register=None, subregister=None, try: response_data = action(context, data_dict) return self._finish_ok(response_data) - except NotAuthorized: - return self._finish_not_authz() + except NotAuthorized, e: + extra_msg = e.extra_msg + return self._finish_not_authz(extra_msg) except NotFound, e: extra_msg = e.extra_msg return self._finish_not_found(extra_msg) From 910ca29a902b265e5788864f533a70d8626cbcf1 Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 14:59:14 +0530 Subject: [PATCH 108/269] New option to disable user creation via web --- ckan/config/deployment.ini_tmpl | 1 + ckan/logic/auth/create.py | 22 +++++++++++++++------- ckan/new_authz.py | 2 ++ ckan/templates/header.html | 4 +++- ckan/templates/user/login.html | 20 +++++++++++--------- test-core.ini | 3 ++- 6 files changed, 34 insertions(+), 18 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index 97c4f9412ba..6a0833d2d9f 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -66,6 +66,7 @@ ckan.auth.user_create_organizations = true ckan.auth.user_delete_groups = true ckan.auth.user_delete_organizations = true ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true ## Search Settings diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index e96b789bcf5..35a9f39d876 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -103,14 +103,22 @@ def rating_create(context, data_dict): # No authz check in the logic function return {'success': True} -def user_create(context, data_dict=None): - user = context['user'] - if ('api_version' in context - and not new_authz.check_config_permission('create_user_via_api')): - return {'success': False, 'msg': _('User %s not authorized to create users') % user} - else: - return {'success': True} +def user_create(context, data_dict=None): + # create_user_via_api is deprecated + using_api = 'api_version' in context + create_user_via_api = new_authz.check_config_permission( + 'create_user_via_api') + create_user_via_web = new_authz.check_config_permission( + 'create_user_via_web') + + if using_api and not create_user_via_api: + return {'success': False, 'msg': _('User {user} not authorized to ' + 'create users via the API').format(user=context.get('user'))} + if not using_api and not create_user_via_web: + return {'success': False, 'msg': _('Not authorized to ' + 'create users')} + return {'success': True} def _check_group_auth(context, data_dict): diff --git a/ckan/new_authz.py b/ckan/new_authz.py index 00a7dfe0e17..03ca209faf4 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -86,6 +86,7 @@ def _build(self): def clear_auth_functions_cache(): _AuthFunctions.clear() + CONFIG_PERMISSIONS.clear() def auth_functions_list(): @@ -319,6 +320,7 @@ def get_user_id_for_username(user_name, allow_none=False): 'user_delete_groups': True, 'user_delete_organizations': True, 'create_user_via_api': False, + 'create_user_via_web': True, } CONFIG_PERMISSIONS = {} diff --git a/ckan/templates/header.html b/ckan/templates/header.html index 5b4ebe225a7..268b370e968 100644 --- a/ckan/templates/header.html +++ b/ckan/templates/header.html @@ -49,7 +49,9 @@
      {% block header_account_notlogged %}
    • {% link_for _('Log in'), controller='user', action='login' %}
      • -
    • {% link_for _('Register'), controller='user', action='register', class_='sub' %}
      • + {% if h.check_access('user_create') %} +
    • {% link_for _('Register'), controller='user', action='register', class_='sub' %}
      • + {% endif %} {% endblock %}
    diff --git a/ckan/templates/user/login.html b/ckan/templates/user/login.html index c84e3522748..98c99510570 100644 --- a/ckan/templates/user/login.html +++ b/ckan/templates/user/login.html @@ -18,15 +18,17 @@

    {% block page_heading %}{{ _('Login') }}{% endblock %}< {% endblock %} {% block secondary_content %} -
    -

    {{ _('Need an Account?') }}

    -
    -

    {% trans %}Then sign right up, it only takes a minute.{% endtrans %}

    -

    - {{ _('Create an Account') }} -

    -
    -
    + {% if h.check_access('user_create') %} +
    +

    {{ _('Need an Account?') }}

    +
    +

    {% trans %}Then sign right up, it only takes a minute.{% endtrans %}

    +

    + {{ _('Create an Account') }} +

    +
    +
    + {% endif %}

    {{ _('Forgotten your details?') }}

    diff --git a/test-core.ini b/test-core.ini index 21dc08cfae9..08aba34ed24 100644 --- a/test-core.ini +++ b/test-core.ini @@ -31,6 +31,7 @@ solr_url = http://127.0.0.1:8983/solr ckan.auth.user_create_organizations = true ckan.auth.user_create_groups = true ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true ckan.auth.create_dataset_if_not_in_organization = true ckan.auth.anon_create_dataset = false ckan.auth.user_delete_groups=true @@ -80,7 +81,7 @@ smtp.mail_from = info@test.ckan.net ckan.locale_default = en ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv -ckan.locales_filtered_out = +ckan.locales_filtered_out = ckan.datastore.enabled = 1 From 6bc0c3ee3b9ea158ad358c5697e26eedd372ba9f Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 14:59:50 +0530 Subject: [PATCH 109/269] Add documentation for new config option --- doc/configuration.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index 0efe9573226..853ffd23939 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -355,6 +355,20 @@ Default value: ``False`` Allow new user accounts to be created via the API. +.. _ckan.auth.create_user_via_web: + +ckan.auth.create_user_via_api +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.auth.create_user_via_web = True + +Default value: ``True`` + + +Allow new user accounts to be created via the Web. + .. end_config-authorization From c75e487ff99765418d5c82e9f683e2d694afc98d Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 16:46:10 +0530 Subject: [PATCH 110/269] Fix docs, remove deprecated comment --- ckan/logic/auth/create.py | 1 - doc/configuration.rst | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 35a9f39d876..ec38483bd0d 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -105,7 +105,6 @@ def rating_create(context, data_dict): def user_create(context, data_dict=None): - # create_user_via_api is deprecated using_api = 'api_version' in context create_user_via_api = new_authz.check_config_permission( 'create_user_via_api') diff --git a/doc/configuration.rst b/doc/configuration.rst index 853ffd23939..29ecbd310d8 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -357,7 +357,7 @@ Allow new user accounts to be created via the API. .. _ckan.auth.create_user_via_web: -ckan.auth.create_user_via_api +ckan.auth.create_user_via_web ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example:: From bf28f1a1a1a0b05fb17f4e8b3e5684f199120709 Mon Sep 17 00:00:00 2001 From: kindly Date: Thu, 5 Sep 2013 16:36:26 +0100 Subject: [PATCH 111/269] add stats and make homepage_style config option --- ckan/controllers/admin.py | 5 +++++ ckan/lib/app_globals.py | 1 + ckan/templates/admin/config.html | 2 ++ ckanext/homepage/plugin.py | 15 +++++++++++++++ .../theme/templates/admin/config.html | 19 ------------------- 5 files changed, 23 insertions(+), 19 deletions(-) delete mode 100644 ckanext/homepage/theme/templates/admin/config.html diff --git a/ckan/controllers/admin.py b/ckan/controllers/admin.py index cb5e84b069b..a8d1f0f257e 100644 --- a/ckan/controllers/admin.py +++ b/ckan/controllers/admin.py @@ -31,6 +31,10 @@ def _get_config_form_items(self): {'text': 'Green', 'value': '/base/css/green.css'}, {'text': 'Maroon', 'value': '/base/css/maroon.css'}, {'text': 'Fuchsia', 'value': '/base/css/fuchsia.css'}] + + homepages = [{'value': '1', 'text': 'Introductory area, search, featured group and featured organization'}, + {'value': '2', 'text': 'Search, featured dataset, stats and featured organization'}] + items = [ {'name': 'ckan.site_title', 'control': 'input', 'label': _('Site Title'), 'placeholder': ''}, {'name': 'ckan.main_css', 'control': 'select', 'options': styles, 'label': _('Style'), 'placeholder': ''}, @@ -39,6 +43,7 @@ def _get_config_form_items(self): {'name': 'ckan.site_about', 'control': 'markdown', 'label': _('About'), 'placeholder': _('About page text')}, {'name': 'ckan.site_intro_text', 'control': 'markdown', 'label': _('Intro Text'), 'placeholder': _('Text on home page')}, {'name': 'ckan.site_custom_css', 'control': 'textarea', 'label': _('Custom CSS'), 'placeholder': _('Customisable css inserted into the page header')}, + {'name': 'ckan.homepage_style', 'control': 'select', 'options': homepages, 'label': _('Homepage'), 'placeholder': ''}, ] return items diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index 0739274c671..074deb4e274 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -46,6 +46,7 @@ 'ckan.dumps_format': {}, 'ckan.api_url': {}, 'ofs.impl': {'name': 'ofs_impl'}, + 'ckan.homepage_style': {'default': '1'}, # split string 'search.facets': {'default': 'organization groups tags res_format license_id', diff --git a/ckan/templates/admin/config.html b/ckan/templates/admin/config.html index 64fd8558b28..5946ce183ef 100644 --- a/ckan/templates/admin/config.html +++ b/ckan/templates/admin/config.html @@ -41,6 +41,8 @@

    <head> tag of every page. If you wish to customize the templates more fully we recommend reading the documentation.

    +

    Homepage: This is for choosing a predefined layout for + the modules that appear on your homepage.

    {% endtrans %} {% endblock %}

    diff --git a/ckanext/homepage/plugin.py b/ckanext/homepage/plugin.py index e59541a16d1..b934f505302 100644 --- a/ckanext/homepage/plugin.py +++ b/ckanext/homepage/plugin.py @@ -1,6 +1,7 @@ import logging import ckan.plugins as p +import ckan.model as model log = logging.getLogger(__name__) @@ -72,8 +73,22 @@ def get_group(id): return groups_data + def get_site_statistics(self): + stats = {} + stats['dataset_count'] = p.toolkit.get_action('package_search')({}, {"rows": 1})['count'] + stats['group_count'] = len(p.toolkit.get_action('group_list')({}, {})) + stats['organization_count'] = len(p.toolkit.get_action('organization_list')({}, {})) + result =model.Session.execute( + '''select count(*) from related r + left join related_dataset rd on r.id = rd.related_id + where rd.status = 'active' or rd.id is null''').first()[0] + stats['related_count'] = len(p.toolkit.get_action('organization_list')({}, {})) + + return stats + def get_helpers(self): return { 'get_featured_organizations': self.get_featured_organizations, 'get_featured_groups': self.get_featured_groups, + 'get_site_statistics': self.get_site_statistics, } diff --git a/ckanext/homepage/theme/templates/admin/config.html b/ckanext/homepage/theme/templates/admin/config.html deleted file mode 100644 index 86e85b47714..00000000000 --- a/ckanext/homepage/theme/templates/admin/config.html +++ /dev/null @@ -1,19 +0,0 @@ -{% ckan_extends %} - -{% import 'macros/form.html' as form %} - -{% block admin_form %} - {{ super() }} - {{ form.select('homepage_layout', label=_('Homepage'), options=[ - {'value': 1, 'text': 'Introductory area, search, featured group and featured organization'}, - {'value': 2, 'text': 'Search, featured dataset, stats and featured organization'} - ], selected=1, error=false) }} -{% endblock %} - -{% block admin_form_help %} - {{ super() }} - {% trans %} -

    Homepage: This is for choosing a predefined layout for - the modules that appear on your homepage.

    - {% endtrans %} -{% endblock %} From 87d0b8b520e1f4860bec04b2a6e5edd67a40b33e Mon Sep 17 00:00:00 2001 From: joetsoi Date: Fri, 6 Sep 2013 14:20:23 +0100 Subject: [PATCH 112/269] [#1228] strip whitespace from title in model_dictize.package_dictize --- ckan/lib/dictization/model_dictize.py | 3 +++ ckan/tests/lib/test_dictization.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index 793f0733c95..c55908fad25 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -209,6 +209,9 @@ def package_dictize(pkg, context): if not result: raise logic.NotFound result_dict = d.table_dictize(result, context) + #strip whitespace from title + if result_dict.get('title'): + result_dict['title'] = result_dict['title'].strip() #resources res_rev = model.resource_revision_table resource_group = model.resource_group_table diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py index fb8bb49e628..a2ea317f18a 100644 --- a/ckan/tests/lib/test_dictization.py +++ b/ckan/tests/lib/test_dictization.py @@ -1222,3 +1222,21 @@ def test_25_user_dictize_as_anonymous(self): # Passwords should never be available assert 'password' not in user_dict + + def test_26_package_dictize_whitespace_strippped_from_title(self): + + context = {"model": model, + "session": model.Session} + + pkg = model.Session.query(model.Package).first() + original_title = pkg.title + pkg.title = " whitespace title \t" + model.Session.add(pkg) + model.Session.commit() + + result = package_dictize(pkg, context) + assert result['title'] == 'whitespace title' + pkg.title = original_title + model.Session.add(pkg) + model.Session.commit() + From 40397fe5c9bd0e2703f0d4e26227863c6cbc6b4a Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Mon, 9 Sep 2013 16:10:10 -0300 Subject: [PATCH 113/269] [#1178] Add help text to the add member forms --- ckan/public/base/less/forms.less | 30 ++++++++++++++ ckan/templates/group/member_new.html | 46 ++++++++++++++++----- ckan/templates/organization/member_new.html | 46 ++++++++++++++++----- 3 files changed, 102 insertions(+), 20 deletions(-) diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 289439c291a..a641129f705 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -670,3 +670,33 @@ textarea { .box-shadow(none); } } + +.add-member-form .control-label { + width: 100%; + text-align: left; +} + +.add-member-form .controls { + margin-left: auto; +} + +.add-member-or { + float: left; + line-height: 100px; + margin-left: 10px; + margin-right: 10px; + text-transform: uppercase; + color: grey; + font-weight: bold; +} + +.add-member-form .row-fluid .control-group { + float: left; + width: 45%; +} + +.add-member-form .row-fluid { + .select2-container, input { + width: 100% !important; + } +} diff --git a/ckan/templates/group/member_new.html b/ckan/templates/group/member_new.html index 2385a1957b9..efa3235461a 100644 --- a/ckan/templates/group/member_new.html +++ b/ckan/templates/group/member_new.html @@ -10,16 +10,42 @@

    {% block page_heading %}{{ _('Edit Member') if user else _('Add Member') }}{% endblock %}

    {% block form %} -
    - {% if user %} - - {% set format_attrs = {'disabled': true} %} - {{ form.input('username', label=_('User'), value=user.name, classes=['control-medium'], attrs=format_attrs) }} - {% else %} - {% set format_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/user/autocomplete?q=?'} %} - {{ form.input('username', id='field-username', label=_('User'), placeholder=_('Username'), value='', error='', classes=['control-medium'], attrs=format_attrs) }} - {% endif %} - {{ form.input('email', label=_('Email'), value=user.email, classes=['control-medium']) }} + +
    +
    + + + {{ _('If you wish to add an existing user, search for their username below.') }} + +
    + {% if user %} + + + {% else %} + + {% endif %} +
    +
    +
    + {{ _('or') }} +
    +
    + + + {{ _('If you wish to invite a new user, enter their email address.') }} + +
    + +
    +
    +
    {% set format_attrs = {'data-module': 'autocomplete'} %} {{ form.select('role', label=_('Role'), options=c.roles, selected=c.user_role, error='', attrs=format_attrs) }}
    diff --git a/ckan/templates/organization/member_new.html b/ckan/templates/organization/member_new.html index 74487100199..b76de7663fa 100644 --- a/ckan/templates/organization/member_new.html +++ b/ckan/templates/organization/member_new.html @@ -12,16 +12,42 @@

    {% block page_heading %}{{ _('Edit Member') if user else _('Add Member') }}{% endblock %}

    {% block form %} - - {% if user %} - - {% set format_attrs = {'disabled': true} %} - {{ form.input('username', label=_('User'), value=user.name, classes=['control-medium'], attrs=format_attrs) }} - {% else %} - {% set format_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/user/autocomplete?q=?'} %} - {{ form.input('username', id='field-username', label=_('User'), placeholder=_('Username'), value='', error='', classes=['control-medium'], attrs=format_attrs) }} - {% endif %} - {{ form.input('email', label=_('Email'), value=user.email, classes=['control-medium']) }} + +
    +
    + + + {{ _('If you wish to add an existing user, search for her username below.') }} + +
    + {% if user %} + + + {% else %} + + {% endif %} +
    +
    +
    + {{ _('or') }} +
    +
    + + + {{ _('If you wish to invite a new user, enter her email address.') }} + +
    + +
    +
    +
    {% set format_attrs = {'data-module': 'autocomplete'} %} {{ form.select('role', label=_('Role'), options=c.roles, selected=c.user_role, error='', attrs=format_attrs) }}
    From 2adda64767ea178e558ccd6844b84efe267a6dcf Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Mon, 9 Sep 2013 17:00:45 -0300 Subject: [PATCH 114/269] [#1178] Fix typo exeecute -> execute in migration --- ckan/migration/versions/071_add_state_column_to_user_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/migration/versions/071_add_state_column_to_user_table.py b/ckan/migration/versions/071_add_state_column_to_user_table.py index 367e3a48c32..791a046bad2 100644 --- a/ckan/migration/versions/071_add_state_column_to_user_table.py +++ b/ckan/migration/versions/071_add_state_column_to_user_table.py @@ -10,7 +10,7 @@ def upgrade(migrate_engine): def downgrade(migrate_engine): - migrate_engine.exeecute( + migrate_engine.execute( ''' ALTER TABLE "user" DROP COLUMN "state" ''' From 6327e2454ae5a98df9e71ecd27725aa0ea9f2e2d Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 9 Sep 2013 16:34:07 -0400 Subject: [PATCH 115/269] [#772] explaion plugin loading order --- doc/configuration.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index 91b73781058..19b7f338ca7 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -508,6 +508,10 @@ Specify which CKAN plugins are to be enabled. Format as a space-separated list of the plugin names. The plugin name is the key in the ``[ckan.plugins]`` section of the extension's ``setup.py``. For more information on plugins and extensions, see :doc:`extensions/index`. +The order that plugin names are specified influences the order that plugins are loaded. Plugins in separate python modules are loaded in exactly the order specified, but plugins in the same module will always be loaded in the order their classes appear in the module. + +Plugin load order is important for plugins that add templates directories: Templates found in template directories added earlier will override templates in template directories added later. + .. _ckan.datastore.enabled: ckan.datastore.enabled From 0c8bbd171f6b919de112f1f33045c0475aae6402 Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 10 Sep 2013 16:43:53 +0100 Subject: [PATCH 116/269] [#1126] Added stats and new layouts 2 and 3 for homepage --- ckan/controllers/admin.py | 3 +- ckan/controllers/home.py | 2 +- ckan/public/base/less/homepage.less | 51 +++++++++++++++---- .../theme/templates/home/layout2.html | 9 ++-- .../theme/templates/home/layout3.html | 23 +++++++++ .../theme/templates/home/snippets/stats.html | 33 ++++++++++++ 6 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 ckanext/homepage/theme/templates/home/layout3.html create mode 100644 ckanext/homepage/theme/templates/home/snippets/stats.html diff --git a/ckan/controllers/admin.py b/ckan/controllers/admin.py index a8d1f0f257e..aec37b5b2c2 100644 --- a/ckan/controllers/admin.py +++ b/ckan/controllers/admin.py @@ -33,7 +33,8 @@ def _get_config_form_items(self): {'text': 'Fuchsia', 'value': '/base/css/fuchsia.css'}] homepages = [{'value': '1', 'text': 'Introductory area, search, featured group and featured organization'}, - {'value': '2', 'text': 'Search, featured dataset, stats and featured organization'}] + {'value': '2', 'text': 'Search, stats, introductory area, featured organization and featured group'}, + {'value': '3', 'text': 'Search, introductory area and stats'}] items = [ {'name': 'ckan.site_title', 'control': 'input', 'label': _('Site Title'), 'placeholder': ''}, diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py index 65f53ca5859..182ced91248 100644 --- a/ckan/controllers/home.py +++ b/ckan/controllers/home.py @@ -177,7 +177,7 @@ def db_to_form_schema(group_type=None): # END OF DIRTYNESS - template = 'home/layout1.html' + template = 'home/layout{0}.html'.format(g.homepage_style) return base.render(template, cache_force=True) def license(self): diff --git a/ckan/public/base/less/homepage.less b/ckan/public/base/less/homepage.less index e53ad89ef6f..3315dfd7772 100644 --- a/ckan/public/base/less/homepage.less +++ b/ckan/public/base/less/homepage.less @@ -2,6 +2,7 @@ [role=main] { padding: 20px 0; + min-height: 0; } .row { @@ -10,7 +11,7 @@ .module-search { padding: 5px; - margin-bottom: 0; + margin: 0; color: @mastheadTextColor; background: @layoutTrimBackgroundColor; .search-giant { @@ -53,17 +54,49 @@ margin: 0; } - &.layout-1 { - .row1 .col2 { - position: absolute; + .box .inner { + padding: @gutterY @gutterX; + } + + .stats { + h3 { + margin: 0 0 10px 0; + } + ul { + .unstyled; + .clearfix; + li { + float: left; + width: 25%; + font-weight: 300; + a { + display: block; + b { + display: block; + font-size: @fontSizeLarge*2; + line-height: 1.5; + } + &:hover { + text-decoration: none; + } + } + } + } + } + + &.layout-1 .row1 .col2 { + position: absolute; + bottom: 0; + right: 0; + .module-search { bottom: 0; + left: 0; right: 0; - .module-search { - bottom: 0; - left: 0; - right: 0; - } } } + &.layout-2 .stats { + margin-top: @gutterY; + } + } diff --git a/ckanext/homepage/theme/templates/home/layout2.html b/ckanext/homepage/theme/templates/home/layout2.html index 99653aa0d86..8ccf432533a 100644 --- a/ckanext/homepage/theme/templates/home/layout2.html +++ b/ckanext/homepage/theme/templates/home/layout2.html @@ -8,9 +8,10 @@
    {% snippet 'home/snippets/search.html' %} + {% snippet 'home/snippets/stats.html' %}
    - {% snippet 'home/snippets/featured_dataset.html' %} + {% snippet 'home/snippets/promoted.html' %}
    @@ -18,12 +19,12 @@
    -
    - {% snippet 'home/snippets/stats.html' %} -
    {% snippet 'home/snippets/featured_organization.html' %}
    +
    + {% snippet 'home/snippets/featured_group.html' %} +
    diff --git a/ckanext/homepage/theme/templates/home/layout3.html b/ckanext/homepage/theme/templates/home/layout3.html new file mode 100644 index 00000000000..4e805845b21 --- /dev/null +++ b/ckanext/homepage/theme/templates/home/layout3.html @@ -0,0 +1,23 @@ +{% extends 'home/index.html' %} + +{% block homepage_layout_class %} layout-3{% endblock %} + +{% block primary_content %} +
    +
    + {% snippet 'home/snippets/search.html' %} +
    +
    +
    +
    +
    +
    + {% snippet 'home/snippets/promoted.html' %} +
    +
    + {% snippet 'home/snippets/stats.html' %} +
    +
    +
    +
    +{% endblock %} diff --git a/ckanext/homepage/theme/templates/home/snippets/stats.html b/ckanext/homepage/theme/templates/home/snippets/stats.html new file mode 100644 index 00000000000..9c5c7fa6970 --- /dev/null +++ b/ckanext/homepage/theme/templates/home/snippets/stats.html @@ -0,0 +1,33 @@ +{% set stats = h.get_site_statistics() %} + +
    +
    +

    {{ _('{0} statistics').format(g.site_title) }}

    + +
    +
    From e079385c05876834fad681261b0e68df6c9b9257 Mon Sep 17 00:00:00 2001 From: kindly Date: Tue, 10 Sep 2013 22:36:32 +0100 Subject: [PATCH 117/269] [1200] add defaults and make sure datapusher down does not cause errors --- ckan/model/meta.py | 2 +- ckan/plugins/interfaces.py | 1 - ckanext/datapusher/logic/action.py | 1 + ckanext/datapusher/plugin.py | 23 +++++++++++++++-------- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/ckan/model/meta.py b/ckan/model/meta.py index 795094965c5..0103019bb3a 100644 --- a/ckan/model/meta.py +++ b/ckan/model/meta.py @@ -163,4 +163,4 @@ def engine_is_pg(sa_engine=None): # Returns true iff the engine is connected to a postgresql database. # According to http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql # all Postgres driver names start with `postgresql` - return (sa_engine or engine).url.drivername.startswith('postgresql') + return (sa_engine or engine).url.drivername.startswith('postgres') diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index aa08fc64bab..d1372d2e895 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -443,7 +443,6 @@ def before_view(self, pkg_dict): class IResourceController(Interface): """ Hook into the resource controller. - (see IGroupController) """ def before_show(self, resource_dict): diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 41d3e063284..3201d35572a 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -50,6 +50,7 @@ def datapusher_submit(context, data_dict): ver=3, qualified=True) user = p.toolkit.get_action('user_show')(context, {'id': context['user']}) + try: r = requests.post( urlparse.urljoin(datapusher_url, 'job'), diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index b20f464d789..04a27db435b 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -10,7 +10,7 @@ log = logging.getLogger(__name__) _get_or_bust = logic.get_or_bust -DEFAULT_FORMATS = [] +DEFAULT_FORMATS = ['csv', 'xls', 'application/csv', 'application/vnd.ms-excel'] class DatastoreException(Exception): @@ -31,7 +31,7 @@ class DatapusherPlugin(p.SingletonPlugin): def configure(self, config): self.config = config - datapusher_formats = config.get('ckan.datapusher.formats', '').split() + datapusher_formats = config.get('ckan.datapusher.formats', '').lower().split() self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS datapusher_url = config.get('ckan.datapusher.url') @@ -46,16 +46,23 @@ def notify(self, entity, operation=None): # if operation is None, resource URL has been changed, as # the notify function in IResourceUrlChange only takes # 1 parameter - context = {'model': model, 'ignore_auth': True} + context = {'model': model, 'ignore_auth': True, 'defer_commit': True} package = p.toolkit.get_action('package_show')(context, { 'id': entity.get_package_id() }) - if (not package['private'] and - entity.format in self.datapusher_formats and + if (not package['private'] and entity.format and + entity.format.lower() in self.datapusher_formats and entity.url_type != 'datapusher'): - p.toolkit.get_action('datapusher_submit')(context, { - 'resource_id': entity.id - }) + try: + p.toolkit.get_action('datapusher_submit')(context, { + 'resource_id': entity.id + }) + except p.toolkit.ValidationError, e: + # If datapusher is offline want to catch error instead of + # raising otherwise resource save will fail with 500 + log.critical(e) + pass + def get_actions(self): return {'datapusher_submit': action.datapusher_submit, From 569987d68ef1d4c63d71e48eb12a798cac57a59f Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 10 Sep 2013 23:02:07 -0700 Subject: [PATCH 118/269] [#1235] Add image block to be able to replace only the image easily --- ckan/templates/snippets/organization.html | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ckan/templates/snippets/organization.html b/ckan/templates/snippets/organization.html index 116cb6e3ac4..ef08a9ed440 100644 --- a/ckan/templates/snippets/organization.html +++ b/ckan/templates/snippets/organization.html @@ -20,11 +20,13 @@

    {{ _('Organization') }}

    {% endif %}
    -
    - - {{ organization.name }} - -
    + {% block image %} +
    + + {{ organization.name }} + +
    + {% endblock %}

    {{ organization.title or organization.name }}

    {% if organization.description %}

    From 50acd34cb95d73adb7213d2c6be54fe4ff71e2a0 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 10 Sep 2013 23:07:08 -0700 Subject: [PATCH 119/269] [#1235] Add block for content of container to allow people to add something to the left of the account block --- ckan/templates/header.html | 100 +++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/ckan/templates/header.html b/ckan/templates/header.html index 5b4ebe225a7..da960e89db7 100644 --- a/ckan/templates/header.html +++ b/ckan/templates/header.html @@ -2,58 +2,60 @@ {% block header_account %}

    {% endblock %} From b0f3e643b2640c2a717d56f802a5d61477fcf7f3 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 11 Sep 2013 12:32:26 +0100 Subject: [PATCH 120/269] [#1237] Improve facet.limit search option A negative value should be allowed if you don't want to limit the facet values, but the search params schema didn't allow it. Also mention in the docs that this can be set via configuration and that the default is 50. Added some tests. --- ckan/logic/action/get.py | 5 ++-- ckan/logic/schema.py | 2 +- ckan/tests/logic/test_action.py | 42 +++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index ab50329c608..a810cfa3cbe 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1289,8 +1289,9 @@ def package_search(context, data_dict): :param facet.mincount: the minimum counts for facet fields should be included in the results. :type facet.mincount: int - :param facet.limit: the maximum number of constraint counts that should be - returned for the facet fields. A negative value means unlimited + :param facet.limit: the maximum number of values the facet fields. A + negative value means unlimited. This can be set instance-wide with + the :ref:`search.facets.limit` config option. Default is 50. :type facet.limit: int :param facet.field: the fields to facet upon. Default empty. If empty, then the returned facet information is empty. diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f8fe8df7fbd..75d49808234 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -539,7 +539,7 @@ def default_package_search_schema(): 'qf': [ignore_missing, unicode], 'facet': [ignore_missing, unicode], 'facet.mincount': [ignore_missing, natural_number_validator], - 'facet.limit': [ignore_missing, natural_number_validator], + 'facet.limit': [ignore_missing, int_validator], 'facet.field': [ignore_missing, list_of_strings], 'extras': [ignore_missing] # Not used by Solr, but useful for extensions } diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 89e030beac8..8b4fe3dca14 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1377,6 +1377,48 @@ def test_1_basic(self): assert_equal(result['count'], 1) assert_equal(result['results'][0]['name'], 'annakarenina') + def test_1_facet_limit(self): + params = { + 'q':'*:*', + 'facet.field': ['groups', 'tags', 'res_format', 'license'], + 'rows': 20, + 'start': 0, + } + postparams = '%s=1' % json.dumps(params) + res = self.app.post('/api/action/package_search', params=postparams) + res = json.loads(res.body) + assert_equal(res['success'], True) + + assert_equal(len(res['result']['search_facets']['groups']['items']), 2) + + params = { + 'q':'*:*', + 'facet.field': ['groups', 'tags', 'res_format', 'license'], + 'facet.limit': 1, + 'rows': 20, + 'start': 0, + } + postparams = '%s=1' % json.dumps(params) + res = self.app.post('/api/action/package_search', params=postparams) + res = json.loads(res.body) + assert_equal(res['success'], True) + + assert_equal(len(res['result']['search_facets']['groups']['items']), 1) + + params = { + 'q':'*:*', + 'facet.field': ['groups', 'tags', 'res_format', 'license'], + 'facet.limit': -1, # No limit + 'rows': 20, + 'start': 0, + } + postparams = '%s=1' % json.dumps(params) + res = self.app.post('/api/action/package_search', params=postparams) + res = json.loads(res.body) + assert_equal(res['success'], True) + + assert_equal(len(res['result']['search_facets']['groups']['items']), 2) + def test_1_basic_no_params(self): postparams = '%s=1' % json.dumps({}) res = self.app.post('/api/action/package_search', params=postparams) From 2d76140db869595b83d2d501380bbe9f248192f5 Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 11 Sep 2013 16:58:41 +0100 Subject: [PATCH 121/269] [#1178] Small CSS tweaks and fix for premature closing div --- ckan/public/base/less/forms.less | 8 ++++---- ckan/templates/organization/member_new.html | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index a641129f705..1263abdd219 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -682,11 +682,11 @@ textarea { .add-member-or { float: left; - line-height: 100px; - margin-left: 10px; - margin-right: 10px; + margin-top: 75px; + width: 7%; + text-align: center; text-transform: uppercase; - color: grey; + color: @grayLight; font-weight: bold; } diff --git a/ckan/templates/organization/member_new.html b/ckan/templates/organization/member_new.html index b76de7663fa..352cb9f3790 100644 --- a/ckan/templates/organization/member_new.html +++ b/ckan/templates/organization/member_new.html @@ -65,7 +65,6 @@

    {% endblock %} -

    {% endblock %} {% block secondary_content %} From 55ed1ef2b6d4eca22c324ddf627eb7fe1e0a4d24 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 12 Sep 2013 18:39:42 +0100 Subject: [PATCH 122/269] [#1241] Raise CkanVersionException properly --- ckan/plugins/toolkit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index 4e4ac5da2d0..34e9d671cf1 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -348,7 +348,7 @@ def _requires_ckan_version(cls, min_version, max_version=None): else: error = 'Requires ckan version between %s and %s' % \ (min_version, max_version) - raise cls.CkanVersionException(error) + raise CkanVersionException(error) def __getattr__(self, name): ''' return the function/object requested ''' From fea743c9289371ecbbb9c2d013d8f176ed8f4b27 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Thu, 12 Sep 2013 16:21:03 -0300 Subject: [PATCH 123/269] Fix bug in (group|organization)_member_create According to the documentation, the "username" parameter accepts either name or id, but the code only accepted name. I've fixed it to accept both. --- ckan/logic/action/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index fcb55ceae64..baac5a2154c 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -1141,7 +1141,7 @@ def _group_or_org_member_create(context, data_dict, is_org=False): role = data_dict.get('role') group_id = data_dict.get('id') group = model.Group.get(group_id) - result = session.query(model.User).filter_by(name=username).first() + result = model.User.get(username) if result: user_id = result.id else: From f03e409b6e8eed6633cf530ff8cf08b882d1b726 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Thu, 12 Sep 2013 17:44:30 -0300 Subject: [PATCH 124/269] [#848] Fix configuration name search.facet.limits -> search.facet.limit --- doc/configuration.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 91b73781058..beaf9a69956 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -450,14 +450,14 @@ Default value: ``false`` Controls whether the default search page (``/dataset``) should show only standard datasets or also custom dataset types. -.. _search.facets.limits: +.. _search.facets.limit: -search.facets.limits -^^^^^^^^^^^^^^^^^^^^ +search.facets.limit +^^^^^^^^^^^^^^^^^^^ Example:: - search.facets.limits = 100 + search.facets.limit = 100 Default value: ``50`` From 43afc32641e9d09293ebb5276f1098598770ff3a Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Thu, 12 Sep 2013 17:46:46 -0300 Subject: [PATCH 125/269] [#848] extra_template_paths and extra_public_paths aren't deprecated --- doc/configuration.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index beaf9a69956..a0af8ec5769 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -914,8 +914,6 @@ To customise the display of CKAN you can supply replacements for the Genshi temp For more information on theming, see :doc:`theming`. -.. note:: This is only for legacy code, and shouldn't be used anymore. - .. _extra_public_paths: extra_public_paths @@ -929,8 +927,6 @@ To customise the display of CKAN you can supply replacements for static files su For more information on theming, see :doc:`theming`. -.. note:: This is only for legacy code, and shouldn't be used anymore. - .. end_config-theming Storage Settings From fca56dceb35bae9889b2b017af0a8b38afddec15 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Thu, 12 Sep 2013 17:48:57 -0300 Subject: [PATCH 126/269] [#848] ckan.template_footer_end is deprecated --- doc/configuration.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index a0af8ec5769..7c2aef44f3d 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -888,6 +888,8 @@ Example (showing insertion of Google Analytics code):: +.. note:: This is only for legacy code, and shouldn't be used anymore. + .. _ckan.template_title_deliminater: ckan.template_title_deliminater From 2994a3c3232a8134746337ff528ff613c1844565 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Thu, 12 Sep 2013 18:35:26 -0300 Subject: [PATCH 127/269] [#848] Reorganize and add examples of FileStore's configuration parameters --- doc/configuration.rst | 54 +++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 7c2aef44f3d..55301872056 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -945,20 +945,7 @@ Example:: Default value: ``None`` -This setting will change the bucket name for the uploaded files. - -.. _ckan.storage.key_prefix: - -ckan.storage.key_prefix -^^^^^^^^^^^^^^^^^^^^^^^ - -Example:: - - ckan.storage.key_prefix = ckan-file/ - -Default value: ``file/`` - -This setting will change the prefix for the uploaded files. This is only for ``pairtree``. +This changes the bucket name for the uploaded files. .. _ckan.storage.max_content_length: @@ -999,6 +986,19 @@ Default value: ``None`` Only used with the local storage backend. Use this to specify where uploaded files should be stored, and also to turn on the handling of file storage. The folder should exist, and will automatically be turned into a valid pairtree repository if it is not already. +.. _ckan.storage.key_prefix: + +ckan.storage.key_prefix +^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.storage.key_prefix = ckan-file/ + +Default value: ``file/`` + +Only used with the local storage backend. This changes the prefix for the uploaded files. + .. _ofs.aws_access_key_id: ofs.aws_access_key_id @@ -1006,13 +1006,11 @@ ofs.aws_access_key_id Example:: - ofs.aws_access_key_id = your_key_id_here + ofs.aws_access_key_id = 022QF06E7MXBSH9DHM02 Default value: ``None`` -Only used with the Amazon S3 storage backend. - -.. todo:: Expand +Only used with the Amazon S3 storage backend. Configure with your AWS Access Key ID. .. _ofs.aws_secret_access_key: @@ -1021,13 +1019,11 @@ ofs.aws_secret_access_key Example:: - ofs.aws_secret_access_key = your_secret_access_key_here + ofs.aws_secret_access_key = kWcrlUX5JEDGM/LtmEENI/aVmYvHNif5zB+d9+ct Default value: ``None`` -Only used with the Amazon S3 storage backend. - -.. todo:: Expand +Only used with the Amazon S3 storage backend. Configure with your AWS Secret Access Key. .. _ofs.gs_access_key_id: @@ -1036,13 +1032,12 @@ ofs.gs_access_key_id Example:: - ofs.gs_access_key_id = your_key_id_here + ofs.gs_access_key_id = GOOGTS7C7FUP3AIRVJTE Default value: ``None`` -Only used with the Google storage backend. - -.. todo:: Expand +Only used with the Google storage backend. Configure with your Google Storage +Access Key ID. .. _ofs.gs_secret_access_key: @@ -1051,13 +1046,12 @@ ofs.gs_secret_access_key Example:: - ofs.gs_secret_access_key = your_secret_access_key_here + ofs.gs_secret_access_key = bGoa+V7g/yqDXvKRqq+JTFn4uQZbPiQJo4pf9RzJ Default value: ``None`` -Only used with the Google storage backend. - -.. todo:: Expand +Only used with the Google storage backend. Configure with your Google Storage +Secret Access Key. DataPusher Settings From 5813ae1f4b3dd0acde1e78a21b13cc533388e8e3 Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 18 Sep 2013 14:27:42 +0100 Subject: [PATCH 128/269] [#1092] Updates helper text for orgs, groups and datasets --- ckan/templates/group/snippets/helper.html | 15 ++++++++------- ckan/templates/organization/snippets/helper.html | 16 +++++++--------- ckan/templates/package/base_form_page.html | 6 +++--- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/ckan/templates/group/snippets/helper.html b/ckan/templates/group/snippets/helper.html index a73c9d6dde7..38585ab7eae 100644 --- a/ckan/templates/group/snippets/helper.html +++ b/ckan/templates/group/snippets/helper.html @@ -4,12 +4,13 @@

    {{ _('What are Groups?') }}

    - {% trans %} -

    Groups allow you to group together datasets under a community (for - example, Civil Liberty data) or topic (e.g. Transport, Health, - Environment) to make it easier for users to browse datasets by theme. - Datasets can be part of a group, but do not belong to the group for - editing or authorisation purposes.

    - {% endtrans %} +

    + {% trans %} + You can use CKAN Groups to create and manage collections of datasets. + This could be to catalogue datasets for a particular project or team, + or on a particular theme, or as a very simple way to help people find + and search your own published datasets. + {% endtrans %} +

\ No newline at end of file diff --git a/ckan/templates/organization/snippets/helper.html b/ckan/templates/organization/snippets/helper.html index f0acbd46d73..6ec71c0d67e 100644 --- a/ckan/templates/organization/snippets/helper.html +++ b/ckan/templates/organization/snippets/helper.html @@ -4,14 +4,12 @@

{{ _('What are Organizations?') }}

- {% trans %} -

Organizations act like publishing departments for datasets (for - example, the Department of Health). This means that datasets can be - published by and belong to a department instead of an individual - user.

-

Within organizations, admins can assign roles and authorisation its - members, giving individual users the right to publish datasets from - that particular organisation (e.g. Office of National Statistics).

- {% endtrans %} +

+ {% trans %} + CKAN Organizations are used to create, manage and publish collections + of datasets. Users can have different roles within an Organization, + depending on their level of authorisation to create, edit and publish. + {% endtrans %} +

\ No newline at end of file diff --git a/ckan/templates/package/base_form_page.html b/ckan/templates/package/base_form_page.html index 049d83b1ebb..a738a8716c8 100644 --- a/ckan/templates/package/base_form_page.html +++ b/ckan/templates/package/base_form_page.html @@ -15,9 +15,9 @@

{{ _('What are dataset

{% trans %} - Datasets are simply used to group related pieces of data. These - can then be found under a single url with a description and - licensing information. + A CKAN Dataset is a collection of data resources (such as files), + together with a description and other information, at a fixed URL. + Datasets are what users see when searching for data. {% endtrans %}

From d1e8b9d40e864520ad26425eada1a7af0595390b Mon Sep 17 00:00:00 2001 From: John Martin Date: Thu, 19 Sep 2013 14:50:14 +0100 Subject: [PATCH 129/269] [#1187] Adds source URL and version to package form --- .../package/snippets/package_metadata_fields.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ckan/templates/package/snippets/package_metadata_fields.html b/ckan/templates/package/snippets/package_metadata_fields.html index fce09b1d8bc..4f2da316cba 100644 --- a/ckan/templates/package/snippets/package_metadata_fields.html +++ b/ckan/templates/package/snippets/package_metadata_fields.html @@ -2,6 +2,14 @@ {% block package_metadata_fields %} +{% block package_metadata_fields_url %} + {{ form.input('url', label=_('Source'), id='field-url', placeholder=_('http://example.com/dataset.json'), value=data.url, error=errors.url, classes=['control-medium']) }} +{% endblock %} + +{% block package_metadata_fields_version %} + {{ form.input('version', label=_('Version'), id='field-version', placeholder=_('1.0'), value=data.version, error=errors.version, classes=['control-medium']) }} +{% endblock %} + {% block package_metadata_author %} {{ form.input('author', label=_('Author'), id='field-author', placeholder=_('Joe Bloggs'), value=data.author, error=errors.author, classes=['control-medium']) }} From b8f449e83555ec850a1f0c4fc4bcbb7ded7b3d04 Mon Sep 17 00:00:00 2001 From: kindly Date: Fri, 20 Sep 2013 05:14:46 +0100 Subject: [PATCH 130/269] [#1200] make resource data page for reuploading and make error messages better --- ckan/config/routing.py | 2 +- ckan/logic/schema.py | 1 + ckan/templates/package/new_resource.html | 12 +- ckan/templates/package/resource_data.html | 45 ++++++++ ckan/templates/package/resource_read.html | 3 - ckanext/datapusher/logic/action.py | 127 +++++++++++++--------- ckanext/datapusher/plugin.py | 56 ++++++++++ ckanext/datastore/controller.py | 1 + 8 files changed, 189 insertions(+), 58 deletions(-) create mode 100644 ckan/templates/package/resource_data.html diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 0fbb02bb886..6b41d8a66f2 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -246,7 +246,7 @@ def make_map(): action='resource_read') m.connect('/dataset/{id}/resource_delete/{resource_id}', action='resource_delete') - m.connect('/dataset/{id}/resource_edit/{resource_id}', + m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}', action='resource_edit') m.connect('/dataset/{id}/resource/{resource_id}/download', action='resource_download') diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f8fe8df7fbd..8058e10aea0 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -85,6 +85,7 @@ def default_resource_schema(): 'cache_last_updated': [ignore_missing, isodate], 'webstore_last_updated': [ignore_missing, isodate], 'tracking_summary': [ignore_missing], + 'datastore_active': [ignore], '__extras': [ignore_missing, extras_unicode_convert, keep_extras], } diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html index 40d8ed97c9e..7eedbad761f 100644 --- a/ckan/templates/package/new_resource.html +++ b/ckan/templates/package/new_resource.html @@ -29,8 +29,16 @@

{{ _('What\'s a resour {% endblock %} {% block primary_content %} - {% if pkg_dict and pkg_dict.state != 'draft' %} -
+ {% set res = c.resource %} + {% if pkg_dict and pkg_dict.state != 'draft' and res %} +
+
    + {{ h.build_nav_icon('resource_edit', _('Edit Resource'), id=pkg_dict.id, resource_id=res.id) }} + {% if 'datapusher' in g.plugins %} + {{ h.build_nav_icon('resource_data', _('Resource Data'), id=pkg_dict.id, resource_id=res.id) }} + {% endif %} +
+
{% endif %} {{ super() }} {% endblock %} diff --git a/ckan/templates/package/resource_data.html b/ckan/templates/package/resource_data.html new file mode 100644 index 00000000000..30c9fe59e98 --- /dev/null +++ b/ckan/templates/package/resource_data.html @@ -0,0 +1,45 @@ +{% extends "package/new_resource.html" %} + +{% set pkg_dict = c.pkg_dict %} +{% set res = c.resource %} + +{% block subtitle %}{{ h.dataset_display_name(pkg_dict) }} - {{ h.resource_display_name(res) }}{% endblock %} + +{% block breadcrumb_content_selected %}{% endblock %} + +{% block breadcrumb_content %} + {{ super() }} +
  • {{ _('Edit') }}
    • +{% endblock %} + +{% block content_action %} + {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %} +{% endblock %} + +{# logged_in is defined in new_resource.html #} +{% block form %} + + {% set action = h.url_for(controller='ckanext.datapusher.plugin:ResourceDataController', action='resource_data', id=pkg_dict.id, resource_id=res.id) %} + +
    +

    {{ _('Last Upload Status:') }} {% if status.status %} {{ status.status.capitalize() }} {% else %} {{ _('Not Uploaded Yet') }} {% endif %}

    + {% if status.error and status.error.message %} +

    {{ _('Upload Error:') }} {{ status.error.message }}

    + {% endif %} + {% if status.status %} +

    {{ _('Last Updated: ') }} {{ h.time_ago_from_timestamp(status.last_updated) }}

    + {% endif %} + + + {% if status.status and status.task_info %} +

    {{ _('Upload Log ') }}

    + {{ h.debug_inspect(status.task_info.logs) }} + {% endif %} + {{ status }} +
    + +{% endblock %} + +{% block secondary_content %} + {% snippet 'package/snippets/info.html', pkg=pkg_dict, active=pkg_dict.id, action='resource_edit' %} +{% endblock %} diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index f914780d179..46406746a6f 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -27,9 +27,6 @@
      {% block resource_actions_inner %} {% if h.check_access('package_update', {'id':pkg.id }) %} - {% if 'datapusher' in g.plugins %} -
    • {% snippet 'snippets/datapusher_status.html', resource=res %}
      • - {% endif %}
    • {% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}
      • {% endif %} {% if res.url %} diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 3201d35572a..653dd44e0b5 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -51,6 +51,29 @@ def datapusher_submit(context, data_dict): user = p.toolkit.get_action('user_show')(context, {'id': context['user']}) + task = { + 'entity_id': res_id, + 'entity_type': 'resource', + 'task_type': 'datapusher', + 'last_updated': str(datetime.datetime.now()), + 'state': 'submitting', + 'key': 'datapusher', + 'value': '{}', + 'error': '{}', + } + try: + task_id = p.toolkit.get_action('task_status_show')(context, { + 'entity_id': res_id, + 'task_type': 'datapusher', + 'key': 'datapusher' + })['id'] + task['id'] = task_id + except logic.NotFound: + pass + + result = p.toolkit.get_action('task_status_update')(context, task) + task_id = result['id'] + try: r = requests.post( urlparse.urljoin(datapusher_url, 'job'), @@ -69,36 +92,37 @@ def datapusher_submit(context, data_dict): })) r.raise_for_status() except requests.exceptions.ConnectionError, e: - raise p.toolkit.ValidationError({'datapusher': { - 'message': 'Could not connect to DataPusher.', - 'details': str(e)}}) + error = {'message': 'Could not connect to DataPusher.', + 'details': str(e)} + task['error'] = json.dumps(error) + task['state'] = 'error' + task['last_updated'] = str(datetime.datetime.now()), + p.toolkit.get_action('task_status_update')(context, task) + raise p.toolkit.ValidationError(error) + except requests.exceptions.HTTPError, e: m = 'An Error occurred while sending the job: {0}'.format(e.message) try: body = e.response.json() except ValueError: body = e.response.text - raise p.toolkit.ValidationError({'datapusher': { - 'message': m, - 'details': body, - 'status_code': r.status_code}}) - - empty_task = { - 'entity_id': res_id, - 'entity_type': 'resource', - 'task_type': 'datapusher', - 'last_updated': str(datetime.datetime.now()), - 'state': 'pending' - } - - tasks = [] - for (k, v) in [('job_id', r.json()['job_id']), - ('job_key', r.json()['job_key'])]: - t = empty_task.copy() - t['key'] = k - t['value'] = v - tasks.append(t) - p.toolkit.get_action('task_status_update_many')(context, {'data': tasks}) + error = {'message': m, + 'details': body, + 'status_code': r.status_code} + task['error'] = json.dumps(error) + task['state'] = 'error' + task['last_updated'] = str(datetime.datetime.now()), + p.toolkit.get_action('task_status_update')(context, task) + raise p.toolkit.ValidationError(error) + + value = json.dumps( + {'job_id': r.json()['job_id'], + 'job_key': r.json()['job_key']} + ) + task['value'] = value + task['state'] = 'pending' + task['last_updated'] = str(datetime.datetime.now()), + p.toolkit.get_action('task_status_update')(context, task) return True @@ -111,30 +135,20 @@ def datapusher_hook(context, data_dict): ''' # TODO: use a schema to validate - p.toolkit.check_access('datapusher_submit', context, data_dict) res_id = data_dict['metadata']['resource_id'] - task_id = p.toolkit.get_action('task_status_show')(context, { + task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': res_id, 'task_type': 'datapusher', - 'key': 'job_id' + 'key': 'datapusher' }) - task_key = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'job_key' - }) - - tasks = [task_id, task_key] - - for task in tasks: - task['state'] = data_dict['status'] - task['last_updated'] = str(datetime.datetime.now()) + task['state'] = data_dict['status'] + task['last_updated'] = str(datetime.datetime.now()) - p.toolkit.get_action('task_status_update_many')(context, {'data': tasks}) + p.toolkit.get_action('task_status_update')(context, task) def datapusher_status(context, data_dict): @@ -151,16 +165,10 @@ def datapusher_status(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - task_id = p.toolkit.get_action('task_status_show')(context, { + task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': res_id, 'task_type': 'datapusher', - 'key': 'job_id' - }) - - task_key = p.toolkit.get_action('task_status_show')(context, { - 'entity_id': res_id, - 'task_type': 'datapusher', - 'key': 'job_key' + 'key': 'datapusher' }) datapusher_url = pylons.config.get('ckan.datapusher.url') @@ -168,11 +176,26 @@ def datapusher_status(context, data_dict): raise p.toolkit.ValidationError( {'configuration': ['DataPusher not configured.']}) - url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value']) + value = json.loads(task['value']) + job_key = value.get('job_key') + job_id = value.get('job_id') + url = None + job_detail = None + if job_id: + url = urlparse.urljoin(datapusher_url, 'job' + '/' + job_id) + try: + r = requests.get(url, headers={'Content-Type': 'application/json', + 'Authorization': job_key}) + job_detail = r.json() + except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: + job_detail = {'error': 'cannot connect to datapusher'} + return { - 'status': task_id['state'], - 'job_id': task_id['value'], + 'status': task['state'], + 'job_id': job_id, 'job_url': url, - 'last_updated': task_id['last_updated'], - 'job_key': task_key['value'] + 'last_updated': task['last_updated'], + 'job_key': job_key, + 'task_info': job_detail, + 'error': json.loads(task['error']) } diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index 04a27db435b..a545323a7dd 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -1,11 +1,14 @@ import logging import ckan.plugins as p +import ckan.lib.base as base +import ckan.lib.helpers as core_helpers import ckanext.datapusher.logic.action as action import ckanext.datapusher.logic.auth as auth import ckanext.datapusher.helpers as helpers import ckan.logic as logic import ckan.model as model +from ckan.common import c, _, request log = logging.getLogger(__name__) _get_or_bust = logic.get_or_bust @@ -16,6 +19,48 @@ class DatastoreException(Exception): pass +class ResourceDataController(base.BaseController): + + def resource_data(self, id, resource_id): + + if request.method == 'POST': + result = request.POST + try: + c.pkg_dict = p.toolkit.get_action('datapusher_submit')( + None, {'resource_id': resource_id} + ) + except logic.ValidationError: + pass + + base.redirect(core_helpers.url_for( + controller='ckanext.datapusher.plugin:ResourceDataController', + action='resource_data', + id=id, + resource_id=resource_id) + ) + + try: + c.pkg_dict = p.toolkit.get_action('package_show')( + None, {'id': id} + ) + c.resource = p.toolkit.get_action('resource_show')( + None, {'id': resource_id} + ) + except logic.NotFound: + base.abort(404, _('Resource not found')) + except logic.NotAuthorized: + base.abort(401, _('Unauthorized to edit this resource')) + + try: + datapusher_status = p.toolkit.get_action('datapusher_status')( + None, {'resource_id': resource_id} + ) + except logic.NotFound: + datapusher_status = {} + + return base.render('package/resource_data.html', + extra_vars={'status': datapusher_status}) + class DatapusherPlugin(p.SingletonPlugin): p.implements(p.IConfigurable, inherit=True) @@ -24,6 +69,7 @@ class DatapusherPlugin(p.SingletonPlugin): p.implements(p.IResourceUrlChange) p.implements(p.IDomainObjectModification, inherit=True) p.implements(p.ITemplateHelpers) + p.implements(p.IRoutes, inherit=True) legacy_mode = False resource_show_action = None @@ -39,6 +85,11 @@ def configure(self, config): raise Exception( 'Config option `ckan.datapusher.url` has to be set.') + self.datapusher_secret_key = config.get('ckan.datapusher.secret_key', '') + if not datapusher_url: + raise Exception( + 'Config option `ckan.datapusher.secret_key` has to be set.') + def notify(self, entity, operation=None): if isinstance(entity, model.Resource): if (operation == model.domain_object.DomainObjectOperation.new @@ -63,6 +114,11 @@ def notify(self, entity, operation=None): log.critical(e) pass + def before_map(self, m): + m.connect('resource_data', '/dataset/{id}/resource_data/{resource_id}', + controller='ckanext.datapusher.plugin:ResourceDataController', + action='resource_data') + return m def get_actions(self): return {'datapusher_submit': action.datapusher_submit, diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index 3bb5def4e73..f4a14a352c6 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -42,3 +42,4 @@ def dump(self, resource_id): for record in result['records']: wr.writerow([record[column] for column in header]) return f.getvalue() + From dcf8761053e91f635b1febd6e25e97722272cd38 Mon Sep 17 00:00:00 2001 From: kindly Date: Fri, 20 Sep 2013 18:25:05 +0100 Subject: [PATCH 131/269] [1200] pep8 and test failure fixups --- ckan/logic/action/get.py | 4 ++-- ckanext/datapusher/logic/action.py | 10 +++++----- ckanext/datapusher/plugin.py | 30 ++++++++++++++++-------------- ckanext/datapusher/tests/test.py | 20 +++++--------------- ckanext/datastore/controller.py | 1 - 5 files changed, 28 insertions(+), 37 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 7c9cdfddace..d02f6e1578b 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1834,11 +1834,11 @@ def task_status_show(context, data_dict): context['task_status'] = task_status + _check_access('task_status_show', context, data_dict) + if task_status is None: raise NotFound - _check_access('task_status_show', context, data_dict) - task_status_dict = model_dictize.task_status_dictize(task_status, context) return task_status_dict diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 653dd44e0b5..66c56d22c82 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -115,10 +115,9 @@ def datapusher_submit(context, data_dict): p.toolkit.get_action('task_status_update')(context, task) raise p.toolkit.ValidationError(error) - value = json.dumps( - {'job_id': r.json()['job_id'], - 'job_key': r.json()['job_key']} - ) + value = json.dumps({'job_id': r.json()['job_id'], + 'job_key': r.json()['job_key']}) + task['value'] = value task['state'] = 'pending' task['last_updated'] = str(datetime.datetime.now()), @@ -187,7 +186,8 @@ def datapusher_status(context, data_dict): r = requests.get(url, headers={'Content-Type': 'application/json', 'Authorization': job_key}) job_detail = r.json() - except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e: + except (requests.exceptions.ConnectionError, + requests.exceptions.HTTPError), e: job_detail = {'error': 'cannot connect to datapusher'} return { diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index a545323a7dd..2138497ad57 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -19,6 +19,7 @@ class DatastoreException(Exception): pass + class ResourceDataController(base.BaseController): def resource_data(self, id, resource_id): @@ -33,10 +34,10 @@ def resource_data(self, id, resource_id): pass base.redirect(core_helpers.url_for( - controller='ckanext.datapusher.plugin:ResourceDataController', - action='resource_data', - id=id, - resource_id=resource_id) + controller='ckanext.datapusher.plugin:ResourceDataController', + action='resource_data', + id=id, + resource_id=resource_id) ) try: @@ -59,7 +60,7 @@ def resource_data(self, id, resource_id): datapusher_status = {} return base.render('package/resource_data.html', - extra_vars={'status': datapusher_status}) + extra_vars={'status': datapusher_status}) class DatapusherPlugin(p.SingletonPlugin): @@ -77,15 +78,14 @@ class DatapusherPlugin(p.SingletonPlugin): def configure(self, config): self.config = config - datapusher_formats = config.get('ckan.datapusher.formats', '').lower().split() - self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS + datapusher_formats = config.get('ckan.datapusher.formats', '').lower() + self.datapusher_formats = datapusher_formats.split() or DEFAULT_FORMATS datapusher_url = config.get('ckan.datapusher.url') if not datapusher_url: raise Exception( 'Config option `ckan.datapusher.url` has to be set.') - self.datapusher_secret_key = config.get('ckan.datapusher.secret_key', '') if not datapusher_url: raise Exception( 'Config option `ckan.datapusher.secret_key` has to be set.') @@ -97,7 +97,8 @@ def notify(self, entity, operation=None): # if operation is None, resource URL has been changed, as # the notify function in IResourceUrlChange only takes # 1 parameter - context = {'model': model, 'ignore_auth': True, 'defer_commit': True} + context = {'model': model, 'ignore_auth': True, + 'defer_commit': True} package = p.toolkit.get_action('package_show')(context, { 'id': entity.get_package_id() }) @@ -109,15 +110,16 @@ def notify(self, entity, operation=None): 'resource_id': entity.id }) except p.toolkit.ValidationError, e: - # If datapusher is offline want to catch error instead of - # raising otherwise resource save will fail with 500 + # If datapusher is offline want to catch error instead + # of raising otherwise resource save will fail with 500 log.critical(e) pass def before_map(self, m): - m.connect('resource_data', '/dataset/{id}/resource_data/{resource_id}', - controller='ckanext.datapusher.plugin:ResourceDataController', - action='resource_data') + m.connect( + 'resource_data', '/dataset/{id}/resource_data/{resource_id}', + controller='ckanext.datapusher.plugin:ResourceDataController', + action='resource_data') return m def get_actions(self): diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index f47f256cef6..9e8e88a939c 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -131,7 +131,7 @@ def test_send_datapusher_creates_task(self): task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': resource.id, 'task_type': 'datapusher', - 'key': 'job_id' + 'key': 'datapusher' }) assert task['state'] == 'pending', task @@ -148,18 +148,8 @@ def test_datapusher_hook(self): 'entity_id': resource.id, 'entity_type': 'resource', 'task_type': 'datapusher', - 'key': 'job_id', - 'value': 'my_id', - 'last_updated': str(datetime.datetime.now()), - 'state': 'pending' - }) - - p.toolkit.get_action('task_status_update')(context, { - 'entity_id': resource.id, - 'entity_type': 'resource', - 'task_type': 'datapusher', - 'key': 'job_key', - 'value': 'my_key', + 'key': 'datapusher', + 'value': '{"job_id": "my_id", "job_key":"my_key"}', 'last_updated': str(datetime.datetime.now()), 'state': 'pending' }) @@ -181,12 +171,12 @@ def test_datapusher_hook(self): task = tests.call_action_api( self.app, 'task_status_show', entity_id=resource.id, - task_type='datapusher', key='job_id') + task_type='datapusher', key='datapusher') assert task['state'] == 'success', task task = tests.call_action_api( self.app, 'task_status_show', entity_id=resource.id, - task_type='datapusher', key='job_key') + task_type='datapusher', key='datapusher') assert task['state'] == 'success', task diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index f4a14a352c6..3bb5def4e73 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -42,4 +42,3 @@ def dump(self, resource_id): for record in result['records']: wr.writerow([record[column] for column in header]) return f.getvalue() - From e39f6362e6c682703da7eb65f32ff7e7375e35ef Mon Sep 17 00:00:00 2001 From: kindly Date: Fri, 20 Sep 2013 21:21:23 +0100 Subject: [PATCH 132/269] [1200] fix task_staus in context --- ckanext/datapusher/tests/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index 9e8e88a939c..2ec2df42d21 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -128,6 +128,8 @@ def test_send_datapusher_creates_task(self): 'resource_id': resource.id }) + context.pop('task_status', None) + task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': resource.id, 'task_type': 'datapusher', From 736a614980f6876d632e0724434d77f93dc7e00a Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 23 Sep 2013 13:46:31 +0200 Subject: [PATCH 133/269] [#1117] Remove testing guidelines about assert messages This is unnecessary in most cases, should be clear from the test name (or else the test may be doing too much) --- doc/testing-coding-standards.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/doc/testing-coding-standards.rst b/doc/testing-coding-standards.rst index 81c9a0026f0..e5082a28de6 100644 --- a/doc/testing-coding-standards.rst +++ b/doc/testing-coding-standards.rst @@ -78,15 +78,6 @@ Clear * Test methods and helper functions should have docstrings. - * Assert statements in tests should have assert messages to explain why the - test is doing this assert, e.g.:: - - assert result['success'] is False, ("Users shouldn't be able to update " - "other users' accounts") - - Assert messages are printed out when tests fail, so the user can often see - what went wrong without even having to look into the test file. - Easy to find It should be easy to know where to add new tests for some new or changed code, or to find the existing tests for some code. From c5ea35f1db74e228aa1e47dbe92ca6db0b6c5ec4 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 23 Sep 2013 14:22:27 +0200 Subject: [PATCH 134/269] [#1117] Add factories.MockUser class --- ckan/new_tests/factories.py | 40 ++++++++++++++++++++++++ ckan/new_tests/logic/auth/test_update.py | 25 +++------------ 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/ckan/new_tests/factories.py b/ckan/new_tests/factories.py index 639c348a67b..6e8385eeacf 100644 --- a/ckan/new_tests/factories.py +++ b/ckan/new_tests/factories.py @@ -37,6 +37,7 @@ ''' import factory +import mock import ckan.model import ckan.logic @@ -49,6 +50,18 @@ def _generate_email(user): return '{0}@ckan.org'.format(user.name).lower() +def _generate_reset_key(user): + '''Return a reset key for the given User factory stub object.''' + + return '{0}_reset_key'.format(user.name).lower() + + +def _generate_user_id(user): + '''Return a user id for the given User factory stub object.''' + + return '{0}_user_id'.format(user.name).lower() + + class User(factory.Factory): '''A factory class for creating CKAN users.''' @@ -86,6 +99,33 @@ def _create(cls, target_class, *args, **kwargs): return user_dict +class MockUser(factory.Factory): + '''A factory class for creating mock CKAN users using the mock library.''' + + FACTORY_FOR = mock.MagicMock + + fullname = 'Mr. Mock User' + password = 'pass' + about = 'Just another mock user.' + name = factory.Sequence(lambda n: 'mock_user_{n}'.format(n=n)) + email = factory.LazyAttribute(_generate_email) + reset_key = factory.LazyAttribute(_generate_reset_key) + id = factory.LazyAttribute(_generate_user_id) + + @classmethod + def _build(cls, target_class, *args, **kwargs): + raise NotImplementedError(".build() isn't supported in CKAN") + + @classmethod + def _create(cls, target_class, *args, **kwargs): + if args: + assert False, "Positional args aren't supported, use keyword args." + mock_user = mock.MagicMock() + for name, value in kwargs.items(): + setattr(mock_user, name, value) + return mock_user + + def validator_data_dict(): '''Return a data dict with some arbitrary data in it, suitable to be passed to validator functions for testing. diff --git a/ckan/new_tests/logic/auth/test_update.py b/ckan/new_tests/logic/auth/test_update.py index a29e3220b3f..0970e2ed0a3 100644 --- a/ckan/new_tests/logic/auth/test_update.py +++ b/ckan/new_tests/logic/auth/test_update.py @@ -4,6 +4,7 @@ import mock import ckan.new_tests.helpers as helpers +import ckan.new_tests.factories as factories class TestUpdate(object): @@ -12,11 +13,7 @@ def test_user_update_visitor_cannot_update_user(self): '''Visitors should not be able to update users' accounts.''' # Make a mock ckan.model.User object, Fred. - fred = mock.MagicMock() - # Give the mock user object some attributes it's going to need. - fred.reset_key = 'fred_reset_key' - fred.id = 'fred_user_id' - fred.name = 'fred' + fred = factories.MockUser(name='fred') # Make a mock ckan.model object. mock_model = mock.MagicMock() @@ -51,11 +48,7 @@ def test_user_update_user_cannot_update_another_user(self): # 1. Setup. # Make a mock ckan.model.User object, Fred. - fred = mock.MagicMock() - # Give the mock user object some attributes it's going to need. - fred.reset_key = 'fred_reset_key' - fred.id = 'fred_user_id' - fred.name = 'fred' + fred = factories.MockUser(name='fred') # Make a mock ckan.model object. mock_model = mock.MagicMock() @@ -93,11 +86,7 @@ def test_user_update_user_can_update_herself(self): '''Users should be authorized to update their own accounts.''' # Make a mock ckan.model.User object, Fred. - fred = mock.MagicMock() - # Give the mock user object some attributes it's going to need. - fred.reset_key = 'fred_reset_key' - fred.id = 'fred_user_id' - fred.name = 'fred' + fred = factories.MockUser(name='fred') # Make a mock ckan.model object. mock_model = mock.MagicMock() @@ -125,11 +114,7 @@ def test_user_update_user_can_update_herself(self): def test_user_update_with_no_user_in_context(self): # Make a mock ckan.model.User object. - mock_user = mock.MagicMock() - # Give the mock user object some attributes it's going to need. - mock_user.reset_key = 'mock reset key' - mock_user.id = 'mock user id' - mock_user.name = 'mock_user' + mock_user = factories.MockUser(name='fred') # Make a mock ckan.model object. mock_model = mock.MagicMock() From e50a79c7382a400d5491e8f7af91c55f6ee8c93a Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 23 Sep 2013 14:31:29 +0200 Subject: [PATCH 135/269] [#1117] Remove an unnecessary deepcopy --- ckan/new_tests/lib/navl/test_validators.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index c2045a42587..c1fd96010b7 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -138,8 +138,6 @@ def call_and_assert(key, data, errors, context=None): if key in original_data_dict: del original_data_dict[key] result = validator(key, data, errors, context=context) - # Copy the data dict because we don't want to modify it. - data = copy.deepcopy(data) assert data == original_data_dict, message return result return call_and_assert From 084feee3bc7b94f70ec6282d90ba122b74342c1c Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 23 Sep 2013 15:02:46 +0200 Subject: [PATCH 136/269] [#1117] Simplify a test decorator --- ckan/new_tests/lib/navl/test_validators.py | 39 ++++++++++------------ 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index c1fd96010b7..3d0f04809e7 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -110,38 +110,36 @@ def call_and_assert(key, data, errors, context=None): return decorator -def does_not_modify_other_keys_in_data_dict(message): +def does_not_modify_other_keys_in_data_dict(validator): '''A decorator that asserts that the decorated validator doesn't add, modify the value of, or remove any other keys from its ``data`` dict param. The function *may* modify its own data dict key. - :param message: the message that will be printed if the function does - modify another key in the data dict and the assert fails + :param validator: the validator function to decorate :type message: string Usage: - @does_not_modify_other_keys_in_data_dict('user_name_validator() ' - 'should not modify other keys in the data dict') + @does_not_modify_other_keys_in_data_dict def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors) ''' - def decorator(validator): - def call_and_assert(key, data, errors, context=None): - if context is None: - context = {} - # Make a copy of the data dict so we can assert against it later. - original_data_dict = copy.deepcopy(data) - if key in original_data_dict: - del original_data_dict[key] - result = validator(key, data, errors, context=context) - assert data == original_data_dict, message - return result - return call_and_assert - return decorator + def call_and_assert(key, data, errors, context=None): + if context is None: + context = {} + # Make a copy of the data dict so we can assert against it later. + original_data_dict = copy.deepcopy(data) + if key in original_data_dict: + del original_data_dict[key] + result = validator(key, data, errors, context=context) + assert data == original_data_dict, ( + "Should not modify other keys in data_dict, when given a " + "value of {value}".format(value=repr(value))) + return result + return call_and_assert def does_not_modify_errors_dict(message): @@ -201,10 +199,7 @@ def test_ignore_missing_with_value_missing(self): errors = factories.validator_errors_dict() errors[key] = [] - @does_not_modify_other_keys_in_data_dict( - 'When given a value of {value} ignore_missing() should ' - 'not modify other items in the data dict'.format( - value=repr(value))) + @does_not_modify_other_keys_in_data_dict @does_not_modify_errors_dict( 'When given a value of {value} ignore_missing() should not ' 'modify the errors dict'.format(value=repr(value))) From 696f98b392d659211d4c04f60418bc206462a46d Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 23 Sep 2013 16:23:49 +0200 Subject: [PATCH 137/269] [#1117] Simplify some more test decorators --- ckan/new_tests/lib/navl/test_validators.py | 199 ++++++++++++--------- 1 file changed, 115 insertions(+), 84 deletions(-) diff --git a/ckan/new_tests/lib/navl/test_validators.py b/ckan/new_tests/lib/navl/test_validators.py index 3d0f04809e7..dccee431453 100644 --- a/ckan/new_tests/lib/navl/test_validators.py +++ b/ckan/new_tests/lib/navl/test_validators.py @@ -9,35 +9,41 @@ import ckan.new_tests.factories as factories -def returns_None(message): +def returns_None(function): '''A decorator that asserts that the decorated function returns None. - :param message: the message that will be printed if the function doesn't - return None and the assert fails - :type message: string + :param function: the function to decorate + :type function: function Usage: - @returns_None('user_name_validator() should return None when given ' - 'valid input') + @returns_None def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors) ''' - def decorator(function): - def call_and_assert(*args, **kwargs): - result = function(*args, **kwargs) - assert result is None, message - return result - return call_and_assert - return decorator + def call_and_assert(*args, **kwargs): + original_args = copy.deepcopy(args) + original_kwargs = copy.deepcopy(kwargs) + + result = function(*args, **kwargs) + + assert result is None, ( + 'Should return None when called with args: {args} and ' + 'kwargs: {kwargs}'.format(args=original_args, + kwargs=original_kwargs)) + return result + return call_and_assert def raises_StopOnError(function): '''A decorator that asserts that the decorated function raises dictization_functions.StopOnError. + :param function: the function to decorate + :type function: function + Usage: @raises_StopOnError @@ -52,62 +58,72 @@ def call_and_assert(*args, **kwargs): return call_and_assert -def does_not_modify_data_dict(message): +def does_not_modify_data_dict(validator): '''A decorator that asserts that the decorated validator doesn't modify its `data` dict param. - :param message: the message that will be printed if the function does - modify the data dict and the assert fails - :type message: string + :param validator: the validator function to decorate + :type validator: function Usage: - @does_not_modify_data_dict('user_name_validator() should not modify ' - 'the data dict') + @does_not_modify_data_dict def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors) ''' - def decorator(validator): - def call_and_assert(key, data, errors, context=None): - if context is None: - context = {} - # Make a copy of the data dict so we can assert against it later. - original_data_dict = copy.deepcopy(data) - result = validator(key, data, errors, context=context) - assert data == original_data_dict, message - return result - return call_and_assert - return decorator - - -def removes_key_from_data_dict(message): + def call_and_assert(key, data, errors, context=None): + if context is None: + context = {} + original_data = copy.deepcopy(data) + original_errors = copy.deepcopy(errors) + original_context = copy.deepcopy(context) + + result = validator(key, data, errors, context=context) + + assert data == original_data, ( + 'Should not modify data dict when called with ' + 'key: {key}, data: {data}, errors: {errors}, ' + 'context: {context}'.format(key=key, data=original_data, + errors=original_errors, + context=original_context)) + return result + return call_and_assert + + +def removes_key_from_data_dict(validator): '''A decorator that asserts that the decorated validator removes its key from the data dict. - :param message: the message that will be printed if the function does not - remove its key and the assert fails - :type message: string + :param validator: the validator function to decorate + :type validator: function Usage: - @removes_key_from_data_dict('user_name_validator() should remove its ' - 'key from the data dict') + @removes_key_from_data_dict def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors) ''' - def decorator(validator): - def call_and_assert(key, data, errors, context=None): - if context is None: - context = {} - result = validator(key, data, errors, context=context) - assert key not in data, message - return result - return call_and_assert - return decorator + def call_and_assert(key, data, errors, context=None): + if context is None: + context = {} + original_data = copy.deepcopy(data) + original_errors = copy.deepcopy(errors) + original_context = copy.deepcopy(context) + + result = validator(key, data, errors, context=context) + + assert key not in data, ( + 'Should remove key from data dict when called with: ' + 'key: {key}, data: {data}, errors: {errors}, ' + 'context: {context} '.format(key=key, data=original_data, + errors=original_errors, + context=original_context)) + return result + return call_and_assert def does_not_modify_other_keys_in_data_dict(validator): @@ -117,7 +133,7 @@ def does_not_modify_other_keys_in_data_dict(validator): The function *may* modify its own data dict key. :param validator: the validator function to decorate - :type message: string + :type validator: function Usage: @@ -130,46 +146,68 @@ def call_validator(*args, **kwargs): def call_and_assert(key, data, errors, context=None): if context is None: context = {} - # Make a copy of the data dict so we can assert against it later. - original_data_dict = copy.deepcopy(data) - if key in original_data_dict: - del original_data_dict[key] + original_data = copy.deepcopy(data) + original_errors = copy.deepcopy(errors) + original_context = copy.deepcopy(context) + result = validator(key, data, errors, context=context) - assert data == original_data_dict, ( - "Should not modify other keys in data_dict, when given a " - "value of {value}".format(value=repr(value))) + + # The validator function is allowed to modify its own key, so remove + # that key from both dicts for the purposes of the assertions below. + if key in data: + del data[key] + if key in original_data: + del original_data[key] + + assert data.keys() == original_data.keys(), ( + 'Should not add or remove keys from data dict when called with ' + 'key: {key}, data: {data}, errors: {errors}, ' + 'context: {context}'.format(key=key, data=original_data, + errors=original_errors, + context=original_context)) + for key_ in data: + assert data[key_] == original_data[key_], ( + 'Should not modify other keys in data dict when called with ' + 'key: {key}, data: {data}, errors: {errors}, ' + 'context: {context}'.format(key=key, data=original_data, + errors=original_errors, + context=original_context)) return result return call_and_assert -def does_not_modify_errors_dict(message): +def does_not_modify_errors_dict(validator): '''A decorator that asserts that the decorated validator doesn't modify its `errors` dict param. - :param message: the message that will be printed if the function does - modify the errors dict and the assert fails - :type message: string + :param validator: the validator function to decorate + :type validator: function Usage: - @does_not_modify_errors_dict('user_name_validator() should not modify ' - 'the errors dict') + @does_not_modify_errors_dict def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors) ''' - def decorator(validator): - def call_and_assert(key, data, errors, context=None): - if context is None: - context = {} - # Make a copy of the errors dict so we can assert against it later. - original_errors_dict = copy.deepcopy(errors) - result = validator(key, data, errors, context=context) - assert errors == original_errors_dict, message - return result - return call_and_assert - return decorator + def call_and_assert(key, data, errors, context=None): + if context is None: + context = {} + original_data = copy.deepcopy(data) + original_errors = copy.deepcopy(errors) + original_context = copy.deepcopy(context) + + result = validator(key, data, errors, context=context) + + assert errors == original_errors, ( + 'Should not modify errors dict when called with key: {key}, ' + 'data: {data}, errors: {errors}, ' + 'context: {context}'.format(key=key, data=original_data, + errors=original_errors, + context=original_context)) + return result + return call_and_assert class TestValidators(object): @@ -200,12 +238,8 @@ def test_ignore_missing_with_value_missing(self): errors[key] = [] @does_not_modify_other_keys_in_data_dict - @does_not_modify_errors_dict( - 'When given a value of {value} ignore_missing() should not ' - 'modify the errors dict'.format(value=repr(value))) - @removes_key_from_data_dict( - 'When given a value of {value} ignore_missing() should remove ' - 'the item from the data dict'.format(value=repr(value))) + @does_not_modify_errors_dict + @removes_key_from_data_dict @raises_StopOnError def call_validator(*args, **kwargs): return validators.ignore_missing(*args, **kwargs) @@ -224,12 +258,9 @@ def test_ignore_missing_with_a_value(self): errors = factories.validator_errors_dict() errors[key] = [] - @returns_None('When called with (non-missing) value, ignore_missing() ' - 'should return None') - @does_not_modify_data_dict("ignore_missing() shouldn't modify the " - 'data dict') - @does_not_modify_errors_dict("ignore_missing() shouldn't modify the " - 'errors dict') + @returns_None + @does_not_modify_data_dict + @does_not_modify_errors_dict def call_validator(*args, **kwargs): return validators.ignore_missing(*args, **kwargs) call_validator(key=key, data=data, errors=errors, context={}) From cf157df17944f68efd37be119e23f7cf06f6125b Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 23 Sep 2013 16:49:59 +0200 Subject: [PATCH 138/269] [#1117] Simplify some more test decorators And also remove some duplicated test decorators --- ckan/new_tests/logic/test_validators.py | 234 ++++++++---------------- 1 file changed, 77 insertions(+), 157 deletions(-) diff --git a/ckan/new_tests/logic/test_validators.py b/ckan/new_tests/logic/test_validators.py index aafda332eaa..301cd3fc540 100644 --- a/ckan/new_tests/logic/test_validators.py +++ b/ckan/new_tests/logic/test_validators.py @@ -8,59 +8,42 @@ import nose.tools import ckan.new_tests.factories as factories - - -def returns_arg(message): +# Import some test helper functions from another module directly. +# This is bad (test modules shouldn't share code with eachother) but because of +# the way validator functions are organised in CKAN (in different modules in +# different places in the code) we have to either do this or introduce a shared +# test helper functions module (which we also don't want to do). +from ckan.new_tests.lib.navl.test_validators import ( + returns_None, + does_not_modify_data_dict, + does_not_modify_errors_dict, + ) + + +def returns_arg(function): '''A decorator that tests that the decorated function returns the argument that it is called with, unmodified. - :param message: the message that will be printed if the function doesn't - return the same argument that it was called with and the assert fails - :type message: string + :param function: the function to decorate + :type function: function Usage: - @returns_arg('user_name_validator() should return the same arg that ' - 'it is called with, when called with a valid arg') + @returns_arg def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors) ''' - def decorator(function): - def call_and_assert(arg, context=None): - if context is None: - context = {} - result = function(arg, context=context) - assert result == arg, message - return result - return call_and_assert - return decorator - - -def returns_None(message): - '''A decorator that asserts that the decorated function returns None. - - :param message: the message that will be printed if the function doesn't - return None and the assert fails - :type message: string - - Usage: - - @returns_None('user_name_validator() should return None when given ' - 'valid input') - def call_validator(*args, **kwargs): - return validators.user_name_validator(*args, **kwargs) - call_validator(key, data, errors) - - ''' - def decorator(function): - def call_and_assert(*args, **kwargs): - result = function(*args, **kwargs) - assert result is None, message - return result - return call_and_assert - return decorator + def call_and_assert(arg, context=None): + if context is None: + context = {} + result = function(arg, context=context) + assert result == arg, ( + 'Should return the argument that was passed to it, unchanged ' + '({arg})'.format(arg=repr(arg))) + return result + return call_and_assert def raises_Invalid(function): @@ -81,102 +64,58 @@ def call_and_assert(*args, **kwargs): return call_and_assert -def does_not_modify_data_dict(message): - '''A decorator that asserts that the decorated validator doesn't modify - its `data` dict param. - - :param message: the message that will be printed if the function does - modify the data dict and the assert fails - :type message: string - - Usage: - - @does_not_modify_data_dict('user_name_validator() should not modify ' - 'the data dict') - def call_validator(*args, **kwargs): - return validators.user_name_validator(*args, **kwargs) - call_validator(key, data, errors) - - ''' - def decorator(validator): - def call_and_assert(key, data, errors, context=None): - if context is None: - context = {} - # Make a copy of the data dict so we can assert against it later. - original_data_dict = copy.deepcopy(data) - result = validator(key, data, errors, context=context) - assert data == original_data_dict, message - return result - return call_and_assert - return decorator - - -def does_not_modify_errors_dict(message): - '''A decorator that asserts that the decorated validator doesn't modify its - `errors` dict param. - - :param message: the message that will be printed if the function does - modify the errors dict and the assert fails - :type message: string - - Usage: - - @does_not_modify_errors_dict('user_name_validator() should not modify ' - 'the errors dict') - def call_validator(*args, **kwargs): - return validators.user_name_validator(*args, **kwargs) - call_validator(key, data, errors) - - ''' - def decorator(validator): - def call_and_assert(key, data, errors, context=None): - if context is None: - context = {} - # Make a copy of the errors dict so we can assert against it later. - original_errors_dict = copy.deepcopy(errors) - result = validator(key, data, errors, context=context) - assert errors == original_errors_dict, message - return result - return call_and_assert - return decorator - - -def does_not_modify_other_keys_in_errors_dict(message): +def does_not_modify_other_keys_in_errors_dict(validator): '''A decorator that asserts that the decorated validator doesn't add, - modify the value of, or remove any other keys from its `errors` dict param. + modify the value of, or remove any other keys from its ``errors`` dict + param. - The function *may* modify its own errors `key`. + The function *may* modify its own errors dict key. - :param message: the message that will be printed if the function does - modify another key in the errors dict and the assert fails - :type message: string + :param validator: the validator function to decorate + :type validator: function Usage: - @does_not_modify_other_keys_in_errors_dict('user_name_validator() ' - 'should not modify other keys in the errors dict') + @does_not_modify_other_keys_in_errors_dict def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors) ''' - def decorator(validator): - def call_and_assert(key, data, errors, context=None): - if context is None: - context = {} - # Make a copy of the errors dict so we can assert against it later. - original_errors_dict = copy.deepcopy(errors) - result = validator(key, data, errors, context=context) - # Copy the errors dict because we don't want to modify it. - errors = copy.deepcopy(errors) - errors[key] = [] - assert errors == original_errors_dict, message - return result - return call_and_assert - return decorator + def call_and_assert(key, data, errors, context=None): + if context is None: + context = {} + original_data = copy.deepcopy(data) + original_errors = copy.deepcopy(errors) + original_context = copy.deepcopy(context) + + result = validator(key, data, errors, context=context) + + # The validator function is allowed to modify its own key, so remove + # that key from both dicts for the purposes of the assertions below. + if key in errors: + del errors[key] + if key in original_errors: + del original_errors[key] + + assert errors.keys() == original_errors.keys(), ( + 'Should not add or remove keys from errors dict when called with ' + 'key: {key}, data: {data}, errors: {errors}, ' + 'context: {context}'.format(key=key, data=original_data, + errors=original_errors, + context=original_context)) + for key_ in errors: + assert errors[key_] == original_errors[key_], ( + 'Should not modify other keys in errors dict when called with ' + 'key: {key}, data: {data}, errors: {errors}, ' + 'context: {context}'.format(key=key, data=original_data, + errors=original_errors, + context=original_context)) + return result + return call_and_assert -def adds_message_to_errors_dict(error_message, message): +def adds_message_to_errors_dict(error_message): '''A decorator that asserts the the decorated validator adds a given error message to the `errors` dict. @@ -184,15 +123,9 @@ def adds_message_to_errors_dict(error_message, message): add to the `errors` dict :type error_message: string - :param message: the message that will be printed if the function doesn't - add the right error message to the errors dict, and the assert fails - :type message: string - Usage: - @adds_message_to_errors_dict('That login name is not available.', - 'user_name_validator() should add to the errors dict when called ' - 'with a user name with already exists') + @adds_message_to_errors_dict('That login name is not available.') def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors) @@ -201,7 +134,9 @@ def call_validator(*args, **kwargs): def decorator(validator): def call_and_assert(key, data, errors, context): result = validator(key, data, errors, context) - assert errors[key] == [error_message], message + assert errors[key] == [error_message], ( + 'Should add message to errors dict: {msg}'.format( + msg=error_message)) return result return call_and_assert return decorator @@ -286,8 +221,7 @@ def test_name_validator_with_valid_value(self): ] for valid_name in valid_names: - @returns_arg('If given a valid string name_validator() should ' - 'return the string unmodified') + @returns_arg def call_validator(*args, **kwargs): return validators.name_validator(*args, **kwargs) call_validator(valid_name) @@ -327,10 +261,7 @@ def test_user_name_validator_with_non_string_value(self): errors = factories.validator_errors_dict() errors[key] = [] - @does_not_modify_errors_dict( - 'user_name_validator() should not modify the errors dict') - @does_not_modify_data_dict( - 'user_name_validator() should not modify the data dict') + @does_not_modify_data_dict @raises_Invalid def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) @@ -356,17 +287,10 @@ def test_user_name_validator_with_a_name_that_already_exists(self): errors = factories.validator_errors_dict() errors[key] = [] - @does_not_modify_other_keys_in_errors_dict('user_name_validator() ' - 'should not modify other ' - 'keys in the errors dict') - @does_not_modify_data_dict('user_name_validator() should not modify ' - 'the data dict') - @returns_None('user_name_validator() should return None if called ' - 'with a user name that already exists') - @adds_message_to_errors_dict('That login name is not available.', - 'user_name_validator() should add to the ' - 'errors dict when called with the name ' - 'of a user that already exists') + @does_not_modify_other_keys_in_errors_dict + @does_not_modify_data_dict + @returns_None + @adds_message_to_errors_dict('That login name is not available.') def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors, context={'model': mock_model}) @@ -388,13 +312,9 @@ def test_user_name_validator_successful(self): # user with that name exists in the database. mock_model.User.get.return_value = None - @does_not_modify_errors_dict('user_name_validator() should not ' - 'modify the errors dict when given ' - 'valid input') - @does_not_modify_data_dict('user_name_validator() should not modify ' - 'the data dict when given valid input') - @returns_None('user_name_validator() should return None when given ' - 'valid input') + @does_not_modify_errors_dict + @does_not_modify_data_dict + @returns_None def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors, context={'model': mock_model}) From 5ca99eee04dae54c9f9542bb21b1114294365445 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 23 Sep 2013 17:25:33 +0200 Subject: [PATCH 139/269] [#1117] Tests use clean_db() not delete_all() This fixes crashes when the database was not initialized or was only partially initialized --- ckan/new_tests/helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ckan/new_tests/helpers.py b/ckan/new_tests/helpers.py index 371f5e02787..fece02a18e1 100644 --- a/ckan/new_tests/helpers.py +++ b/ckan/new_tests/helpers.py @@ -38,9 +38,7 @@ def reset_db(): # This prevents CKAN from hanging waiting for some unclosed connection. model.Session.close_all() - # Clean out any data from the db. This prevents tests failing due to data - # leftover from other tests or from previous test runs. - model.repo.delete_all() + model.repo.clean_db() def call_action(action_name, context=None, **kwargs): From 21c21df965609e9dbfd1a20d1e01dc5d44807f34 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Tue, 24 Sep 2013 15:01:23 -0300 Subject: [PATCH 140/269] [#1243] Add tests to group_member_create. --- ckan/tests/logic/test_action.py | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 89e030beac8..9d7bb3c1dbf 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1722,3 +1722,52 @@ def test_2_view_group(self): res_json = json.loads(res.body) assert res_json['success'] is False + +class TestMember(WsgiAppCase): + + sysadmin = None + + group = None + + def setup(self): + username = 'sysadmin' + groupname = 'test group' + organization_name = 'test organization' + CreateTestData.create_user('sysadmin', **{ 'sysadmin': True }) + CreateTestData.create_groups([{ 'name': groupname }, + { 'name': organization_name, + 'type': 'organization'}]) + self.sysadmin = model.User.get(username) + self.group = model.Group.get(groupname) + + def teardown(self): + model.repo.rebuild_db() + + def test_group_member_create_works_user_id_and_group_id(self): + self._assert_we_can_add_user_to_group(self.sysadmin.id, self.group.id) + + def test_group_member_create_works_with_user_id_and_group_name(self): + self._assert_we_can_add_user_to_group(self.sysadmin.id, self.group.name) + + def test_group_member_create_works_with_user_name_and_group_name(self): + self._assert_we_can_add_user_to_group(self.sysadmin.name, self.group.name) + + def _assert_we_can_add_user_to_group(self, user_id, group_id): + user = model.User.get(user_id) + group = model.Group.get(group_id) + url = '/api/action/group_member_create' + role = 'member' + postparams = '%s=1' % json.dumps({ + 'id': group_id, + 'username': user_id, + 'role': role}) + + res = self.app.post(url, params=postparams, + extra_environ={'Authorization': str(user.apikey)}) + + res = json.loads(res.body) + groups = user.get_groups(group.type, role) + group_ids = [g.id for g in groups] + assert res['success'] is True, res + assert group.id in group_ids, (group, user_groups) + From e7db320c5f514048f18e892e3158bfe0c4c83c1f Mon Sep 17 00:00:00 2001 From: Guy Sheffer Date: Wed, 25 Sep 2013 13:02:44 +0300 Subject: [PATCH 141/269] Add notes and changes to new language submission, since pull requests are not accepted anymore --- doc/i18n.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/i18n.rst b/doc/i18n.rst index 9d90ce18b23..2718298b925 100644 --- a/doc/i18n.rst +++ b/doc/i18n.rst @@ -27,8 +27,9 @@ Adding a New Language If you want to add an entirely new language to CKAN, you have two options. * :ref:`i18n-transifex`. Creating translation files using Transifex, the open source translation software. -* :ref:`i18n-manual`. Creating translation files manually. +* :ref:`i18n-manual`. Creating translation files manually in your own branch. +.. note:: Translating CKAN in Transifex is only enabled when a 'call for translations' is issued. .. _i18n-transifex: @@ -64,8 +65,6 @@ The Transifex workflow is as follows: Manual Setup ------------ -If you prefer not to use Transifex, you can create translation files manually. - .. note:: Please keep the CKAN core developers aware of new languages created in this way. All the English strings in CKAN are extracted into the ``ckan.pot`` file, which can be found in ``ckan/i18n``. @@ -106,13 +105,13 @@ We recommend using a translation tool, such as `poedit ` 3. Commit the Translation ++++++++++++++++++++++++++ -When the po is complete, commit it to the CKAN i18n repo:: - +When the po is complete, create a branch in your source, then commit it to the CKAN i18n repo:: + + git checkout master + git branch translation-YOUR_LANGUAGE git add ckan/i18n/YOUR_LANGUAGE/LC_MESSAGES/ckan.po git commit -m '[i18n]: New language po added: YOUR_LANGUAGE' ckan/i18n/YOUR_LANGUAGE/LC_MESSAGES/ckan.po - git push - -.. note:: You need to be given credentials to do this - to request these, contact the `ckan-discuss `_ list. + git push origin translation-YOUR_LANGUAGE 4. Compile a Translation ++++++++++++++++++++++++ From 2d46b36aa53cb96ca37c4040b514935e9abe5ced Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Wed, 25 Sep 2013 15:33:59 -0300 Subject: [PATCH 142/269] Correct typo in CHANGELOG, forntend -> frontend. --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 96bee677cd1..b97798a4522 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -93,7 +93,7 @@ Deprecated and removed: one. Please update your config files if using it. (#226) Known issues: - * Under certain authorization setups the forntend for the groups functionality + * Under certain authorization setups the frontend for the groups functionality may not work as expected (See #1176 #1175). From c8a483a77f82c4c8a545c569e87cd7843a3d92dc Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 30 Sep 2013 11:58:35 +0100 Subject: [PATCH 143/269] [#1221] use write url for search as this uses correct priviledges --- ckanext/datastore/logic/action.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 47a464518ce..d9d291b9df3 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -300,9 +300,7 @@ def datastore_search(context, data_dict): raise p.toolkit.ValidationError(errors) res_id = data_dict['resource_id'] - data_dict['connection_url'] = pylons.config.get( - 'ckan.datastore.read_url', - pylons.config['ckan.datastore.write_url']) + data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] resources_sql = sqlalchemy.text(u'''SELECT alias_of FROM "_table_metadata" WHERE name = :id''') From 9147540ab8e381a41ab849239b919c19a2641a35 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 1 Oct 2013 12:17:54 +0200 Subject: [PATCH 144/269] [#1117] Discourage overuse of mocking Tweak the testing guidelines to discourage overuse of mocking. --- ckan/new_tests/controllers/__init__.py | 1 + ckan/new_tests/logic/action/__init__.py | 3 +- ckan/new_tests/logic/action/test_update.py | 72 ---------------------- doc/testing-coding-standards.rst | 40 ++++++++---- 4 files changed, 30 insertions(+), 86 deletions(-) diff --git a/ckan/new_tests/controllers/__init__.py b/ckan/new_tests/controllers/__init__.py index 6a901790973..916996429f3 100644 --- a/ckan/new_tests/controllers/__init__.py +++ b/ckan/new_tests/controllers/__init__.py @@ -1,4 +1,5 @@ ''' +Controller tests probably shouldn't use mocking. .. todo:: diff --git a/ckan/new_tests/logic/action/__init__.py b/ckan/new_tests/logic/action/__init__.py index d666170367e..6272c5b24b2 100644 --- a/ckan/new_tests/logic/action/__init__.py +++ b/ckan/new_tests/logic/action/__init__.py @@ -3,7 +3,8 @@ Most action function tests will be high-level tests that both test the code in the action function itself, and also indirectly test the code in :mod:`ckan.lib`, :mod:`ckan.model`, :mod:`ckan.logic.schema` etc. that the -action function calls. +action function calls. This means that most action function tests should *not* +use mocking. Tests for action functions should use the :func:`ckan.new_tests.helpers.call_action` function to call the action diff --git a/ckan/new_tests/logic/action/test_update.py b/ckan/new_tests/logic/action/test_update.py index dd32c4f7c37..8413a3a8178 100644 --- a/ckan/new_tests/logic/action/test_update.py +++ b/ckan/new_tests/logic/action/test_update.py @@ -344,75 +344,3 @@ def test_user_update_does_not_return_reset_key(self): updated_user = helpers.call_action('user_update', **params) assert 'reset_key' not in updated_user - - ## START-COMPLEX-MOCK-EXAMPLE - - def test_user_update_with_deferred_commit(self): - '''Test that user_update()'s deferred_commit option works. - - In this test we mock out the rest of CKAN and test the user_update() - action function in isolation. What we're testing is simply that when - called with 'deferred_commit': True in its context, user_update() does - not call ckan.model.repo.commit(). - - ''' - # Patch ckan.model, so user_update will be accessing a mock object - # instead of the real model. - # It's ckan.logic.__init__.py:get_action() that actually adds the model - # into the context dict, and that module does - # `import ckan.model as model`, so what we actually need to patch is - # 'ckan.logic.model' (the name that the model has when get_action() - # accesses it), not 'ckan.model'. - model_patch = mock.patch('ckan.logic.model') - mock_model = model_patch.start() - - # Patch the validate() function, so validate() won't really be called - # when user_update() calls it. update.py does - # `_validate = ckan.lib.navl.dictization_functions.validate` so we - # actually to patch this new name that's assigned to the function, not - # its original name. - validate_patch = mock.patch('ckan.logic.action.update._validate') - mock_validate = validate_patch.start() - - # user_update() is going to call validate() with some params and it's - # going to use the result that validate() returns. So we need to give - # a mock validate function that will accept the right number of params - # and return the right result. - def mock_validate_function(data_dict, schema, context): - '''Simply return the data_dict as given (without doing any - validation) and an empty error dict.''' - return data_dict, {} - mock_validate.side_effect = mock_validate_function - - # Patch model_save, again update.py does - # `import ckan.logic.action.update.model_save as model_save` so we - # need to patch it by its new name - # 'ckan.logic.action.update.model_save'. - model_save_patch = mock.patch('ckan.logic.action.update.model_save') - model_save_patch.start() - - # Patch model_dictize, again using the new name that update.py imports - # it as. - model_dictize_patch = mock.patch( - 'ckan.logic.action.update.model_dictize') - model_dictize_patch.start() - - # Patch the get_action() function (using the name that update.py - # assigns to it) with a default mock function that does nothing and - # returns None. - get_action_patch = mock.patch('ckan.logic.action.update._get_action') - get_action_patch.start() - - # After all that patching, we can finally call user_update passing - # 'defer_commit': True. The logic code in user_update will be run but - # it'll have no effect because everything it calls is mocked out. - helpers.call_action('user_update', - context={'defer_commit': True}, - id='foobar', name='new_name', - ) - - # Assert that user_update did *not* call our mock model object's - # model.repo.commit() method. - assert not mock_model.repo.commit.called - - ## END-COMPLEX-MOCK-EXAMPLE diff --git a/doc/testing-coding-standards.rst b/doc/testing-coding-standards.rst index e5082a28de6..2bd89545da2 100644 --- a/doc/testing-coding-standards.rst +++ b/doc/testing-coding-standards.rst @@ -41,9 +41,6 @@ Guidelines for writing new-style tests We want the tests in :mod:`ckan.new_tests` to be: Fast - * Use the ``mock`` library to avoid pulling in other parts of CKAN - (especially the database), see :ref:`mock`. - * Don't share setup code between tests (e.g. in test class ``setup()`` or ``setup_class()`` methods, saved against the ``self`` attribute of test classes, or in test helper modules). @@ -52,6 +49,9 @@ Fast and have each test method call just the helpers it needs to do the setup that it needs. + * Where appropriate, use the ``mock`` library to avoid pulling in other parts + of CKAN (especially the database), see :ref:`mock`. + Independent * Each test module, class and method should be able to be run on its own. @@ -269,6 +269,30 @@ faster (especially when :py:mod:`ckan.model` is mocked out so that the tests don't touch the database). With mock objects we can also make assertions about what methods the function called on the mock object and with which arguments. +.. note:: + + Overuse of mocking is discouraged as it can make tests difficult to + understand and maintain. Mocking can be useful and make tests both faster + and simpler when used appropriately. Some rules of thumb: + + * Don't mock out more than one or two objects in a single test method. + + * Don't use mocking in more functional-style tests. For example the action + function tests in :py:mod:`ckan.new_tests.logic.action` and the + frontend tests in :py:mod:`ckan.new_tests.controllers` are functional + tests, and probably shouldn't do any mocking. + + * Do use mocking in more unit-style tests. For example the authorization + function tests in :py:mod:`ckan.new_tests.logic.auth`, the converter and + validator tests in :py:mod:`ckan.new_tests.logic.auth`, and most (all?) + lib tests in :py:mod:`ckan.new_tests.lib` are unit tests and should use + mocking when necessary (often it's possible to unit test a method in + isolation from other CKAN code without doing any mocking, which is ideal). + + In these kind of tests we can often mock one or two objects in a simple + and easy to understand way, and make the test both simpler and faster. + + A mock object is a special object that allows user code to access any attribute name or call any method name (and pass any parameters) on the object, and the code will always get another mock object back: @@ -301,16 +325,6 @@ out :py:mod:`ckan.model`: :start-after: ## START-AFTER :end-before: ## END-BEFORE -Here's a much more complex example that patches a number of CKAN modules with -mock modules, replaces CKAN functions with mock functions with mock behavior, -and makes assertions about how CKAN called the mock objects (you shouldn't -have to do mocking as complex as this too often!): - -.. literalinclude:: ../ckan/new_tests/logic/action/test_update.py - :language: python - :start-after: ## START-COMPLEX-MOCK-EXAMPLE - :end-before: ## END-COMPLEX-MOCK-EXAMPLE - ---- The following sections will give specific guidelines and examples for writing From 098ac5de1ece4a5851a2ee4cd4290847f3472aeb Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 1 Oct 2013 12:18:40 +0200 Subject: [PATCH 145/269] [#1117] Add another example to testing guidelines --- doc/testing-coding-standards.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/testing-coding-standards.rst b/doc/testing-coding-standards.rst index 2bd89545da2..dfc4f19169d 100644 --- a/doc/testing-coding-standards.rst +++ b/doc/testing-coding-standards.rst @@ -187,7 +187,12 @@ give the following recipe for all unit test methods to follow: 3. Make assertions about the return value, and / or any side effects. 4. Do absolutely nothing else. -Most CKAN tests should follow this form. +Most CKAN tests should follow this form. Here's an example of a simple action +function test demonstrating the recipe: + +.. literalinclude:: ../ckan/new_tests/logic/action/test_update.py + :start-after: ## START-AFTER + :end-before: ## END-BEFORE One common exception is when you want to use a ``for`` loop to call the function being tested multiple times, passing it lots of different arguments From af2ed73934b49cece0920266d0dba3b025ac114e Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 1 Oct 2013 12:19:18 +0200 Subject: [PATCH 146/269] [#1117] Testing guidelines small tweaks --- ckan/new_tests/helpers.py | 8 ++++---- doc/testing-coding-standards.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ckan/new_tests/helpers.py b/ckan/new_tests/helpers.py index fece02a18e1..51640a7e3dd 100644 --- a/ckan/new_tests/helpers.py +++ b/ckan/new_tests/helpers.py @@ -1,6 +1,6 @@ '''This is a collection of helper functions for use in tests. -We want to avoid using sharing test helper functions between test modules as +We want to avoid sharing test helper functions between test modules as much as possible, and we definitely don't want to share test fixtures between test modules, or to introduce a complex hierarchy of test class subclasses, etc. @@ -42,12 +42,12 @@ def reset_db(): def call_action(action_name, context=None, **kwargs): - '''Call the named ``ckan.logic.action`` and return the result. + '''Call the named ``ckan.logic.action`` function and return the result. For example:: - call_action('user_create', name='seanh', email='seanh@seanh.com', - password='pass') + user_dict = call_action('user_create', name='seanh', + email='seanh@seanh.com', password='pass') Any keyword arguments given will be wrapped in a dict and passed to the action function as its ``data_dict`` argument. diff --git a/doc/testing-coding-standards.rst b/doc/testing-coding-standards.rst index dfc4f19169d..8e0b7fd4349 100644 --- a/doc/testing-coding-standards.rst +++ b/doc/testing-coding-standards.rst @@ -219,7 +219,7 @@ How detailed should tests be? Generally, what we're trying to do is test the *interfaces* between modules in a way that supports modularization: if you change the code within a function, -method, class or module, if you don't break any of that code's unit tests you +method, class or module, if you don't break any of that code's tests you should be able to expect that CKAN as a whole will not be broken. As a general guideline, the tests for a function or method should: From 7e61b82eac629c92ba04e0d9567e96b7c709f49b Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 1 Oct 2013 12:22:40 +0200 Subject: [PATCH 147/269] Fix a Sphinx error --- ckan/plugins/interfaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index a75d6e99906..74462835474 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -578,7 +578,7 @@ def user_create(context, data_dict=None): be decorated with the ``auth_allow_anonymous_access`` decorator, available on the plugins toolkit. - For example: + For example:: import ckan.plugins as p From 3fbe844985a00aee1450fd7235ba61bcf293623b Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 1 Oct 2013 11:22:52 +0100 Subject: [PATCH 148/269] [#1126] Removed because it's not used --- ckanext/homepage/plugin.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ckanext/homepage/plugin.py b/ckanext/homepage/plugin.py index b934f505302..d9a79b48a2c 100644 --- a/ckanext/homepage/plugin.py +++ b/ckanext/homepage/plugin.py @@ -10,9 +10,6 @@ class HomepagePlugin(p.SingletonPlugin): p.implements(p.ITemplateHelpers, inherit=True) p.implements(p.IConfigurable, inherit=True) - featured_groups_cache = None - featured_orgs_cache = None - def configure(self, config): groups = config.get('ckan.featured_groups', '') if groups: @@ -29,16 +26,14 @@ def get_featured_organizations(self, count=1): list_action='organization_list', count=count, items=self.orgs) - self.featured_orgs_cache = orgs - return self.featured_orgs_cache + return orgs def get_featured_groups(self, count=1): groups = self.featured_group_org(get_action='group_show', list_action='group_list', count=count, items=self.groups) - self.featured_groups_cache = groups - return self.featured_groups_cache + return groups def featured_group_org(self, items, get_action, list_action, count): def get_group(id): From 66645627a9ac2a23cc0335e4ae3766d60e0aedc3 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 1 Oct 2013 16:17:29 +0200 Subject: [PATCH 149/269] [#1117] Fix an import (PEP8) --- ckan/new_tests/logic/test_validators.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/ckan/new_tests/logic/test_validators.py b/ckan/new_tests/logic/test_validators.py index 301cd3fc540..4aab0fad23c 100644 --- a/ckan/new_tests/logic/test_validators.py +++ b/ckan/new_tests/logic/test_validators.py @@ -8,16 +8,12 @@ import nose.tools import ckan.new_tests.factories as factories -# Import some test helper functions from another module directly. +# Import some test helper functions from another module. # This is bad (test modules shouldn't share code with eachother) but because of # the way validator functions are organised in CKAN (in different modules in # different places in the code) we have to either do this or introduce a shared # test helper functions module (which we also don't want to do). -from ckan.new_tests.lib.navl.test_validators import ( - returns_None, - does_not_modify_data_dict, - does_not_modify_errors_dict, - ) +import ckan.new_tests.lib.navl.test_validators as t def returns_arg(function): @@ -261,7 +257,7 @@ def test_user_name_validator_with_non_string_value(self): errors = factories.validator_errors_dict() errors[key] = [] - @does_not_modify_data_dict + @t.does_not_modify_data_dict @raises_Invalid def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) @@ -288,8 +284,8 @@ def test_user_name_validator_with_a_name_that_already_exists(self): errors[key] = [] @does_not_modify_other_keys_in_errors_dict - @does_not_modify_data_dict - @returns_None + @t.does_not_modify_data_dict + @t.returns_None @adds_message_to_errors_dict('That login name is not available.') def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) @@ -312,9 +308,9 @@ def test_user_name_validator_successful(self): # user with that name exists in the database. mock_model.User.get.return_value = None - @does_not_modify_errors_dict - @does_not_modify_data_dict - @returns_None + @t.does_not_modify_errors_dict + @t.does_not_modify_data_dict + @t.returns_None def call_validator(*args, **kwargs): return validators.user_name_validator(*args, **kwargs) call_validator(key, data, errors, context={'model': mock_model}) From ad55287efb4a706ad4e957bc43cf7b019ad3b3c6 Mon Sep 17 00:00:00 2001 From: kindly Date: Wed, 2 Oct 2013 09:38:54 +0100 Subject: [PATCH 150/269] [1126] move homepage modules into core and fix tests --- ckan/controllers/home.py | 3 +- ckan/lib/app_globals.py | 1 + ckan/lib/helpers.py | 71 +++++++++++++++ ckan/templates/home/index.html | 16 +--- ckan/templates/home/index_base.html | 15 ++++ .../templates/home/layout1.html | 2 +- .../templates/home/layout2.html | 2 +- .../templates/home/layout3.html | 2 +- .../home/snippets/featured_group.html | 0 .../home/snippets/featured_organization.html | 0 .../templates/home/snippets/promoted.html | 0 .../templates/home/snippets/search.html | 0 .../templates/home/snippets/stats.html | 0 ckanext/homepage/__init__.py | 0 ckanext/homepage/plugin.py | 89 ------------------- doc/configuration.rst | 14 +++ setup.py | 1 - 17 files changed, 106 insertions(+), 110 deletions(-) create mode 100644 ckan/templates/home/index_base.html rename {ckanext/homepage/theme => ckan}/templates/home/layout1.html (95%) rename {ckanext/homepage/theme => ckan}/templates/home/layout2.html (95%) rename {ckanext/homepage/theme => ckan}/templates/home/layout3.html (93%) rename {ckanext/homepage/theme => ckan}/templates/home/snippets/featured_group.html (100%) rename {ckanext/homepage/theme => ckan}/templates/home/snippets/featured_organization.html (100%) rename {ckanext/homepage/theme => ckan}/templates/home/snippets/promoted.html (100%) rename {ckanext/homepage/theme => ckan}/templates/home/snippets/search.html (100%) rename {ckanext/homepage/theme => ckan}/templates/home/snippets/stats.html (100%) delete mode 100644 ckanext/homepage/__init__.py delete mode 100644 ckanext/homepage/plugin.py diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py index ad34bd702ca..da3d4b3901b 100644 --- a/ckan/controllers/home.py +++ b/ckan/controllers/home.py @@ -179,8 +179,7 @@ def db_to_form_schema(group_type=None): # END OF DIRTYNESS - template = 'home/layout{0}.html'.format(g.homepage_style) - return base.render(template, cache_force=True) + return base.render('home/index.html', cache_force=True) def license(self): return base.render('home/license.html') diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index 074deb4e274..dde98f0a1ed 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -29,6 +29,7 @@ 'ckan.site_about', 'ckan.site_intro_text', 'ckan.site_custom_css', + 'ckan.homepage_style', ] config_details = { diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index c70d74d7dbc..94e7c97b0e6 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -1663,6 +1663,74 @@ def new_activities(): action = logic.get_action('dashboard_new_activities_count') return action({}, {}) +def get_featured_organizations(count=1): + '''Returns a list of favourite organization in the form + of organization_list action function + ''' + config_orgs = config.get('ckan.featured_orgs', '').split() + orgs = featured_group_org(get_action='organization_show', + list_action='organization_list', + count=count, + items=config_orgs) + return orgs + +def get_featured_groups(count=1): + '''Returns a list of favourite group the form + of organization_list action function + ''' + config_groups = config.get('ckan.featured_groups', '').split() + groups = featured_group_org(get_action='group_show', + list_action='group_list', + count=count, + items=config_groups) + return groups + +def featured_group_org(items, get_action, list_action, count): + def get_group(id): + context = {'ignore_auth': True, + 'limits': {'packages': 2}, + 'for_view': True} + data_dict = {'id': id} + + try: + out = logic.get_action(get_action)(context, data_dict) + except logic.ObjectNotFound: + return None + return out + + groups_data = [] + + extras = logic.get_action(list_action)({}, {}) + + # list of found ids to prevent duplicates + found = [] + for group_name in items + extras: + group = get_group(group_name) + if not group: + continue + # ckeck if duplicate + if group['id'] in found: + continue + found.append(group['id']) + groups_data.append(group) + if len(groups_data) == count: + break + + return groups_data + +def get_site_statistics(): + stats = {} + stats['dataset_count'] = logic.get_action('package_search')({}, {"rows": 1})['count'] + stats['group_count'] = len(logic.get_action('group_list')({}, {})) + stats['organization_count'] = len(logic.get_action('organization_list')({}, {})) + result =model.Session.execute( + '''select count(*) from related r + left join related_dataset rd on r.id = rd.related_id + where rd.status = 'active' or rd.id is null''').first()[0] + stats['related_count'] = result + + return stats + # these are the functions that will end up in `h` template helpers __allowed_functions__ = [ @@ -1761,4 +1829,7 @@ def new_activities(): 'radio', 'submit', 'asbool', + 'get_featured_organizations', + 'get_featured_groups', + 'get_site_statistics', ] diff --git a/ckan/templates/home/index.html b/ckan/templates/home/index.html index d9e4abfb17b..8684c551732 100644 --- a/ckan/templates/home/index.html +++ b/ckan/templates/home/index.html @@ -1,15 +1 @@ -{% extends "page.html" %} - -{% block subtitle %}{{ _("Welcome") }}{% endblock %} - -{% block maintag %}{% endblock %} -{% block toolbar %}{% endblock %} - -{% block content %} -
      -
      - {{ self.flash() }} -
      - {% block primary_content %}{% endblock %} -
      -{% endblock %} +{% include "home/layout" + (g.homepage_style or '1') + ".html" %} diff --git a/ckan/templates/home/index_base.html b/ckan/templates/home/index_base.html new file mode 100644 index 00000000000..d9e4abfb17b --- /dev/null +++ b/ckan/templates/home/index_base.html @@ -0,0 +1,15 @@ +{% extends "page.html" %} + +{% block subtitle %}{{ _("Welcome") }}{% endblock %} + +{% block maintag %}{% endblock %} +{% block toolbar %}{% endblock %} + +{% block content %} +
      +
      + {{ self.flash() }} +
      + {% block primary_content %}{% endblock %} +
      +{% endblock %} diff --git a/ckanext/homepage/theme/templates/home/layout1.html b/ckan/templates/home/layout1.html similarity index 95% rename from ckanext/homepage/theme/templates/home/layout1.html rename to ckan/templates/home/layout1.html index e40b5fe5e1b..50621df3a1b 100644 --- a/ckanext/homepage/theme/templates/home/layout1.html +++ b/ckan/templates/home/layout1.html @@ -1,4 +1,4 @@ -{% extends 'home/index.html' %} +{% extends 'home/index_base.html' %} {% block homepage_layout_class %} layout-1{% endblock %} diff --git a/ckanext/homepage/theme/templates/home/layout2.html b/ckan/templates/home/layout2.html similarity index 95% rename from ckanext/homepage/theme/templates/home/layout2.html rename to ckan/templates/home/layout2.html index 8ccf432533a..41e625092d2 100644 --- a/ckanext/homepage/theme/templates/home/layout2.html +++ b/ckan/templates/home/layout2.html @@ -1,4 +1,4 @@ -{% extends 'home/index.html' %} +{% extends 'home/index_base.html' %} {% block homepage_layout_class %} layout-2{% endblock %} diff --git a/ckanext/homepage/theme/templates/home/layout3.html b/ckan/templates/home/layout3.html similarity index 93% rename from ckanext/homepage/theme/templates/home/layout3.html rename to ckan/templates/home/layout3.html index 4e805845b21..08e56f45ff3 100644 --- a/ckanext/homepage/theme/templates/home/layout3.html +++ b/ckan/templates/home/layout3.html @@ -1,4 +1,4 @@ -{% extends 'home/index.html' %} +{% extends 'home/index_base.html' %} {% block homepage_layout_class %} layout-3{% endblock %} diff --git a/ckanext/homepage/theme/templates/home/snippets/featured_group.html b/ckan/templates/home/snippets/featured_group.html similarity index 100% rename from ckanext/homepage/theme/templates/home/snippets/featured_group.html rename to ckan/templates/home/snippets/featured_group.html diff --git a/ckanext/homepage/theme/templates/home/snippets/featured_organization.html b/ckan/templates/home/snippets/featured_organization.html similarity index 100% rename from ckanext/homepage/theme/templates/home/snippets/featured_organization.html rename to ckan/templates/home/snippets/featured_organization.html diff --git a/ckanext/homepage/theme/templates/home/snippets/promoted.html b/ckan/templates/home/snippets/promoted.html similarity index 100% rename from ckanext/homepage/theme/templates/home/snippets/promoted.html rename to ckan/templates/home/snippets/promoted.html diff --git a/ckanext/homepage/theme/templates/home/snippets/search.html b/ckan/templates/home/snippets/search.html similarity index 100% rename from ckanext/homepage/theme/templates/home/snippets/search.html rename to ckan/templates/home/snippets/search.html diff --git a/ckanext/homepage/theme/templates/home/snippets/stats.html b/ckan/templates/home/snippets/stats.html similarity index 100% rename from ckanext/homepage/theme/templates/home/snippets/stats.html rename to ckan/templates/home/snippets/stats.html diff --git a/ckanext/homepage/__init__.py b/ckanext/homepage/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/ckanext/homepage/plugin.py b/ckanext/homepage/plugin.py deleted file mode 100644 index d9a79b48a2c..00000000000 --- a/ckanext/homepage/plugin.py +++ /dev/null @@ -1,89 +0,0 @@ -import logging - -import ckan.plugins as p -import ckan.model as model - -log = logging.getLogger(__name__) - -class HomepagePlugin(p.SingletonPlugin): - p.implements(p.IConfigurer, inherit=True) - p.implements(p.ITemplateHelpers, inherit=True) - p.implements(p.IConfigurable, inherit=True) - - def configure(self, config): - groups = config.get('ckan.featured_groups', '') - if groups: - log.warning('Config setting `ckan.featured_groups` is deprecated ' - 'please use `ckanext.homepage.groups`') - self.groups = config.get('ckanext.homepage.groups', groups).split() - self.orgs = config.get('ckanext.homepage.orgs', '').split() - - def update_config(self, config): - p.toolkit.add_template_directory(config, 'theme/templates') - - def get_featured_organizations(self, count=1): - orgs = self.featured_group_org(get_action='organization_show', - list_action='organization_list', - count=count, - items=self.orgs) - return orgs - - def get_featured_groups(self, count=1): - groups = self.featured_group_org(get_action='group_show', - list_action='group_list', - count=count, - items=self.groups) - return groups - - def featured_group_org(self, items, get_action, list_action, count): - def get_group(id): - context = {'ignore_auth': True, - 'limits': {'packages': 2}, - 'for_view': True} - data_dict = {'id': id} - - try: - out = p.toolkit.get_action(get_action)(context, data_dict) - except p.toolkit.ObjectNotFound: - return None - return out - - groups_data = [] - - extras = p.toolkit.get_action(list_action)({}, {}) - - # list of found ids to prevent duplicates - found = [] - for group_name in items + extras: - group = get_group(group_name) - if not group: - continue - # ckeck if duplicate - if group['id'] in found: - continue - found.append(group['id']) - groups_data.append(group) - if len(groups_data) == count: - break - - return groups_data - - def get_site_statistics(self): - stats = {} - stats['dataset_count'] = p.toolkit.get_action('package_search')({}, {"rows": 1})['count'] - stats['group_count'] = len(p.toolkit.get_action('group_list')({}, {})) - stats['organization_count'] = len(p.toolkit.get_action('organization_list')({}, {})) - result =model.Session.execute( - '''select count(*) from related r - left join related_dataset rd on r.id = rd.related_id - where rd.status = 'active' or rd.id is null''').first()[0] - stats['related_count'] = len(p.toolkit.get_action('organization_list')({}, {})) - - return stats - - def get_helpers(self): - return { - 'get_featured_organizations': self.get_featured_organizations, - 'get_featured_groups': self.get_featured_groups, - 'get_site_statistics': self.get_site_statistics, - } diff --git a/doc/configuration.rst b/doc/configuration.rst index 089693d3fbd..3b2b5956153 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -810,6 +810,20 @@ Defines a list of group names or group ids. This setting is used to display groups and datasets from each group on the home page in the default templates (2 groups and 2 datasets for each group are displayed). +.. _ckan.featured_organizations: + +ckan.featured_orgs +^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.featured_orgs = org_one org_two + +Default Value: (empty) + +Defines a list of organization names or ids. This setting is used to display +organization and datasets from each group on the home page in the default +templates (2 groups and 2 datasets for each group are displayed). .. _ckan.gravatar_default: diff --git a/setup.py b/setup.py index 5245b96223d..3f5115bfa8f 100644 --- a/setup.py +++ b/setup.py @@ -129,7 +129,6 @@ [ckan.system_plugins] domain_object_mods = ckan.model.modification:DomainObjectModificationExtension - homepage = ckanext.homepage.plugin:HomepagePlugin [ckan.test_plugins] routes_plugin=tests.ckantestplugins:RoutesPlugin From f1616290b2f96c1c5cb5ad98d3ccdfbc3e9be492 Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 2 Oct 2013 11:30:27 +0100 Subject: [PATCH 151/269] [#1126] Removed index_base.html and changed layouts into snippets --- ckan/templates/home/index.html | 19 ++++++++++++- ckan/templates/home/index_base.html | 15 ---------- ckan/templates/home/layout1.html | 42 ++++++++++++--------------- ckan/templates/home/layout2.html | 44 +++++++++++++---------------- ckan/templates/home/layout3.html | 32 +++++++++------------ 5 files changed, 68 insertions(+), 84 deletions(-) delete mode 100644 ckan/templates/home/index_base.html diff --git a/ckan/templates/home/index.html b/ckan/templates/home/index.html index 8684c551732..b28ac82c05d 100644 --- a/ckan/templates/home/index.html +++ b/ckan/templates/home/index.html @@ -1 +1,18 @@ -{% include "home/layout" + (g.homepage_style or '1') + ".html" %} +{% extends "page.html" %} +{% set homepage_style = ( g.homepage_style or '1' ) %} + +{% block subtitle %}{{ _("Welcome") }}{% endblock %} + +{% block maintag %}{% endblock %} +{% block toolbar %}{% endblock %} + +{% block content %} +
      +
      + {{ self.flash() }} +
      + {% block primary_content %} + {% snippet "home/layout{0}.html".format(homepage_style) %} + {% endblock %} +
      +{% endblock %} diff --git a/ckan/templates/home/index_base.html b/ckan/templates/home/index_base.html deleted file mode 100644 index d9e4abfb17b..00000000000 --- a/ckan/templates/home/index_base.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "page.html" %} - -{% block subtitle %}{{ _("Welcome") }}{% endblock %} - -{% block maintag %}{% endblock %} -{% block toolbar %}{% endblock %} - -{% block content %} -
      -
      - {{ self.flash() }} -
      - {% block primary_content %}{% endblock %} -
      -{% endblock %} diff --git a/ckan/templates/home/layout1.html b/ckan/templates/home/layout1.html index 50621df3a1b..53ac0d390bb 100644 --- a/ckan/templates/home/layout1.html +++ b/ckan/templates/home/layout1.html @@ -1,30 +1,24 @@ -{% extends 'home/index_base.html' %} - -{% block homepage_layout_class %} layout-1{% endblock %} - -{% block primary_content %} -
      -
      -
      -
      - {% snippet 'home/snippets/promoted.html' %} -
      -
      - {% snippet 'home/snippets/search.html' %} -
      +
      +
      +
      +
      + {% snippet 'home/snippets/promoted.html' %} +
      +
      + {% snippet 'home/snippets/search.html' %}
      -
      -
      -
      -
      - {% snippet 'home/snippets/featured_group.html' %} -
      -
      - {% snippet 'home/snippets/featured_organization.html' %} -
      +
      +
      +
      +
      +
      + {% snippet 'home/snippets/featured_group.html' %} +
      +
      + {% snippet 'home/snippets/featured_organization.html' %}
      -{% endblock %} +
      diff --git a/ckan/templates/home/layout2.html b/ckan/templates/home/layout2.html index 41e625092d2..e1b62bea09f 100644 --- a/ckan/templates/home/layout2.html +++ b/ckan/templates/home/layout2.html @@ -1,31 +1,25 @@ -{% extends 'home/index_base.html' %} - -{% block homepage_layout_class %} layout-2{% endblock %} - -{% block primary_content %} -
      -
      -
      -
      - {% snippet 'home/snippets/search.html' %} - {% snippet 'home/snippets/stats.html' %} -
      -
      - {% snippet 'home/snippets/promoted.html' %} -
      +
      +
      +
      +
      + {% snippet 'home/snippets/search.html' %} + {% snippet 'home/snippets/stats.html' %} +
      +
      + {% snippet 'home/snippets/promoted.html' %}
      -
      -
      -
      -
      - {% snippet 'home/snippets/featured_organization.html' %} -
      -
      - {% snippet 'home/snippets/featured_group.html' %} -
      +
      +
      +
      +
      +
      + {% snippet 'home/snippets/featured_organization.html' %} +
      +
      + {% snippet 'home/snippets/featured_group.html' %}
      -{% endblock %} +
      diff --git a/ckan/templates/home/layout3.html b/ckan/templates/home/layout3.html index 08e56f45ff3..80c10c22ba0 100644 --- a/ckan/templates/home/layout3.html +++ b/ckan/templates/home/layout3.html @@ -1,23 +1,17 @@ -{% extends 'home/index_base.html' %} - -{% block homepage_layout_class %} layout-3{% endblock %} - -{% block primary_content %} -
      -
      - {% snippet 'home/snippets/search.html' %} -
      +
      +
      + {% snippet 'home/snippets/search.html' %}
      -
      -
      -
      -
      - {% snippet 'home/snippets/promoted.html' %} -
      -
      - {% snippet 'home/snippets/stats.html' %} -
      +
      +
      +
      +
      +
      + {% snippet 'home/snippets/promoted.html' %} +
      +
      + {% snippet 'home/snippets/stats.html' %}
      -{% endblock %} +
      From c339162836695677f02e9090f2ce40f6a65ad194 Mon Sep 17 00:00:00 2001 From: John Glover Date: Wed, 2 Oct 2013 16:59:33 +0200 Subject: [PATCH 152/269] [#1178] PEP8 (and correct a comment) --- ckan/logic/action/create.py | 3 +++ ckan/logic/action/delete.py | 1 + ckan/logic/auth/delete.py | 4 +++- ckan/model/follower.py | 13 +++++++------ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 371ca62e625..066ed7bcc87 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -841,6 +841,7 @@ def user_create(context, data_dict): log.debug('Created user {name}'.format(name=user.name)) return user_dict + def user_invite(context, data_dict): '''Invite a new user. @@ -889,6 +890,7 @@ def user_invite(context, data_dict): if 'name' not in e.error_dict: raise e + def _get_random_username_from_email(email): localpart = email.split('@')[0] cleaned_localpart = re.sub(r'[^\w]', '-', localpart) @@ -896,6 +898,7 @@ def _get_random_username_from_email(email): name = '%s-%d' % (cleaned_localpart, random_number) return name + ## Modifications for rest api def package_create_rest(context, data_dict): diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index 9191151aa01..210f9d72387 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -18,6 +18,7 @@ _get_or_bust = ckan.logic.get_or_bust _get_action = ckan.logic.get_action + def user_delete(context, data_dict): '''Delete a user. diff --git a/ckan/logic/auth/delete.py b/ckan/logic/auth/delete.py index a2b121c11ad..08974144a71 100644 --- a/ckan/logic/auth/delete.py +++ b/ckan/logic/auth/delete.py @@ -4,10 +4,12 @@ from ckan.logic.auth import get_resource_object from ckan.lib.base import _ + def user_delete(context, data_dict): - # Only sysadmins are authorized to purge organizations. + # sysadmins only return {'success': False} + def package_delete(context, data_dict): user = context['user'] package = get_package_object(context, data_dict) diff --git a/ckan/model/follower.py b/ckan/model/follower.py index a8afd07706f..eb54cd9ca67 100644 --- a/ckan/model/follower.py +++ b/ckan/model/follower.py @@ -1,7 +1,6 @@ import meta import datetime import sqlalchemy -import vdm.sqlalchemy import core import ckan.model @@ -78,15 +77,17 @@ def _get(cls, follower_id=None, object_id=None): object_id = object_id or cls.object_id query = meta.Session.query(cls, follower_alias, object_alias)\ - .filter(sqlalchemy.and_(follower_alias.id == follower_id,\ - cls.follower_id == follower_alias.id,\ - cls.object_id == object_alias.id,\ - follower_alias.state != core.State.DELETED,\ - object_alias.state != core.State.DELETED,\ + .filter(sqlalchemy.and_( + follower_alias.id == follower_id, + cls.follower_id == follower_alias.id, + cls.object_id == object_alias.id, + follower_alias.state != core.State.DELETED, + object_alias.state != core.State.DELETED, object_alias.id == object_id)) return query + class UserFollowingUser(ModelFollowingModel): '''A many-many relationship between users. From 2e0266f12003244bc32795e82a9825574b93bf72 Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 2 Oct 2013 16:04:32 +0100 Subject: [PATCH 153/269] [#1070] Makes recaptcha work with the jinja2 templates --- ckan/lib/app_globals.py | 2 +- ckan/public/base/less/forms.less | 5 +++++ ckan/templates/user/new_user_form.html | 5 +++++ ckan/templates/user/snippets/recaptcha.html | 12 ++++++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 ckan/templates/user/snippets/recaptcha.html diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index 0739274c671..1b95264ad1e 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -38,7 +38,7 @@ # has been setup in load_environment(): 'ckan.site_id': {}, 'ckan.recaptcha.publickey': {'name': 'recaptcha_publickey'}, - 'ckan.recaptcha.privatekey': {'name': 'recaptcha_publickey'}, + 'ckan.recaptcha.privatekey': {'name': 'recaptcha_privatekey'}, 'ckan.template_title_deliminater': {'default': '-'}, 'ckan.template_head_end': {}, 'ckan.template_footer_end': {}, diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 289439c291a..c9de5810ee4 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -670,3 +670,8 @@ textarea { .box-shadow(none); } } + +#recaptcha_table { + table-layout: inherit; + line-height: 1; +} diff --git a/ckan/templates/user/new_user_form.html b/ckan/templates/user/new_user_form.html index 3879f2dde6c..57a0f1d85e0 100644 --- a/ckan/templates/user/new_user_form.html +++ b/ckan/templates/user/new_user_form.html @@ -7,6 +7,11 @@ {{ form.input("email", id="field-email", label=_("Email"), type="email", placeholder="joe@example.com", value=data.email, error=errors.email, classes=["control-medium"]) }} {{ form.input("password1", id="field-password", label=_("Password"), type="password", placeholder="••••••••", value=data.password1, error=errors.password1, classes=["control-medium"]) }} {{ form.input("password2", id="field-confirm-password", label=_("Confirm"), type="password", placeholder="••••••••", value=data.password2, error=errors.password1, classes=["control-medium"]) }} + + {% if g.recaptcha_publickey %} + {% snippet "user/snippets/recaptcha.html", public_key=g.recaptcha_publickey %} + {% endif %} +
      diff --git a/ckan/templates/user/snippets/recaptcha.html b/ckan/templates/user/snippets/recaptcha.html new file mode 100644 index 00000000000..994867dc97f --- /dev/null +++ b/ckan/templates/user/snippets/recaptcha.html @@ -0,0 +1,12 @@ +
      +
      + + + + +
      +
      From 4ee4ddb84f6c4f34adfa9783e7ee531c9664a8b3 Mon Sep 17 00:00:00 2001 From: John Glover Date: Thu, 3 Oct 2013 10:31:42 +0200 Subject: [PATCH 154/269] [#1126] PEP8 and fix typo --- ckan/lib/helpers.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 94e7c97b0e6..edd9746f0a8 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -1663,6 +1663,7 @@ def new_activities(): action = logic.get_action('dashboard_new_activities_count') return action({}, {}) + def get_featured_organizations(count=1): '''Returns a list of favourite organization in the form of organization_list action function @@ -1674,6 +1675,7 @@ def get_featured_organizations(count=1): items=config_orgs) return orgs + def get_featured_groups(count=1): '''Returns a list of favourite group the form of organization_list action function @@ -1685,6 +1687,7 @@ def get_featured_groups(count=1): items=config_groups) return groups + def featured_group_org(items, get_action, list_action, count): def get_group(id): context = {'ignore_auth': True, @@ -1708,7 +1711,7 @@ def get_group(id): group = get_group(group_name) if not group: continue - # ckeck if duplicate + # check if duplicate if group['id'] in found: continue found.append(group['id']) @@ -1718,12 +1721,15 @@ def get_group(id): return groups_data + def get_site_statistics(): stats = {} - stats['dataset_count'] = logic.get_action('package_search')({}, {"rows": 1})['count'] + stats['dataset_count'] = logic.get_action('package_search')( + {}, {"rows": 1})['count'] stats['group_count'] = len(logic.get_action('group_list')({}, {})) - stats['organization_count'] = len(logic.get_action('organization_list')({}, {})) - result =model.Session.execute( + stats['organization_count'] = len( + logic.get_action('organization_list')({}, {})) + result = model.Session.execute( '''select count(*) from related r left join related_dataset rd on r.id = rd.related_id where rd.status = 'active' or rd.id is null''').first()[0] From ae5a51c6d1765bf2d8271fa4c67a4a79ed5d8451 Mon Sep 17 00:00:00 2001 From: John Glover Date: Thu, 3 Oct 2013 10:51:58 +0200 Subject: [PATCH 155/269] [#1126] Update css after merge --- ckan/public/base/css/fuchsia.css | 971 +++---------------------------- ckan/public/base/css/green.css | 971 +++---------------------------- ckan/public/base/css/main.css | 122 ++-- ckan/public/base/css/maroon.css | 971 +++---------------------------- ckan/public/base/css/red.css | 971 +++---------------------------- 5 files changed, 374 insertions(+), 3632 deletions(-) diff --git a/ckan/public/base/css/fuchsia.css b/ckan/public/base/css/fuchsia.css index 300d2981817..a8a33f21105 100644 --- a/ckan/public/base/css/fuchsia.css +++ b/ckan/public/base/css/fuchsia.css @@ -2055,6 +2055,14 @@ table th[class*="span"], .open > .dropdown-menu { display: block; } +.dropdown-backdrop { + position: fixed; + left: 0; + right: 0; + bottom: 0; + top: 0; + z-index: 990; +} .pull-right > .dropdown-menu { right: 0; left: auto; @@ -5572,6 +5580,12 @@ textarea { .control-large input { height: 41px; } +.control-required { + color: #c6898b; +} +.form-actions .control-required-message { + float: left; +} .form-actions { background: none; margin-left: -25px; @@ -6904,842 +6918,6 @@ h4 small { height: 35px; background-position: -320px -62px; } -/* This is a modified version of the font-awesome core less document. */ -@font-face { - font-family: 'FontAwesome'; - src: url('../../../base/vendor/font-awesome/font/fontawesome-webfont.eot?v=3.0.1'); - src: url('../../../base/vendor/font-awesome/font/fontawesome-webfont.eot?#iefix&v=3.0.1') format('embedded-opentype'), url('../../../base/vendor/font-awesome/font/fontawesome-webfont.woff?v=3.0.1') format('woff'), url('../../../base/vendor/font-awesome/font/fontawesome-webfont.ttf?v=3.0.1') format('truetype'); - font-weight: normal; - font-style: normal; -} -[class^="icon-"], -[class*=" icon-"] { - font-family: FontAwesome; - font-weight: normal; - font-style: normal; - text-decoration: inherit; - -webkit-font-smoothing: antialiased; - display: inline; - width: auto; - height: auto; - line-height: normal; - vertical-align: baseline; - background-image: none; - background-position: 0% 0%; - background-repeat: repeat; - margin-top: 0; -} -.icon-spin { - display: inline-block; - -moz-animation: spin 2s infinite linear; - -o-animation: spin 2s infinite linear; - -webkit-animation: spin 2s infinite linear; - animation: spin 2s infinite linear; -} -@-moz-keyframes spin { - 0% { - -moz-transform: rotate(0deg); - } - 100% { - -moz-transform: rotate(359deg); - } -} -@-webkit-keyframes spin { - 0% { - -webkit-transform: rotate(0deg); - } - 100% { - -webkit-transform: rotate(359deg); - } -} -@-o-keyframes spin { - 0% { - -o-transform: rotate(0deg); - } - 100% { - -o-transform: rotate(359deg); - } -} -@-ms-keyframes spin { - 0% { - -ms-transform: rotate(0deg); - } - 100% { - -ms-transform: rotate(359deg); - } -} -@keyframes spin { - 0% { - transform: rotate(0deg); - } - 100% { - transform: rotate(359deg); - } -} -@-moz-document url-prefix() { - .icon-spin { - height: .9em; - } - .btn .icon-spin { - height: auto; - } - .icon-spin.icon-large { - height: 1.25em; - } - .btn .icon-spin.icon-large { - height: .75em; - } -} -/* Font Awesome uses the Unicode Private Use Area (PUA) to ensure screen - readers do not read off random characters that represent icons */ -.icon-glass:before { - content: "\f000"; -} -.icon-music:before { - content: "\f001"; -} -.icon-search:before { - content: "\f002"; -} -.icon-envelope:before { - content: "\f003"; -} -.icon-heart:before { - content: "\f004"; -} -.icon-star:before { - content: "\f005"; -} -.icon-star-empty:before { - content: "\f006"; -} -.icon-user:before { - content: "\f007"; -} -.icon-film:before { - content: "\f008"; -} -.icon-th-large:before { - content: "\f009"; -} -.icon-th:before { - content: "\f00a"; -} -.icon-th-list:before { - content: "\f00b"; -} -.icon-ok:before { - content: "\f00c"; -} -.icon-remove:before { - content: "\f00d"; -} -.icon-zoom-in:before { - content: "\f00e"; -} -.icon-zoom-out:before { - content: "\f010"; -} -.icon-off:before { - content: "\f011"; -} -.icon-signal:before { - content: "\f012"; -} -.icon-cog:before { - content: "\f013"; -} -.icon-trash:before { - content: "\f014"; -} -.icon-home:before { - content: "\f015"; -} -.icon-file:before { - content: "\f016"; -} -.icon-time:before { - content: "\f017"; -} -.icon-road:before { - content: "\f018"; -} -.icon-download-alt:before { - content: "\f019"; -} -.icon-download:before { - content: "\f01a"; -} -.icon-upload:before { - content: "\f01b"; -} -.icon-inbox:before { - content: "\f01c"; -} -.icon-play-circle:before { - content: "\f01d"; -} -.icon-repeat:before { - content: "\f01e"; -} -/* \f020 doesn't work in Safari. all shifted one down */ -.icon-refresh:before { - content: "\f021"; -} -.icon-list-alt:before { - content: "\f022"; -} -.icon-lock:before { - content: "\f023"; -} -.icon-flag:before { - content: "\f024"; -} -.icon-headphones:before { - content: "\f025"; -} -.icon-volume-off:before { - content: "\f026"; -} -.icon-volume-down:before { - content: "\f027"; -} -.icon-volume-up:before { - content: "\f028"; -} -.icon-qrcode:before { - content: "\f029"; -} -.icon-barcode:before { - content: "\f02a"; -} -.icon-tag:before { - content: "\f02b"; -} -.icon-tags:before { - content: "\f02c"; -} -.icon-book:before { - content: "\f02d"; -} -.icon-bookmark:before { - content: "\f02e"; -} -.icon-print:before { - content: "\f02f"; -} -.icon-camera:before { - content: "\f030"; -} -.icon-font:before { - content: "\f031"; -} -.icon-bold:before { - content: "\f032"; -} -.icon-italic:before { - content: "\f033"; -} -.icon-text-height:before { - content: "\f034"; -} -.icon-text-width:before { - content: "\f035"; -} -.icon-align-left:before { - content: "\f036"; -} -.icon-align-center:before { - content: "\f037"; -} -.icon-align-right:before { - content: "\f038"; -} -.icon-align-justify:before { - content: "\f039"; -} -.icon-list:before { - content: "\f03a"; -} -.icon-indent-left:before { - content: "\f03b"; -} -.icon-indent-right:before { - content: "\f03c"; -} -.icon-facetime-video:before { - content: "\f03d"; -} -.icon-picture:before { - content: "\f03e"; -} -.icon-pencil:before { - content: "\f040"; -} -.icon-map-marker:before { - content: "\f041"; -} -.icon-adjust:before { - content: "\f042"; -} -.icon-tint:before { - content: "\f043"; -} -.icon-edit:before { - content: "\f044"; -} -.icon-share:before { - content: "\f045"; -} -.icon-check:before { - content: "\f046"; -} -.icon-move:before { - content: "\f047"; -} -.icon-step-backward:before { - content: "\f048"; -} -.icon-fast-backward:before { - content: "\f049"; -} -.icon-backward:before { - content: "\f04a"; -} -.icon-play:before { - content: "\f04b"; -} -.icon-pause:before { - content: "\f04c"; -} -.icon-stop:before { - content: "\f04d"; -} -.icon-forward:before { - content: "\f04e"; -} -.icon-fast-forward:before { - content: "\f050"; -} -.icon-step-forward:before { - content: "\f051"; -} -.icon-eject:before { - content: "\f052"; -} -.icon-chevron-left:before { - content: "\f053"; -} -.icon-chevron-right:before { - content: "\f054"; -} -.icon-plus-sign:before { - content: "\f055"; -} -.icon-minus-sign:before { - content: "\f056"; -} -.icon-remove-sign:before { - content: "\f057"; -} -.icon-ok-sign:before { - content: "\f058"; -} -.icon-question-sign:before { - content: "\f059"; -} -.icon-info-sign:before { - content: "\f05a"; -} -.icon-screenshot:before { - content: "\f05b"; -} -.icon-remove-circle:before { - content: "\f05c"; -} -.icon-ok-circle:before { - content: "\f05d"; -} -.icon-ban-circle:before { - content: "\f05e"; -} -.icon-arrow-left:before { - content: "\f060"; -} -.icon-arrow-right:before { - content: "\f061"; -} -.icon-arrow-up:before { - content: "\f062"; -} -.icon-arrow-down:before { - content: "\f063"; -} -.icon-share-alt:before { - content: "\f064"; -} -.icon-resize-full:before { - content: "\f065"; -} -.icon-resize-small:before { - content: "\f066"; -} -.icon-plus:before { - content: "\f067"; -} -.icon-minus:before { - content: "\f068"; -} -.icon-asterisk:before { - content: "\f069"; -} -.icon-exclamation-sign:before { - content: "\f06a"; -} -.icon-gift:before { - content: "\f06b"; -} -.icon-leaf:before { - content: "\f06c"; -} -.icon-fire:before { - content: "\f06d"; -} -.icon-eye-open:before { - content: "\f06e"; -} -.icon-eye-close:before { - content: "\f070"; -} -.icon-warning-sign:before { - content: "\f071"; -} -.icon-plane:before { - content: "\f072"; -} -.icon-calendar:before { - content: "\f073"; -} -.icon-random:before { - content: "\f074"; -} -.icon-comment:before { - content: "\f075"; -} -.icon-magnet:before { - content: "\f076"; -} -.icon-chevron-up:before { - content: "\f077"; -} -.icon-chevron-down:before { - content: "\f078"; -} -.icon-retweet:before { - content: "\f079"; -} -.icon-shopping-cart:before { - content: "\f07a"; -} -.icon-folder-close:before { - content: "\f07b"; -} -.icon-folder-open:before { - content: "\f07c"; -} -.icon-resize-vertical:before { - content: "\f07d"; -} -.icon-resize-horizontal:before { - content: "\f07e"; -} -.icon-bar-chart:before { - content: "\f080"; -} -.icon-twitter-sign:before { - content: "\f081"; -} -.icon-facebook-sign:before { - content: "\f082"; -} -.icon-camera-retro:before { - content: "\f083"; -} -.icon-key:before { - content: "\f084"; -} -.icon-cogs:before { - content: "\f085"; -} -.icon-comments:before { - content: "\f086"; -} -.icon-thumbs-up:before { - content: "\f087"; -} -.icon-thumbs-down:before { - content: "\f088"; -} -.icon-star-half:before { - content: "\f089"; -} -.icon-heart-empty:before { - content: "\f08a"; -} -.icon-signout:before { - content: "\f08b"; -} -.icon-linkedin-sign:before { - content: "\f08c"; -} -.icon-pushpin:before { - content: "\f08d"; -} -.icon-external-link:before { - content: "\f08e"; -} -.icon-signin:before { - content: "\f090"; -} -.icon-trophy:before { - content: "\f091"; -} -.icon-github-sign:before { - content: "\f092"; -} -.icon-upload-alt:before { - content: "\f093"; -} -.icon-lemon:before { - content: "\f094"; -} -.icon-phone:before { - content: "\f095"; -} -.icon-check-empty:before { - content: "\f096"; -} -.icon-bookmark-empty:before { - content: "\f097"; -} -.icon-phone-sign:before { - content: "\f098"; -} -.icon-twitter:before { - content: "\f099"; -} -.icon-facebook:before { - content: "\f09a"; -} -.icon-github:before { - content: "\f09b"; -} -.icon-unlock:before { - content: "\f09c"; -} -.icon-credit-card:before { - content: "\f09d"; -} -.icon-rss:before { - content: "\f09e"; -} -.icon-hdd:before { - content: "\f0a0"; -} -.icon-bullhorn:before { - content: "\f0a1"; -} -.icon-bell:before { - content: "\f0a2"; -} -.icon-certificate:before { - content: "\f0a3"; -} -.icon-hand-right:before { - content: "\f0a4"; -} -.icon-hand-left:before { - content: "\f0a5"; -} -.icon-hand-up:before { - content: "\f0a6"; -} -.icon-hand-down:before { - content: "\f0a7"; -} -.icon-circle-arrow-left:before { - content: "\f0a8"; -} -.icon-circle-arrow-right:before { - content: "\f0a9"; -} -.icon-circle-arrow-up:before { - content: "\f0aa"; -} -.icon-circle-arrow-down:before { - content: "\f0ab"; -} -.icon-globe:before { - content: "\f0ac"; -} -.icon-wrench:before { - content: "\f0ad"; -} -.icon-tasks:before { - content: "\f0ae"; -} -.icon-filter:before { - content: "\f0b0"; -} -.icon-briefcase:before { - content: "\f0b1"; -} -.icon-fullscreen:before { - content: "\f0b2"; -} -.icon-group:before { - content: "\f0c0"; -} -.icon-link:before { - content: "\f0c1"; -} -.icon-cloud:before { - content: "\f0c2"; -} -.icon-beaker:before { - content: "\f0c3"; -} -.icon-cut:before { - content: "\f0c4"; -} -.icon-copy:before { - content: "\f0c5"; -} -.icon-paper-clip:before { - content: "\f0c6"; -} -.icon-save:before { - content: "\f0c7"; -} -.icon-sign-blank:before { - content: "\f0c8"; -} -.icon-reorder:before { - content: "\f0c9"; -} -.icon-list-ul:before { - content: "\f0ca"; -} -.icon-list-ol:before { - content: "\f0cb"; -} -.icon-strikethrough:before { - content: "\f0cc"; -} -.icon-underline:before { - content: "\f0cd"; -} -.icon-table:before { - content: "\f0ce"; -} -.icon-magic:before { - content: "\f0d0"; -} -.icon-truck:before { - content: "\f0d1"; -} -.icon-pinterest:before { - content: "\f0d2"; -} -.icon-pinterest-sign:before { - content: "\f0d3"; -} -.icon-google-plus-sign:before { - content: "\f0d4"; -} -.icon-google-plus:before { - content: "\f0d5"; -} -.icon-money:before { - content: "\f0d6"; -} -.icon-caret-down:before { - content: "\f0d7"; -} -.icon-caret-up:before { - content: "\f0d8"; -} -.icon-caret-left:before { - content: "\f0d9"; -} -.icon-caret-right:before { - content: "\f0da"; -} -.icon-columns:before { - content: "\f0db"; -} -.icon-sort:before { - content: "\f0dc"; -} -.icon-sort-down:before { - content: "\f0dd"; -} -.icon-sort-up:before { - content: "\f0de"; -} -.icon-envelope-alt:before { - content: "\f0e0"; -} -.icon-linkedin:before { - content: "\f0e1"; -} -.icon-undo:before { - content: "\f0e2"; -} -.icon-legal:before { - content: "\f0e3"; -} -.icon-dashboard:before { - content: "\f0e4"; -} -.icon-comment-alt:before { - content: "\f0e5"; -} -.icon-comments-alt:before { - content: "\f0e6"; -} -.icon-bolt:before { - content: "\f0e7"; -} -.icon-sitemap:before { - content: "\f0e8"; -} -.icon-umbrella:before { - content: "\f0e9"; -} -.icon-paste:before { - content: "\f0ea"; -} -.icon-lightbulb:before { - content: "\f0eb"; -} -.icon-exchange:before { - content: "\f0ec"; -} -.icon-cloud-download:before { - content: "\f0ed"; -} -.icon-cloud-upload:before { - content: "\f0ee"; -} -.icon-user-md:before { - content: "\f0f0"; -} -.icon-stethoscope:before { - content: "\f0f1"; -} -.icon-suitcase:before { - content: "\f0f2"; -} -.icon-bell-alt:before { - content: "\f0f3"; -} -.icon-coffee:before { - content: "\f0f4"; -} -.icon-food:before { - content: "\f0f5"; -} -.icon-file-alt:before { - content: "\f0f6"; -} -.icon-building:before { - content: "\f0f7"; -} -.icon-hospital:before { - content: "\f0f8"; -} -.icon-ambulance:before { - content: "\f0f9"; -} -.icon-medkit:before { - content: "\f0fa"; -} -.icon-fighter-jet:before { - content: "\f0fb"; -} -.icon-beer:before { - content: "\f0fc"; -} -.icon-h-sign:before { - content: "\f0fd"; -} -.icon-plus-sign-alt:before { - content: "\f0fe"; -} -.icon-double-angle-left:before { - content: "\f100"; -} -.icon-double-angle-right:before { - content: "\f101"; -} -.icon-double-angle-up:before { - content: "\f102"; -} -.icon-double-angle-down:before { - content: "\f103"; -} -.icon-angle-left:before { - content: "\f104"; -} -.icon-angle-right:before { - content: "\f105"; -} -.icon-angle-up:before { - content: "\f106"; -} -.icon-angle-down:before { - content: "\f107"; -} -.icon-desktop:before { - content: "\f108"; -} -.icon-laptop:before { - content: "\f109"; -} -.icon-tablet:before { - content: "\f10a"; -} -.icon-mobile-phone:before { - content: "\f10b"; -} -.icon-circle-blank:before { - content: "\f10c"; -} -.icon-quote-left:before { - content: "\f10d"; -} -.icon-quote-right:before { - content: "\f10e"; -} -.icon-spinner:before { - content: "\f110"; -} -.icon-circle:before { - content: "\f111"; -} -.icon-reply:before { - content: "\f112"; -} -.icon-github-alt:before { - content: "\f113"; -} -.icon-folder-close-alt:before { - content: "\f114"; -} -.icon-folder-open-alt:before { - content: "\f115"; -} [class^="icon-"], [class*=" icon-"] { display: inline-block; @@ -7963,45 +7141,39 @@ h4 small { .context-info.editing .module-content { margin-top: 0; } -.hero { - background: url("../../../base/images/background-tile.png"); +.homepage [role=main] { padding: 20px 0; min-height: 0; } -.hero > .container { +.homepage .row { position: relative; - padding-bottom: 0; } -.hero .search-giant { - margin-bottom: 10px; -} -.hero .search-giant input { - border-color: #003f52; -} -.hero .page-heading { - font-size: 18px; - margin-bottom: 0; -} -.hero .module-dark { +.homepage .module-search { padding: 5px; - margin-bottom: 0; + margin: 0; color: #ffffff; background: #ffffff; } -.hero .module-dark .module-content { +.homepage .module-search .search-giant { + margin-bottom: 10px; +} +.homepage .module-search .search-giant input { + border-color: #003f52; +} +.homepage .module-search .module-content { -webkit-border-radius: 3px 3px 0 0; -moz-border-radius: 3px 3px 0 0; border-radius: 3px 3px 0 0; background-color: #e73892; border-bottom: none; } -.hero .module-dark .module-content .heading { +.homepage .module-search .module-content .heading { margin-top: 0; margin-bottom: 7px; font-size: 24px; line-height: 40px; } -.hero .tags { +.homepage .module-search .tags { *zoom: 1; padding: 5px 10px 10px 10px; background-color: #d31979; @@ -8009,69 +7181,77 @@ h4 small { -moz-border-radius: 0 0 3px 3px; border-radius: 0 0 3px 3px; } -.hero .tags:before, -.hero .tags:after { +.homepage .module-search .tags:before, +.homepage .module-search .tags:after { display: table; content: ""; line-height: 0; } -.hero .tags:after { +.homepage .module-search .tags:after { clear: both; } -.hero .tags h3, -.hero .tags .tag { +.homepage .module-search .tags h3, +.homepage .module-search .tags .tag { display: block; float: left; margin: 5px 10px 0 0; } -.hero .tags h3 { +.homepage .module-search .tags h3 { font-size: 14px; line-height: 20px; padding: 2px 8px; } -.hero-primary, -.hero-secondary { +.homepage .group-list { + margin: 0; +} +.homepage .box .inner { + padding: 20px 25px; +} +.homepage .stats h3 { + margin: 0 0 10px 0; +} +.homepage .stats ul { + margin: 0; + list-style: none; + *zoom: 1; +} +.homepage .stats ul:before, +.homepage .stats ul:after { + display: table; + content: ""; + line-height: 0; +} +.homepage .stats ul:after { + clear: both; +} +.homepage .stats ul li { float: left; - margin-left: 20px; - width: 460px; + width: 25%; + font-weight: 300; } -.hero-primary { - margin-left: 0; - margin-bottom: 0; +.homepage .stats ul li a { + display: block; } -.hero-secondary { +.homepage .stats ul li a b { + display: block; + font-size: 35px; + line-height: 1.5; +} +.homepage .stats ul li a:hover { + text-decoration: none; +} +.homepage.layout-1 .row1 .col2 { position: absolute; bottom: 0; right: 0; } -.hero-secondary .hero-secondary-inner { +.homepage.layout-1 .row1 .col2 .module-search { bottom: 0; left: 0; right: 0; } -.main.homepage { - padding-top: 20px; - padding-bottom: 20px; - border-top: 1px solid #cccccc; -} -.main.homepage .module-heading .media-image { - margin-right: 15px; - max-height: 53px; - -webkit-box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); - -moz-box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); - box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); -} -.main.homepage .group-listing .box { - min-height: 275px; -} -.main.homepage .group-list { - margin-bottom: 0; -} -.main.homepage .group-list .dataset-content { - min-height: 70px; -} -.main.homepage .box .module { - margin-top: 0; +.homepage.layout-2 .stats { + margin-top: 20px; } .account-masthead { *zoom: 1; @@ -8920,6 +8100,11 @@ iframe { padding-bottom: 20px !important; margin-bottom: 0 !important; } +.ie8 .lang-dropdown, +.ie7 .lang-dropdown { + position: relative !important; + top: -90px !important; +} .ie7 .alert { position: relative; } diff --git a/ckan/public/base/css/green.css b/ckan/public/base/css/green.css index 1cc58fa82dc..83bbcd5eefa 100644 --- a/ckan/public/base/css/green.css +++ b/ckan/public/base/css/green.css @@ -2055,6 +2055,14 @@ table th[class*="span"], .open > .dropdown-menu { display: block; } +.dropdown-backdrop { + position: fixed; + left: 0; + right: 0; + bottom: 0; + top: 0; + z-index: 990; +} .pull-right > .dropdown-menu { right: 0; left: auto; @@ -5572,6 +5580,12 @@ textarea { .control-large input { height: 41px; } +.control-required { + color: #c6898b; +} +.form-actions .control-required-message { + float: left; +} .form-actions { background: none; margin-left: -25px; @@ -6904,842 +6918,6 @@ h4 small { height: 35px; background-position: -320px -62px; } -/* This is a modified version of the font-awesome core less document. */ -@font-face { - font-family: 'FontAwesome'; - src: url('../../../base/vendor/font-awesome/font/fontawesome-webfont.eot?v=3.0.1'); - src: url('../../../base/vendor/font-awesome/font/fontawesome-webfont.eot?#iefix&v=3.0.1') format('embedded-opentype'), url('../../../base/vendor/font-awesome/font/fontawesome-webfont.woff?v=3.0.1') format('woff'), url('../../../base/vendor/font-awesome/font/fontawesome-webfont.ttf?v=3.0.1') format('truetype'); - font-weight: normal; - font-style: normal; -} -[class^="icon-"], -[class*=" icon-"] { - font-family: FontAwesome; - font-weight: normal; - font-style: normal; - text-decoration: inherit; - -webkit-font-smoothing: antialiased; - display: inline; - width: auto; - height: auto; - line-height: normal; - vertical-align: baseline; - background-image: none; - background-position: 0% 0%; - background-repeat: repeat; - margin-top: 0; -} -.icon-spin { - display: inline-block; - -moz-animation: spin 2s infinite linear; - -o-animation: spin 2s infinite linear; - -webkit-animation: spin 2s infinite linear; - animation: spin 2s infinite linear; -} -@-moz-keyframes spin { - 0% { - -moz-transform: rotate(0deg); - } - 100% { - -moz-transform: rotate(359deg); - } -} -@-webkit-keyframes spin { - 0% { - -webkit-transform: rotate(0deg); - } - 100% { - -webkit-transform: rotate(359deg); - } -} -@-o-keyframes spin { - 0% { - -o-transform: rotate(0deg); - } - 100% { - -o-transform: rotate(359deg); - } -} -@-ms-keyframes spin { - 0% { - -ms-transform: rotate(0deg); - } - 100% { - -ms-transform: rotate(359deg); - } -} -@keyframes spin { - 0% { - transform: rotate(0deg); - } - 100% { - transform: rotate(359deg); - } -} -@-moz-document url-prefix() { - .icon-spin { - height: .9em; - } - .btn .icon-spin { - height: auto; - } - .icon-spin.icon-large { - height: 1.25em; - } - .btn .icon-spin.icon-large { - height: .75em; - } -} -/* Font Awesome uses the Unicode Private Use Area (PUA) to ensure screen - readers do not read off random characters that represent icons */ -.icon-glass:before { - content: "\f000"; -} -.icon-music:before { - content: "\f001"; -} -.icon-search:before { - content: "\f002"; -} -.icon-envelope:before { - content: "\f003"; -} -.icon-heart:before { - content: "\f004"; -} -.icon-star:before { - content: "\f005"; -} -.icon-star-empty:before { - content: "\f006"; -} -.icon-user:before { - content: "\f007"; -} -.icon-film:before { - content: "\f008"; -} -.icon-th-large:before { - content: "\f009"; -} -.icon-th:before { - content: "\f00a"; -} -.icon-th-list:before { - content: "\f00b"; -} -.icon-ok:before { - content: "\f00c"; -} -.icon-remove:before { - content: "\f00d"; -} -.icon-zoom-in:before { - content: "\f00e"; -} -.icon-zoom-out:before { - content: "\f010"; -} -.icon-off:before { - content: "\f011"; -} -.icon-signal:before { - content: "\f012"; -} -.icon-cog:before { - content: "\f013"; -} -.icon-trash:before { - content: "\f014"; -} -.icon-home:before { - content: "\f015"; -} -.icon-file:before { - content: "\f016"; -} -.icon-time:before { - content: "\f017"; -} -.icon-road:before { - content: "\f018"; -} -.icon-download-alt:before { - content: "\f019"; -} -.icon-download:before { - content: "\f01a"; -} -.icon-upload:before { - content: "\f01b"; -} -.icon-inbox:before { - content: "\f01c"; -} -.icon-play-circle:before { - content: "\f01d"; -} -.icon-repeat:before { - content: "\f01e"; -} -/* \f020 doesn't work in Safari. all shifted one down */ -.icon-refresh:before { - content: "\f021"; -} -.icon-list-alt:before { - content: "\f022"; -} -.icon-lock:before { - content: "\f023"; -} -.icon-flag:before { - content: "\f024"; -} -.icon-headphones:before { - content: "\f025"; -} -.icon-volume-off:before { - content: "\f026"; -} -.icon-volume-down:before { - content: "\f027"; -} -.icon-volume-up:before { - content: "\f028"; -} -.icon-qrcode:before { - content: "\f029"; -} -.icon-barcode:before { - content: "\f02a"; -} -.icon-tag:before { - content: "\f02b"; -} -.icon-tags:before { - content: "\f02c"; -} -.icon-book:before { - content: "\f02d"; -} -.icon-bookmark:before { - content: "\f02e"; -} -.icon-print:before { - content: "\f02f"; -} -.icon-camera:before { - content: "\f030"; -} -.icon-font:before { - content: "\f031"; -} -.icon-bold:before { - content: "\f032"; -} -.icon-italic:before { - content: "\f033"; -} -.icon-text-height:before { - content: "\f034"; -} -.icon-text-width:before { - content: "\f035"; -} -.icon-align-left:before { - content: "\f036"; -} -.icon-align-center:before { - content: "\f037"; -} -.icon-align-right:before { - content: "\f038"; -} -.icon-align-justify:before { - content: "\f039"; -} -.icon-list:before { - content: "\f03a"; -} -.icon-indent-left:before { - content: "\f03b"; -} -.icon-indent-right:before { - content: "\f03c"; -} -.icon-facetime-video:before { - content: "\f03d"; -} -.icon-picture:before { - content: "\f03e"; -} -.icon-pencil:before { - content: "\f040"; -} -.icon-map-marker:before { - content: "\f041"; -} -.icon-adjust:before { - content: "\f042"; -} -.icon-tint:before { - content: "\f043"; -} -.icon-edit:before { - content: "\f044"; -} -.icon-share:before { - content: "\f045"; -} -.icon-check:before { - content: "\f046"; -} -.icon-move:before { - content: "\f047"; -} -.icon-step-backward:before { - content: "\f048"; -} -.icon-fast-backward:before { - content: "\f049"; -} -.icon-backward:before { - content: "\f04a"; -} -.icon-play:before { - content: "\f04b"; -} -.icon-pause:before { - content: "\f04c"; -} -.icon-stop:before { - content: "\f04d"; -} -.icon-forward:before { - content: "\f04e"; -} -.icon-fast-forward:before { - content: "\f050"; -} -.icon-step-forward:before { - content: "\f051"; -} -.icon-eject:before { - content: "\f052"; -} -.icon-chevron-left:before { - content: "\f053"; -} -.icon-chevron-right:before { - content: "\f054"; -} -.icon-plus-sign:before { - content: "\f055"; -} -.icon-minus-sign:before { - content: "\f056"; -} -.icon-remove-sign:before { - content: "\f057"; -} -.icon-ok-sign:before { - content: "\f058"; -} -.icon-question-sign:before { - content: "\f059"; -} -.icon-info-sign:before { - content: "\f05a"; -} -.icon-screenshot:before { - content: "\f05b"; -} -.icon-remove-circle:before { - content: "\f05c"; -} -.icon-ok-circle:before { - content: "\f05d"; -} -.icon-ban-circle:before { - content: "\f05e"; -} -.icon-arrow-left:before { - content: "\f060"; -} -.icon-arrow-right:before { - content: "\f061"; -} -.icon-arrow-up:before { - content: "\f062"; -} -.icon-arrow-down:before { - content: "\f063"; -} -.icon-share-alt:before { - content: "\f064"; -} -.icon-resize-full:before { - content: "\f065"; -} -.icon-resize-small:before { - content: "\f066"; -} -.icon-plus:before { - content: "\f067"; -} -.icon-minus:before { - content: "\f068"; -} -.icon-asterisk:before { - content: "\f069"; -} -.icon-exclamation-sign:before { - content: "\f06a"; -} -.icon-gift:before { - content: "\f06b"; -} -.icon-leaf:before { - content: "\f06c"; -} -.icon-fire:before { - content: "\f06d"; -} -.icon-eye-open:before { - content: "\f06e"; -} -.icon-eye-close:before { - content: "\f070"; -} -.icon-warning-sign:before { - content: "\f071"; -} -.icon-plane:before { - content: "\f072"; -} -.icon-calendar:before { - content: "\f073"; -} -.icon-random:before { - content: "\f074"; -} -.icon-comment:before { - content: "\f075"; -} -.icon-magnet:before { - content: "\f076"; -} -.icon-chevron-up:before { - content: "\f077"; -} -.icon-chevron-down:before { - content: "\f078"; -} -.icon-retweet:before { - content: "\f079"; -} -.icon-shopping-cart:before { - content: "\f07a"; -} -.icon-folder-close:before { - content: "\f07b"; -} -.icon-folder-open:before { - content: "\f07c"; -} -.icon-resize-vertical:before { - content: "\f07d"; -} -.icon-resize-horizontal:before { - content: "\f07e"; -} -.icon-bar-chart:before { - content: "\f080"; -} -.icon-twitter-sign:before { - content: "\f081"; -} -.icon-facebook-sign:before { - content: "\f082"; -} -.icon-camera-retro:before { - content: "\f083"; -} -.icon-key:before { - content: "\f084"; -} -.icon-cogs:before { - content: "\f085"; -} -.icon-comments:before { - content: "\f086"; -} -.icon-thumbs-up:before { - content: "\f087"; -} -.icon-thumbs-down:before { - content: "\f088"; -} -.icon-star-half:before { - content: "\f089"; -} -.icon-heart-empty:before { - content: "\f08a"; -} -.icon-signout:before { - content: "\f08b"; -} -.icon-linkedin-sign:before { - content: "\f08c"; -} -.icon-pushpin:before { - content: "\f08d"; -} -.icon-external-link:before { - content: "\f08e"; -} -.icon-signin:before { - content: "\f090"; -} -.icon-trophy:before { - content: "\f091"; -} -.icon-github-sign:before { - content: "\f092"; -} -.icon-upload-alt:before { - content: "\f093"; -} -.icon-lemon:before { - content: "\f094"; -} -.icon-phone:before { - content: "\f095"; -} -.icon-check-empty:before { - content: "\f096"; -} -.icon-bookmark-empty:before { - content: "\f097"; -} -.icon-phone-sign:before { - content: "\f098"; -} -.icon-twitter:before { - content: "\f099"; -} -.icon-facebook:before { - content: "\f09a"; -} -.icon-github:before { - content: "\f09b"; -} -.icon-unlock:before { - content: "\f09c"; -} -.icon-credit-card:before { - content: "\f09d"; -} -.icon-rss:before { - content: "\f09e"; -} -.icon-hdd:before { - content: "\f0a0"; -} -.icon-bullhorn:before { - content: "\f0a1"; -} -.icon-bell:before { - content: "\f0a2"; -} -.icon-certificate:before { - content: "\f0a3"; -} -.icon-hand-right:before { - content: "\f0a4"; -} -.icon-hand-left:before { - content: "\f0a5"; -} -.icon-hand-up:before { - content: "\f0a6"; -} -.icon-hand-down:before { - content: "\f0a7"; -} -.icon-circle-arrow-left:before { - content: "\f0a8"; -} -.icon-circle-arrow-right:before { - content: "\f0a9"; -} -.icon-circle-arrow-up:before { - content: "\f0aa"; -} -.icon-circle-arrow-down:before { - content: "\f0ab"; -} -.icon-globe:before { - content: "\f0ac"; -} -.icon-wrench:before { - content: "\f0ad"; -} -.icon-tasks:before { - content: "\f0ae"; -} -.icon-filter:before { - content: "\f0b0"; -} -.icon-briefcase:before { - content: "\f0b1"; -} -.icon-fullscreen:before { - content: "\f0b2"; -} -.icon-group:before { - content: "\f0c0"; -} -.icon-link:before { - content: "\f0c1"; -} -.icon-cloud:before { - content: "\f0c2"; -} -.icon-beaker:before { - content: "\f0c3"; -} -.icon-cut:before { - content: "\f0c4"; -} -.icon-copy:before { - content: "\f0c5"; -} -.icon-paper-clip:before { - content: "\f0c6"; -} -.icon-save:before { - content: "\f0c7"; -} -.icon-sign-blank:before { - content: "\f0c8"; -} -.icon-reorder:before { - content: "\f0c9"; -} -.icon-list-ul:before { - content: "\f0ca"; -} -.icon-list-ol:before { - content: "\f0cb"; -} -.icon-strikethrough:before { - content: "\f0cc"; -} -.icon-underline:before { - content: "\f0cd"; -} -.icon-table:before { - content: "\f0ce"; -} -.icon-magic:before { - content: "\f0d0"; -} -.icon-truck:before { - content: "\f0d1"; -} -.icon-pinterest:before { - content: "\f0d2"; -} -.icon-pinterest-sign:before { - content: "\f0d3"; -} -.icon-google-plus-sign:before { - content: "\f0d4"; -} -.icon-google-plus:before { - content: "\f0d5"; -} -.icon-money:before { - content: "\f0d6"; -} -.icon-caret-down:before { - content: "\f0d7"; -} -.icon-caret-up:before { - content: "\f0d8"; -} -.icon-caret-left:before { - content: "\f0d9"; -} -.icon-caret-right:before { - content: "\f0da"; -} -.icon-columns:before { - content: "\f0db"; -} -.icon-sort:before { - content: "\f0dc"; -} -.icon-sort-down:before { - content: "\f0dd"; -} -.icon-sort-up:before { - content: "\f0de"; -} -.icon-envelope-alt:before { - content: "\f0e0"; -} -.icon-linkedin:before { - content: "\f0e1"; -} -.icon-undo:before { - content: "\f0e2"; -} -.icon-legal:before { - content: "\f0e3"; -} -.icon-dashboard:before { - content: "\f0e4"; -} -.icon-comment-alt:before { - content: "\f0e5"; -} -.icon-comments-alt:before { - content: "\f0e6"; -} -.icon-bolt:before { - content: "\f0e7"; -} -.icon-sitemap:before { - content: "\f0e8"; -} -.icon-umbrella:before { - content: "\f0e9"; -} -.icon-paste:before { - content: "\f0ea"; -} -.icon-lightbulb:before { - content: "\f0eb"; -} -.icon-exchange:before { - content: "\f0ec"; -} -.icon-cloud-download:before { - content: "\f0ed"; -} -.icon-cloud-upload:before { - content: "\f0ee"; -} -.icon-user-md:before { - content: "\f0f0"; -} -.icon-stethoscope:before { - content: "\f0f1"; -} -.icon-suitcase:before { - content: "\f0f2"; -} -.icon-bell-alt:before { - content: "\f0f3"; -} -.icon-coffee:before { - content: "\f0f4"; -} -.icon-food:before { - content: "\f0f5"; -} -.icon-file-alt:before { - content: "\f0f6"; -} -.icon-building:before { - content: "\f0f7"; -} -.icon-hospital:before { - content: "\f0f8"; -} -.icon-ambulance:before { - content: "\f0f9"; -} -.icon-medkit:before { - content: "\f0fa"; -} -.icon-fighter-jet:before { - content: "\f0fb"; -} -.icon-beer:before { - content: "\f0fc"; -} -.icon-h-sign:before { - content: "\f0fd"; -} -.icon-plus-sign-alt:before { - content: "\f0fe"; -} -.icon-double-angle-left:before { - content: "\f100"; -} -.icon-double-angle-right:before { - content: "\f101"; -} -.icon-double-angle-up:before { - content: "\f102"; -} -.icon-double-angle-down:before { - content: "\f103"; -} -.icon-angle-left:before { - content: "\f104"; -} -.icon-angle-right:before { - content: "\f105"; -} -.icon-angle-up:before { - content: "\f106"; -} -.icon-angle-down:before { - content: "\f107"; -} -.icon-desktop:before { - content: "\f108"; -} -.icon-laptop:before { - content: "\f109"; -} -.icon-tablet:before { - content: "\f10a"; -} -.icon-mobile-phone:before { - content: "\f10b"; -} -.icon-circle-blank:before { - content: "\f10c"; -} -.icon-quote-left:before { - content: "\f10d"; -} -.icon-quote-right:before { - content: "\f10e"; -} -.icon-spinner:before { - content: "\f110"; -} -.icon-circle:before { - content: "\f111"; -} -.icon-reply:before { - content: "\f112"; -} -.icon-github-alt:before { - content: "\f113"; -} -.icon-folder-close-alt:before { - content: "\f114"; -} -.icon-folder-open-alt:before { - content: "\f115"; -} [class^="icon-"], [class*=" icon-"] { display: inline-block; @@ -7963,45 +7141,39 @@ h4 small { .context-info.editing .module-content { margin-top: 0; } -.hero { - background: url("../../../base/images/background-tile.png"); +.homepage [role=main] { padding: 20px 0; min-height: 0; } -.hero > .container { +.homepage .row { position: relative; - padding-bottom: 0; } -.hero .search-giant { - margin-bottom: 10px; -} -.hero .search-giant input { - border-color: #003f52; -} -.hero .page-heading { - font-size: 18px; - margin-bottom: 0; -} -.hero .module-dark { +.homepage .module-search { padding: 5px; - margin-bottom: 0; + margin: 0; color: #ffffff; background: #ffffff; } -.hero .module-dark .module-content { +.homepage .module-search .search-giant { + margin-bottom: 10px; +} +.homepage .module-search .search-giant input { + border-color: #003f52; +} +.homepage .module-search .module-content { -webkit-border-radius: 3px 3px 0 0; -moz-border-radius: 3px 3px 0 0; border-radius: 3px 3px 0 0; background-color: #2f9b45; border-bottom: none; } -.hero .module-dark .module-content .heading { +.homepage .module-search .module-content .heading { margin-top: 0; margin-bottom: 7px; font-size: 24px; line-height: 40px; } -.hero .tags { +.homepage .module-search .tags { *zoom: 1; padding: 5px 10px 10px 10px; background-color: #237434; @@ -8009,69 +7181,77 @@ h4 small { -moz-border-radius: 0 0 3px 3px; border-radius: 0 0 3px 3px; } -.hero .tags:before, -.hero .tags:after { +.homepage .module-search .tags:before, +.homepage .module-search .tags:after { display: table; content: ""; line-height: 0; } -.hero .tags:after { +.homepage .module-search .tags:after { clear: both; } -.hero .tags h3, -.hero .tags .tag { +.homepage .module-search .tags h3, +.homepage .module-search .tags .tag { display: block; float: left; margin: 5px 10px 0 0; } -.hero .tags h3 { +.homepage .module-search .tags h3 { font-size: 14px; line-height: 20px; padding: 2px 8px; } -.hero-primary, -.hero-secondary { +.homepage .group-list { + margin: 0; +} +.homepage .box .inner { + padding: 20px 25px; +} +.homepage .stats h3 { + margin: 0 0 10px 0; +} +.homepage .stats ul { + margin: 0; + list-style: none; + *zoom: 1; +} +.homepage .stats ul:before, +.homepage .stats ul:after { + display: table; + content: ""; + line-height: 0; +} +.homepage .stats ul:after { + clear: both; +} +.homepage .stats ul li { float: left; - margin-left: 20px; - width: 460px; + width: 25%; + font-weight: 300; } -.hero-primary { - margin-left: 0; - margin-bottom: 0; +.homepage .stats ul li a { + display: block; } -.hero-secondary { +.homepage .stats ul li a b { + display: block; + font-size: 35px; + line-height: 1.5; +} +.homepage .stats ul li a:hover { + text-decoration: none; +} +.homepage.layout-1 .row1 .col2 { position: absolute; bottom: 0; right: 0; } -.hero-secondary .hero-secondary-inner { +.homepage.layout-1 .row1 .col2 .module-search { bottom: 0; left: 0; right: 0; } -.main.homepage { - padding-top: 20px; - padding-bottom: 20px; - border-top: 1px solid #cccccc; -} -.main.homepage .module-heading .media-image { - margin-right: 15px; - max-height: 53px; - -webkit-box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); - -moz-box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); - box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); -} -.main.homepage .group-listing .box { - min-height: 275px; -} -.main.homepage .group-list { - margin-bottom: 0; -} -.main.homepage .group-list .dataset-content { - min-height: 70px; -} -.main.homepage .box .module { - margin-top: 0; +.homepage.layout-2 .stats { + margin-top: 20px; } .account-masthead { *zoom: 1; @@ -8920,6 +8100,11 @@ iframe { padding-bottom: 20px !important; margin-bottom: 0 !important; } +.ie8 .lang-dropdown, +.ie7 .lang-dropdown { + position: relative !important; + top: -90px !important; +} .ie7 .alert { position: relative; } diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index 2ed06e1eb12..77b93ef3cfc 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -7141,45 +7141,39 @@ h4 small { .context-info.editing .module-content { margin-top: 0; } -.hero { - background: url("../../../base/images/background-tile.png"); +.homepage [role=main] { padding: 20px 0; min-height: 0; } -.hero > .container { +.homepage .row { position: relative; - padding-bottom: 0; -} -.hero .search-giant { - margin-bottom: 10px; -} -.hero .search-giant input { - border-color: #003f52; } -.hero .page-heading { - font-size: 18px; - margin-bottom: 0; -} -.hero .module-dark { +.homepage .module-search { padding: 5px; - margin-bottom: 0; + margin: 0; color: #ffffff; background: #ffffff; } -.hero .module-dark .module-content { +.homepage .module-search .search-giant { + margin-bottom: 10px; +} +.homepage .module-search .search-giant input { + border-color: #003f52; +} +.homepage .module-search .module-content { -webkit-border-radius: 3px 3px 0 0; -moz-border-radius: 3px 3px 0 0; border-radius: 3px 3px 0 0; background-color: #005d7a; border-bottom: none; } -.hero .module-dark .module-content .heading { +.homepage .module-search .module-content .heading { margin-top: 0; margin-bottom: 7px; font-size: 24px; line-height: 40px; } -.hero .tags { +.homepage .module-search .tags { *zoom: 1; padding: 5px 10px 10px 10px; background-color: #003647; @@ -7187,69 +7181,77 @@ h4 small { -moz-border-radius: 0 0 3px 3px; border-radius: 0 0 3px 3px; } -.hero .tags:before, -.hero .tags:after { +.homepage .module-search .tags:before, +.homepage .module-search .tags:after { display: table; content: ""; line-height: 0; } -.hero .tags:after { +.homepage .module-search .tags:after { clear: both; } -.hero .tags h3, -.hero .tags .tag { +.homepage .module-search .tags h3, +.homepage .module-search .tags .tag { display: block; float: left; margin: 5px 10px 0 0; } -.hero .tags h3 { +.homepage .module-search .tags h3 { font-size: 14px; line-height: 20px; padding: 2px 8px; } -.hero-primary, -.hero-secondary { +.homepage .group-list { + margin: 0; +} +.homepage .box .inner { + padding: 20px 25px; +} +.homepage .stats h3 { + margin: 0 0 10px 0; +} +.homepage .stats ul { + margin: 0; + list-style: none; + *zoom: 1; +} +.homepage .stats ul:before, +.homepage .stats ul:after { + display: table; + content: ""; + line-height: 0; +} +.homepage .stats ul:after { + clear: both; +} +.homepage .stats ul li { float: left; - margin-left: 20px; - width: 460px; + width: 25%; + font-weight: 300; } -.hero-primary { - margin-left: 0; - margin-bottom: 0; +.homepage .stats ul li a { + display: block; +} +.homepage .stats ul li a b { + display: block; + font-size: 35px; + line-height: 1.5; } -.hero-secondary { +.homepage .stats ul li a:hover { + text-decoration: none; +} +.homepage.layout-1 .row1 .col2 { position: absolute; bottom: 0; right: 0; } -.hero-secondary .hero-secondary-inner { +.homepage.layout-1 .row1 .col2 .module-search { bottom: 0; left: 0; right: 0; } -.main.homepage { - padding-top: 20px; - padding-bottom: 20px; - border-top: 1px solid #cccccc; -} -.main.homepage .module-heading .media-image { - margin-right: 15px; - max-height: 53px; - -webkit-box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); - -moz-box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); - box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); -} -.main.homepage .group-listing .box { - min-height: 275px; -} -.main.homepage .group-list { - margin-bottom: 0; -} -.main.homepage .group-list .dataset-content { - min-height: 70px; -} -.main.homepage .box .module { - margin-top: 0; +.homepage.layout-2 .stats { + margin-top: 20px; } .account-masthead { *zoom: 1; @@ -7925,9 +7927,9 @@ h4 small { -webkit-border-radius: 100px; -moz-border-radius: 100px; border-radius: 100px; - -webkit-box-shadow: inset 0 1px 2 x rgba(0, 0, 0, 0.2); - -moz-box-shadow: inset 0 1px 2 x rgba(0, 0, 0, 0.2); - box-shadow: inset 0 1px 2 x rgba(0, 0, 0, 0.2); + -webkit-box-shadow: inset 0 1px 2x rgba(0, 0, 0, 0.2); + -moz-box-shadow: inset 0 1px 2x rgba(0, 0, 0, 0.2); + box-shadow: inset 0 1px 2x rgba(0, 0, 0, 0.2); } .popover-followee .nav li a:hover i { background-color: #000; diff --git a/ckan/public/base/css/maroon.css b/ckan/public/base/css/maroon.css index c052d27ea74..585ed6129c2 100644 --- a/ckan/public/base/css/maroon.css +++ b/ckan/public/base/css/maroon.css @@ -2055,6 +2055,14 @@ table th[class*="span"], .open > .dropdown-menu { display: block; } +.dropdown-backdrop { + position: fixed; + left: 0; + right: 0; + bottom: 0; + top: 0; + z-index: 990; +} .pull-right > .dropdown-menu { right: 0; left: auto; @@ -5572,6 +5580,12 @@ textarea { .control-large input { height: 41px; } +.control-required { + color: #c6898b; +} +.form-actions .control-required-message { + float: left; +} .form-actions { background: none; margin-left: -25px; @@ -6904,842 +6918,6 @@ h4 small { height: 35px; background-position: -320px -62px; } -/* This is a modified version of the font-awesome core less document. */ -@font-face { - font-family: 'FontAwesome'; - src: url('../../../base/vendor/font-awesome/font/fontawesome-webfont.eot?v=3.0.1'); - src: url('../../../base/vendor/font-awesome/font/fontawesome-webfont.eot?#iefix&v=3.0.1') format('embedded-opentype'), url('../../../base/vendor/font-awesome/font/fontawesome-webfont.woff?v=3.0.1') format('woff'), url('../../../base/vendor/font-awesome/font/fontawesome-webfont.ttf?v=3.0.1') format('truetype'); - font-weight: normal; - font-style: normal; -} -[class^="icon-"], -[class*=" icon-"] { - font-family: FontAwesome; - font-weight: normal; - font-style: normal; - text-decoration: inherit; - -webkit-font-smoothing: antialiased; - display: inline; - width: auto; - height: auto; - line-height: normal; - vertical-align: baseline; - background-image: none; - background-position: 0% 0%; - background-repeat: repeat; - margin-top: 0; -} -.icon-spin { - display: inline-block; - -moz-animation: spin 2s infinite linear; - -o-animation: spin 2s infinite linear; - -webkit-animation: spin 2s infinite linear; - animation: spin 2s infinite linear; -} -@-moz-keyframes spin { - 0% { - -moz-transform: rotate(0deg); - } - 100% { - -moz-transform: rotate(359deg); - } -} -@-webkit-keyframes spin { - 0% { - -webkit-transform: rotate(0deg); - } - 100% { - -webkit-transform: rotate(359deg); - } -} -@-o-keyframes spin { - 0% { - -o-transform: rotate(0deg); - } - 100% { - -o-transform: rotate(359deg); - } -} -@-ms-keyframes spin { - 0% { - -ms-transform: rotate(0deg); - } - 100% { - -ms-transform: rotate(359deg); - } -} -@keyframes spin { - 0% { - transform: rotate(0deg); - } - 100% { - transform: rotate(359deg); - } -} -@-moz-document url-prefix() { - .icon-spin { - height: .9em; - } - .btn .icon-spin { - height: auto; - } - .icon-spin.icon-large { - height: 1.25em; - } - .btn .icon-spin.icon-large { - height: .75em; - } -} -/* Font Awesome uses the Unicode Private Use Area (PUA) to ensure screen - readers do not read off random characters that represent icons */ -.icon-glass:before { - content: "\f000"; -} -.icon-music:before { - content: "\f001"; -} -.icon-search:before { - content: "\f002"; -} -.icon-envelope:before { - content: "\f003"; -} -.icon-heart:before { - content: "\f004"; -} -.icon-star:before { - content: "\f005"; -} -.icon-star-empty:before { - content: "\f006"; -} -.icon-user:before { - content: "\f007"; -} -.icon-film:before { - content: "\f008"; -} -.icon-th-large:before { - content: "\f009"; -} -.icon-th:before { - content: "\f00a"; -} -.icon-th-list:before { - content: "\f00b"; -} -.icon-ok:before { - content: "\f00c"; -} -.icon-remove:before { - content: "\f00d"; -} -.icon-zoom-in:before { - content: "\f00e"; -} -.icon-zoom-out:before { - content: "\f010"; -} -.icon-off:before { - content: "\f011"; -} -.icon-signal:before { - content: "\f012"; -} -.icon-cog:before { - content: "\f013"; -} -.icon-trash:before { - content: "\f014"; -} -.icon-home:before { - content: "\f015"; -} -.icon-file:before { - content: "\f016"; -} -.icon-time:before { - content: "\f017"; -} -.icon-road:before { - content: "\f018"; -} -.icon-download-alt:before { - content: "\f019"; -} -.icon-download:before { - content: "\f01a"; -} -.icon-upload:before { - content: "\f01b"; -} -.icon-inbox:before { - content: "\f01c"; -} -.icon-play-circle:before { - content: "\f01d"; -} -.icon-repeat:before { - content: "\f01e"; -} -/* \f020 doesn't work in Safari. all shifted one down */ -.icon-refresh:before { - content: "\f021"; -} -.icon-list-alt:before { - content: "\f022"; -} -.icon-lock:before { - content: "\f023"; -} -.icon-flag:before { - content: "\f024"; -} -.icon-headphones:before { - content: "\f025"; -} -.icon-volume-off:before { - content: "\f026"; -} -.icon-volume-down:before { - content: "\f027"; -} -.icon-volume-up:before { - content: "\f028"; -} -.icon-qrcode:before { - content: "\f029"; -} -.icon-barcode:before { - content: "\f02a"; -} -.icon-tag:before { - content: "\f02b"; -} -.icon-tags:before { - content: "\f02c"; -} -.icon-book:before { - content: "\f02d"; -} -.icon-bookmark:before { - content: "\f02e"; -} -.icon-print:before { - content: "\f02f"; -} -.icon-camera:before { - content: "\f030"; -} -.icon-font:before { - content: "\f031"; -} -.icon-bold:before { - content: "\f032"; -} -.icon-italic:before { - content: "\f033"; -} -.icon-text-height:before { - content: "\f034"; -} -.icon-text-width:before { - content: "\f035"; -} -.icon-align-left:before { - content: "\f036"; -} -.icon-align-center:before { - content: "\f037"; -} -.icon-align-right:before { - content: "\f038"; -} -.icon-align-justify:before { - content: "\f039"; -} -.icon-list:before { - content: "\f03a"; -} -.icon-indent-left:before { - content: "\f03b"; -} -.icon-indent-right:before { - content: "\f03c"; -} -.icon-facetime-video:before { - content: "\f03d"; -} -.icon-picture:before { - content: "\f03e"; -} -.icon-pencil:before { - content: "\f040"; -} -.icon-map-marker:before { - content: "\f041"; -} -.icon-adjust:before { - content: "\f042"; -} -.icon-tint:before { - content: "\f043"; -} -.icon-edit:before { - content: "\f044"; -} -.icon-share:before { - content: "\f045"; -} -.icon-check:before { - content: "\f046"; -} -.icon-move:before { - content: "\f047"; -} -.icon-step-backward:before { - content: "\f048"; -} -.icon-fast-backward:before { - content: "\f049"; -} -.icon-backward:before { - content: "\f04a"; -} -.icon-play:before { - content: "\f04b"; -} -.icon-pause:before { - content: "\f04c"; -} -.icon-stop:before { - content: "\f04d"; -} -.icon-forward:before { - content: "\f04e"; -} -.icon-fast-forward:before { - content: "\f050"; -} -.icon-step-forward:before { - content: "\f051"; -} -.icon-eject:before { - content: "\f052"; -} -.icon-chevron-left:before { - content: "\f053"; -} -.icon-chevron-right:before { - content: "\f054"; -} -.icon-plus-sign:before { - content: "\f055"; -} -.icon-minus-sign:before { - content: "\f056"; -} -.icon-remove-sign:before { - content: "\f057"; -} -.icon-ok-sign:before { - content: "\f058"; -} -.icon-question-sign:before { - content: "\f059"; -} -.icon-info-sign:before { - content: "\f05a"; -} -.icon-screenshot:before { - content: "\f05b"; -} -.icon-remove-circle:before { - content: "\f05c"; -} -.icon-ok-circle:before { - content: "\f05d"; -} -.icon-ban-circle:before { - content: "\f05e"; -} -.icon-arrow-left:before { - content: "\f060"; -} -.icon-arrow-right:before { - content: "\f061"; -} -.icon-arrow-up:before { - content: "\f062"; -} -.icon-arrow-down:before { - content: "\f063"; -} -.icon-share-alt:before { - content: "\f064"; -} -.icon-resize-full:before { - content: "\f065"; -} -.icon-resize-small:before { - content: "\f066"; -} -.icon-plus:before { - content: "\f067"; -} -.icon-minus:before { - content: "\f068"; -} -.icon-asterisk:before { - content: "\f069"; -} -.icon-exclamation-sign:before { - content: "\f06a"; -} -.icon-gift:before { - content: "\f06b"; -} -.icon-leaf:before { - content: "\f06c"; -} -.icon-fire:before { - content: "\f06d"; -} -.icon-eye-open:before { - content: "\f06e"; -} -.icon-eye-close:before { - content: "\f070"; -} -.icon-warning-sign:before { - content: "\f071"; -} -.icon-plane:before { - content: "\f072"; -} -.icon-calendar:before { - content: "\f073"; -} -.icon-random:before { - content: "\f074"; -} -.icon-comment:before { - content: "\f075"; -} -.icon-magnet:before { - content: "\f076"; -} -.icon-chevron-up:before { - content: "\f077"; -} -.icon-chevron-down:before { - content: "\f078"; -} -.icon-retweet:before { - content: "\f079"; -} -.icon-shopping-cart:before { - content: "\f07a"; -} -.icon-folder-close:before { - content: "\f07b"; -} -.icon-folder-open:before { - content: "\f07c"; -} -.icon-resize-vertical:before { - content: "\f07d"; -} -.icon-resize-horizontal:before { - content: "\f07e"; -} -.icon-bar-chart:before { - content: "\f080"; -} -.icon-twitter-sign:before { - content: "\f081"; -} -.icon-facebook-sign:before { - content: "\f082"; -} -.icon-camera-retro:before { - content: "\f083"; -} -.icon-key:before { - content: "\f084"; -} -.icon-cogs:before { - content: "\f085"; -} -.icon-comments:before { - content: "\f086"; -} -.icon-thumbs-up:before { - content: "\f087"; -} -.icon-thumbs-down:before { - content: "\f088"; -} -.icon-star-half:before { - content: "\f089"; -} -.icon-heart-empty:before { - content: "\f08a"; -} -.icon-signout:before { - content: "\f08b"; -} -.icon-linkedin-sign:before { - content: "\f08c"; -} -.icon-pushpin:before { - content: "\f08d"; -} -.icon-external-link:before { - content: "\f08e"; -} -.icon-signin:before { - content: "\f090"; -} -.icon-trophy:before { - content: "\f091"; -} -.icon-github-sign:before { - content: "\f092"; -} -.icon-upload-alt:before { - content: "\f093"; -} -.icon-lemon:before { - content: "\f094"; -} -.icon-phone:before { - content: "\f095"; -} -.icon-check-empty:before { - content: "\f096"; -} -.icon-bookmark-empty:before { - content: "\f097"; -} -.icon-phone-sign:before { - content: "\f098"; -} -.icon-twitter:before { - content: "\f099"; -} -.icon-facebook:before { - content: "\f09a"; -} -.icon-github:before { - content: "\f09b"; -} -.icon-unlock:before { - content: "\f09c"; -} -.icon-credit-card:before { - content: "\f09d"; -} -.icon-rss:before { - content: "\f09e"; -} -.icon-hdd:before { - content: "\f0a0"; -} -.icon-bullhorn:before { - content: "\f0a1"; -} -.icon-bell:before { - content: "\f0a2"; -} -.icon-certificate:before { - content: "\f0a3"; -} -.icon-hand-right:before { - content: "\f0a4"; -} -.icon-hand-left:before { - content: "\f0a5"; -} -.icon-hand-up:before { - content: "\f0a6"; -} -.icon-hand-down:before { - content: "\f0a7"; -} -.icon-circle-arrow-left:before { - content: "\f0a8"; -} -.icon-circle-arrow-right:before { - content: "\f0a9"; -} -.icon-circle-arrow-up:before { - content: "\f0aa"; -} -.icon-circle-arrow-down:before { - content: "\f0ab"; -} -.icon-globe:before { - content: "\f0ac"; -} -.icon-wrench:before { - content: "\f0ad"; -} -.icon-tasks:before { - content: "\f0ae"; -} -.icon-filter:before { - content: "\f0b0"; -} -.icon-briefcase:before { - content: "\f0b1"; -} -.icon-fullscreen:before { - content: "\f0b2"; -} -.icon-group:before { - content: "\f0c0"; -} -.icon-link:before { - content: "\f0c1"; -} -.icon-cloud:before { - content: "\f0c2"; -} -.icon-beaker:before { - content: "\f0c3"; -} -.icon-cut:before { - content: "\f0c4"; -} -.icon-copy:before { - content: "\f0c5"; -} -.icon-paper-clip:before { - content: "\f0c6"; -} -.icon-save:before { - content: "\f0c7"; -} -.icon-sign-blank:before { - content: "\f0c8"; -} -.icon-reorder:before { - content: "\f0c9"; -} -.icon-list-ul:before { - content: "\f0ca"; -} -.icon-list-ol:before { - content: "\f0cb"; -} -.icon-strikethrough:before { - content: "\f0cc"; -} -.icon-underline:before { - content: "\f0cd"; -} -.icon-table:before { - content: "\f0ce"; -} -.icon-magic:before { - content: "\f0d0"; -} -.icon-truck:before { - content: "\f0d1"; -} -.icon-pinterest:before { - content: "\f0d2"; -} -.icon-pinterest-sign:before { - content: "\f0d3"; -} -.icon-google-plus-sign:before { - content: "\f0d4"; -} -.icon-google-plus:before { - content: "\f0d5"; -} -.icon-money:before { - content: "\f0d6"; -} -.icon-caret-down:before { - content: "\f0d7"; -} -.icon-caret-up:before { - content: "\f0d8"; -} -.icon-caret-left:before { - content: "\f0d9"; -} -.icon-caret-right:before { - content: "\f0da"; -} -.icon-columns:before { - content: "\f0db"; -} -.icon-sort:before { - content: "\f0dc"; -} -.icon-sort-down:before { - content: "\f0dd"; -} -.icon-sort-up:before { - content: "\f0de"; -} -.icon-envelope-alt:before { - content: "\f0e0"; -} -.icon-linkedin:before { - content: "\f0e1"; -} -.icon-undo:before { - content: "\f0e2"; -} -.icon-legal:before { - content: "\f0e3"; -} -.icon-dashboard:before { - content: "\f0e4"; -} -.icon-comment-alt:before { - content: "\f0e5"; -} -.icon-comments-alt:before { - content: "\f0e6"; -} -.icon-bolt:before { - content: "\f0e7"; -} -.icon-sitemap:before { - content: "\f0e8"; -} -.icon-umbrella:before { - content: "\f0e9"; -} -.icon-paste:before { - content: "\f0ea"; -} -.icon-lightbulb:before { - content: "\f0eb"; -} -.icon-exchange:before { - content: "\f0ec"; -} -.icon-cloud-download:before { - content: "\f0ed"; -} -.icon-cloud-upload:before { - content: "\f0ee"; -} -.icon-user-md:before { - content: "\f0f0"; -} -.icon-stethoscope:before { - content: "\f0f1"; -} -.icon-suitcase:before { - content: "\f0f2"; -} -.icon-bell-alt:before { - content: "\f0f3"; -} -.icon-coffee:before { - content: "\f0f4"; -} -.icon-food:before { - content: "\f0f5"; -} -.icon-file-alt:before { - content: "\f0f6"; -} -.icon-building:before { - content: "\f0f7"; -} -.icon-hospital:before { - content: "\f0f8"; -} -.icon-ambulance:before { - content: "\f0f9"; -} -.icon-medkit:before { - content: "\f0fa"; -} -.icon-fighter-jet:before { - content: "\f0fb"; -} -.icon-beer:before { - content: "\f0fc"; -} -.icon-h-sign:before { - content: "\f0fd"; -} -.icon-plus-sign-alt:before { - content: "\f0fe"; -} -.icon-double-angle-left:before { - content: "\f100"; -} -.icon-double-angle-right:before { - content: "\f101"; -} -.icon-double-angle-up:before { - content: "\f102"; -} -.icon-double-angle-down:before { - content: "\f103"; -} -.icon-angle-left:before { - content: "\f104"; -} -.icon-angle-right:before { - content: "\f105"; -} -.icon-angle-up:before { - content: "\f106"; -} -.icon-angle-down:before { - content: "\f107"; -} -.icon-desktop:before { - content: "\f108"; -} -.icon-laptop:before { - content: "\f109"; -} -.icon-tablet:before { - content: "\f10a"; -} -.icon-mobile-phone:before { - content: "\f10b"; -} -.icon-circle-blank:before { - content: "\f10c"; -} -.icon-quote-left:before { - content: "\f10d"; -} -.icon-quote-right:before { - content: "\f10e"; -} -.icon-spinner:before { - content: "\f110"; -} -.icon-circle:before { - content: "\f111"; -} -.icon-reply:before { - content: "\f112"; -} -.icon-github-alt:before { - content: "\f113"; -} -.icon-folder-close-alt:before { - content: "\f114"; -} -.icon-folder-open-alt:before { - content: "\f115"; -} [class^="icon-"], [class*=" icon-"] { display: inline-block; @@ -7963,45 +7141,39 @@ h4 small { .context-info.editing .module-content { margin-top: 0; } -.hero { - background: url("../../../base/images/background-tile.png"); +.homepage [role=main] { padding: 20px 0; min-height: 0; } -.hero > .container { +.homepage .row { position: relative; - padding-bottom: 0; } -.hero .search-giant { - margin-bottom: 10px; -} -.hero .search-giant input { - border-color: #003f52; -} -.hero .page-heading { - font-size: 18px; - margin-bottom: 0; -} -.hero .module-dark { +.homepage .module-search { padding: 5px; - margin-bottom: 0; + margin: 0; color: #ffffff; background: #ffffff; } -.hero .module-dark .module-content { +.homepage .module-search .search-giant { + margin-bottom: 10px; +} +.homepage .module-search .search-giant input { + border-color: #003f52; +} +.homepage .module-search .module-content { -webkit-border-radius: 3px 3px 0 0; -moz-border-radius: 3px 3px 0 0; border-radius: 3px 3px 0 0; background-color: #810606; border-bottom: none; } -.hero .module-dark .module-content .heading { +.homepage .module-search .module-content .heading { margin-top: 0; margin-bottom: 7px; font-size: 24px; line-height: 40px; } -.hero .tags { +.homepage .module-search .tags { *zoom: 1; padding: 5px 10px 10px 10px; background-color: #500404; @@ -8009,69 +7181,77 @@ h4 small { -moz-border-radius: 0 0 3px 3px; border-radius: 0 0 3px 3px; } -.hero .tags:before, -.hero .tags:after { +.homepage .module-search .tags:before, +.homepage .module-search .tags:after { display: table; content: ""; line-height: 0; } -.hero .tags:after { +.homepage .module-search .tags:after { clear: both; } -.hero .tags h3, -.hero .tags .tag { +.homepage .module-search .tags h3, +.homepage .module-search .tags .tag { display: block; float: left; margin: 5px 10px 0 0; } -.hero .tags h3 { +.homepage .module-search .tags h3 { font-size: 14px; line-height: 20px; padding: 2px 8px; } -.hero-primary, -.hero-secondary { +.homepage .group-list { + margin: 0; +} +.homepage .box .inner { + padding: 20px 25px; +} +.homepage .stats h3 { + margin: 0 0 10px 0; +} +.homepage .stats ul { + margin: 0; + list-style: none; + *zoom: 1; +} +.homepage .stats ul:before, +.homepage .stats ul:after { + display: table; + content: ""; + line-height: 0; +} +.homepage .stats ul:after { + clear: both; +} +.homepage .stats ul li { float: left; - margin-left: 20px; - width: 460px; + width: 25%; + font-weight: 300; } -.hero-primary { - margin-left: 0; - margin-bottom: 0; +.homepage .stats ul li a { + display: block; } -.hero-secondary { +.homepage .stats ul li a b { + display: block; + font-size: 35px; + line-height: 1.5; +} +.homepage .stats ul li a:hover { + text-decoration: none; +} +.homepage.layout-1 .row1 .col2 { position: absolute; bottom: 0; right: 0; } -.hero-secondary .hero-secondary-inner { +.homepage.layout-1 .row1 .col2 .module-search { bottom: 0; left: 0; right: 0; } -.main.homepage { - padding-top: 20px; - padding-bottom: 20px; - border-top: 1px solid #cccccc; -} -.main.homepage .module-heading .media-image { - margin-right: 15px; - max-height: 53px; - -webkit-box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); - -moz-box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); - box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); -} -.main.homepage .group-listing .box { - min-height: 275px; -} -.main.homepage .group-list { - margin-bottom: 0; -} -.main.homepage .group-list .dataset-content { - min-height: 70px; -} -.main.homepage .box .module { - margin-top: 0; +.homepage.layout-2 .stats { + margin-top: 20px; } .account-masthead { *zoom: 1; @@ -8920,6 +8100,11 @@ iframe { padding-bottom: 20px !important; margin-bottom: 0 !important; } +.ie8 .lang-dropdown, +.ie7 .lang-dropdown { + position: relative !important; + top: -90px !important; +} .ie7 .alert { position: relative; } diff --git a/ckan/public/base/css/red.css b/ckan/public/base/css/red.css index 26f22422970..d24e241544a 100644 --- a/ckan/public/base/css/red.css +++ b/ckan/public/base/css/red.css @@ -2055,6 +2055,14 @@ table th[class*="span"], .open > .dropdown-menu { display: block; } +.dropdown-backdrop { + position: fixed; + left: 0; + right: 0; + bottom: 0; + top: 0; + z-index: 990; +} .pull-right > .dropdown-menu { right: 0; left: auto; @@ -5572,6 +5580,12 @@ textarea { .control-large input { height: 41px; } +.control-required { + color: #c6898b; +} +.form-actions .control-required-message { + float: left; +} .form-actions { background: none; margin-left: -25px; @@ -6904,842 +6918,6 @@ h4 small { height: 35px; background-position: -320px -62px; } -/* This is a modified version of the font-awesome core less document. */ -@font-face { - font-family: 'FontAwesome'; - src: url('../../../base/vendor/font-awesome/font/fontawesome-webfont.eot?v=3.0.1'); - src: url('../../../base/vendor/font-awesome/font/fontawesome-webfont.eot?#iefix&v=3.0.1') format('embedded-opentype'), url('../../../base/vendor/font-awesome/font/fontawesome-webfont.woff?v=3.0.1') format('woff'), url('../../../base/vendor/font-awesome/font/fontawesome-webfont.ttf?v=3.0.1') format('truetype'); - font-weight: normal; - font-style: normal; -} -[class^="icon-"], -[class*=" icon-"] { - font-family: FontAwesome; - font-weight: normal; - font-style: normal; - text-decoration: inherit; - -webkit-font-smoothing: antialiased; - display: inline; - width: auto; - height: auto; - line-height: normal; - vertical-align: baseline; - background-image: none; - background-position: 0% 0%; - background-repeat: repeat; - margin-top: 0; -} -.icon-spin { - display: inline-block; - -moz-animation: spin 2s infinite linear; - -o-animation: spin 2s infinite linear; - -webkit-animation: spin 2s infinite linear; - animation: spin 2s infinite linear; -} -@-moz-keyframes spin { - 0% { - -moz-transform: rotate(0deg); - } - 100% { - -moz-transform: rotate(359deg); - } -} -@-webkit-keyframes spin { - 0% { - -webkit-transform: rotate(0deg); - } - 100% { - -webkit-transform: rotate(359deg); - } -} -@-o-keyframes spin { - 0% { - -o-transform: rotate(0deg); - } - 100% { - -o-transform: rotate(359deg); - } -} -@-ms-keyframes spin { - 0% { - -ms-transform: rotate(0deg); - } - 100% { - -ms-transform: rotate(359deg); - } -} -@keyframes spin { - 0% { - transform: rotate(0deg); - } - 100% { - transform: rotate(359deg); - } -} -@-moz-document url-prefix() { - .icon-spin { - height: .9em; - } - .btn .icon-spin { - height: auto; - } - .icon-spin.icon-large { - height: 1.25em; - } - .btn .icon-spin.icon-large { - height: .75em; - } -} -/* Font Awesome uses the Unicode Private Use Area (PUA) to ensure screen - readers do not read off random characters that represent icons */ -.icon-glass:before { - content: "\f000"; -} -.icon-music:before { - content: "\f001"; -} -.icon-search:before { - content: "\f002"; -} -.icon-envelope:before { - content: "\f003"; -} -.icon-heart:before { - content: "\f004"; -} -.icon-star:before { - content: "\f005"; -} -.icon-star-empty:before { - content: "\f006"; -} -.icon-user:before { - content: "\f007"; -} -.icon-film:before { - content: "\f008"; -} -.icon-th-large:before { - content: "\f009"; -} -.icon-th:before { - content: "\f00a"; -} -.icon-th-list:before { - content: "\f00b"; -} -.icon-ok:before { - content: "\f00c"; -} -.icon-remove:before { - content: "\f00d"; -} -.icon-zoom-in:before { - content: "\f00e"; -} -.icon-zoom-out:before { - content: "\f010"; -} -.icon-off:before { - content: "\f011"; -} -.icon-signal:before { - content: "\f012"; -} -.icon-cog:before { - content: "\f013"; -} -.icon-trash:before { - content: "\f014"; -} -.icon-home:before { - content: "\f015"; -} -.icon-file:before { - content: "\f016"; -} -.icon-time:before { - content: "\f017"; -} -.icon-road:before { - content: "\f018"; -} -.icon-download-alt:before { - content: "\f019"; -} -.icon-download:before { - content: "\f01a"; -} -.icon-upload:before { - content: "\f01b"; -} -.icon-inbox:before { - content: "\f01c"; -} -.icon-play-circle:before { - content: "\f01d"; -} -.icon-repeat:before { - content: "\f01e"; -} -/* \f020 doesn't work in Safari. all shifted one down */ -.icon-refresh:before { - content: "\f021"; -} -.icon-list-alt:before { - content: "\f022"; -} -.icon-lock:before { - content: "\f023"; -} -.icon-flag:before { - content: "\f024"; -} -.icon-headphones:before { - content: "\f025"; -} -.icon-volume-off:before { - content: "\f026"; -} -.icon-volume-down:before { - content: "\f027"; -} -.icon-volume-up:before { - content: "\f028"; -} -.icon-qrcode:before { - content: "\f029"; -} -.icon-barcode:before { - content: "\f02a"; -} -.icon-tag:before { - content: "\f02b"; -} -.icon-tags:before { - content: "\f02c"; -} -.icon-book:before { - content: "\f02d"; -} -.icon-bookmark:before { - content: "\f02e"; -} -.icon-print:before { - content: "\f02f"; -} -.icon-camera:before { - content: "\f030"; -} -.icon-font:before { - content: "\f031"; -} -.icon-bold:before { - content: "\f032"; -} -.icon-italic:before { - content: "\f033"; -} -.icon-text-height:before { - content: "\f034"; -} -.icon-text-width:before { - content: "\f035"; -} -.icon-align-left:before { - content: "\f036"; -} -.icon-align-center:before { - content: "\f037"; -} -.icon-align-right:before { - content: "\f038"; -} -.icon-align-justify:before { - content: "\f039"; -} -.icon-list:before { - content: "\f03a"; -} -.icon-indent-left:before { - content: "\f03b"; -} -.icon-indent-right:before { - content: "\f03c"; -} -.icon-facetime-video:before { - content: "\f03d"; -} -.icon-picture:before { - content: "\f03e"; -} -.icon-pencil:before { - content: "\f040"; -} -.icon-map-marker:before { - content: "\f041"; -} -.icon-adjust:before { - content: "\f042"; -} -.icon-tint:before { - content: "\f043"; -} -.icon-edit:before { - content: "\f044"; -} -.icon-share:before { - content: "\f045"; -} -.icon-check:before { - content: "\f046"; -} -.icon-move:before { - content: "\f047"; -} -.icon-step-backward:before { - content: "\f048"; -} -.icon-fast-backward:before { - content: "\f049"; -} -.icon-backward:before { - content: "\f04a"; -} -.icon-play:before { - content: "\f04b"; -} -.icon-pause:before { - content: "\f04c"; -} -.icon-stop:before { - content: "\f04d"; -} -.icon-forward:before { - content: "\f04e"; -} -.icon-fast-forward:before { - content: "\f050"; -} -.icon-step-forward:before { - content: "\f051"; -} -.icon-eject:before { - content: "\f052"; -} -.icon-chevron-left:before { - content: "\f053"; -} -.icon-chevron-right:before { - content: "\f054"; -} -.icon-plus-sign:before { - content: "\f055"; -} -.icon-minus-sign:before { - content: "\f056"; -} -.icon-remove-sign:before { - content: "\f057"; -} -.icon-ok-sign:before { - content: "\f058"; -} -.icon-question-sign:before { - content: "\f059"; -} -.icon-info-sign:before { - content: "\f05a"; -} -.icon-screenshot:before { - content: "\f05b"; -} -.icon-remove-circle:before { - content: "\f05c"; -} -.icon-ok-circle:before { - content: "\f05d"; -} -.icon-ban-circle:before { - content: "\f05e"; -} -.icon-arrow-left:before { - content: "\f060"; -} -.icon-arrow-right:before { - content: "\f061"; -} -.icon-arrow-up:before { - content: "\f062"; -} -.icon-arrow-down:before { - content: "\f063"; -} -.icon-share-alt:before { - content: "\f064"; -} -.icon-resize-full:before { - content: "\f065"; -} -.icon-resize-small:before { - content: "\f066"; -} -.icon-plus:before { - content: "\f067"; -} -.icon-minus:before { - content: "\f068"; -} -.icon-asterisk:before { - content: "\f069"; -} -.icon-exclamation-sign:before { - content: "\f06a"; -} -.icon-gift:before { - content: "\f06b"; -} -.icon-leaf:before { - content: "\f06c"; -} -.icon-fire:before { - content: "\f06d"; -} -.icon-eye-open:before { - content: "\f06e"; -} -.icon-eye-close:before { - content: "\f070"; -} -.icon-warning-sign:before { - content: "\f071"; -} -.icon-plane:before { - content: "\f072"; -} -.icon-calendar:before { - content: "\f073"; -} -.icon-random:before { - content: "\f074"; -} -.icon-comment:before { - content: "\f075"; -} -.icon-magnet:before { - content: "\f076"; -} -.icon-chevron-up:before { - content: "\f077"; -} -.icon-chevron-down:before { - content: "\f078"; -} -.icon-retweet:before { - content: "\f079"; -} -.icon-shopping-cart:before { - content: "\f07a"; -} -.icon-folder-close:before { - content: "\f07b"; -} -.icon-folder-open:before { - content: "\f07c"; -} -.icon-resize-vertical:before { - content: "\f07d"; -} -.icon-resize-horizontal:before { - content: "\f07e"; -} -.icon-bar-chart:before { - content: "\f080"; -} -.icon-twitter-sign:before { - content: "\f081"; -} -.icon-facebook-sign:before { - content: "\f082"; -} -.icon-camera-retro:before { - content: "\f083"; -} -.icon-key:before { - content: "\f084"; -} -.icon-cogs:before { - content: "\f085"; -} -.icon-comments:before { - content: "\f086"; -} -.icon-thumbs-up:before { - content: "\f087"; -} -.icon-thumbs-down:before { - content: "\f088"; -} -.icon-star-half:before { - content: "\f089"; -} -.icon-heart-empty:before { - content: "\f08a"; -} -.icon-signout:before { - content: "\f08b"; -} -.icon-linkedin-sign:before { - content: "\f08c"; -} -.icon-pushpin:before { - content: "\f08d"; -} -.icon-external-link:before { - content: "\f08e"; -} -.icon-signin:before { - content: "\f090"; -} -.icon-trophy:before { - content: "\f091"; -} -.icon-github-sign:before { - content: "\f092"; -} -.icon-upload-alt:before { - content: "\f093"; -} -.icon-lemon:before { - content: "\f094"; -} -.icon-phone:before { - content: "\f095"; -} -.icon-check-empty:before { - content: "\f096"; -} -.icon-bookmark-empty:before { - content: "\f097"; -} -.icon-phone-sign:before { - content: "\f098"; -} -.icon-twitter:before { - content: "\f099"; -} -.icon-facebook:before { - content: "\f09a"; -} -.icon-github:before { - content: "\f09b"; -} -.icon-unlock:before { - content: "\f09c"; -} -.icon-credit-card:before { - content: "\f09d"; -} -.icon-rss:before { - content: "\f09e"; -} -.icon-hdd:before { - content: "\f0a0"; -} -.icon-bullhorn:before { - content: "\f0a1"; -} -.icon-bell:before { - content: "\f0a2"; -} -.icon-certificate:before { - content: "\f0a3"; -} -.icon-hand-right:before { - content: "\f0a4"; -} -.icon-hand-left:before { - content: "\f0a5"; -} -.icon-hand-up:before { - content: "\f0a6"; -} -.icon-hand-down:before { - content: "\f0a7"; -} -.icon-circle-arrow-left:before { - content: "\f0a8"; -} -.icon-circle-arrow-right:before { - content: "\f0a9"; -} -.icon-circle-arrow-up:before { - content: "\f0aa"; -} -.icon-circle-arrow-down:before { - content: "\f0ab"; -} -.icon-globe:before { - content: "\f0ac"; -} -.icon-wrench:before { - content: "\f0ad"; -} -.icon-tasks:before { - content: "\f0ae"; -} -.icon-filter:before { - content: "\f0b0"; -} -.icon-briefcase:before { - content: "\f0b1"; -} -.icon-fullscreen:before { - content: "\f0b2"; -} -.icon-group:before { - content: "\f0c0"; -} -.icon-link:before { - content: "\f0c1"; -} -.icon-cloud:before { - content: "\f0c2"; -} -.icon-beaker:before { - content: "\f0c3"; -} -.icon-cut:before { - content: "\f0c4"; -} -.icon-copy:before { - content: "\f0c5"; -} -.icon-paper-clip:before { - content: "\f0c6"; -} -.icon-save:before { - content: "\f0c7"; -} -.icon-sign-blank:before { - content: "\f0c8"; -} -.icon-reorder:before { - content: "\f0c9"; -} -.icon-list-ul:before { - content: "\f0ca"; -} -.icon-list-ol:before { - content: "\f0cb"; -} -.icon-strikethrough:before { - content: "\f0cc"; -} -.icon-underline:before { - content: "\f0cd"; -} -.icon-table:before { - content: "\f0ce"; -} -.icon-magic:before { - content: "\f0d0"; -} -.icon-truck:before { - content: "\f0d1"; -} -.icon-pinterest:before { - content: "\f0d2"; -} -.icon-pinterest-sign:before { - content: "\f0d3"; -} -.icon-google-plus-sign:before { - content: "\f0d4"; -} -.icon-google-plus:before { - content: "\f0d5"; -} -.icon-money:before { - content: "\f0d6"; -} -.icon-caret-down:before { - content: "\f0d7"; -} -.icon-caret-up:before { - content: "\f0d8"; -} -.icon-caret-left:before { - content: "\f0d9"; -} -.icon-caret-right:before { - content: "\f0da"; -} -.icon-columns:before { - content: "\f0db"; -} -.icon-sort:before { - content: "\f0dc"; -} -.icon-sort-down:before { - content: "\f0dd"; -} -.icon-sort-up:before { - content: "\f0de"; -} -.icon-envelope-alt:before { - content: "\f0e0"; -} -.icon-linkedin:before { - content: "\f0e1"; -} -.icon-undo:before { - content: "\f0e2"; -} -.icon-legal:before { - content: "\f0e3"; -} -.icon-dashboard:before { - content: "\f0e4"; -} -.icon-comment-alt:before { - content: "\f0e5"; -} -.icon-comments-alt:before { - content: "\f0e6"; -} -.icon-bolt:before { - content: "\f0e7"; -} -.icon-sitemap:before { - content: "\f0e8"; -} -.icon-umbrella:before { - content: "\f0e9"; -} -.icon-paste:before { - content: "\f0ea"; -} -.icon-lightbulb:before { - content: "\f0eb"; -} -.icon-exchange:before { - content: "\f0ec"; -} -.icon-cloud-download:before { - content: "\f0ed"; -} -.icon-cloud-upload:before { - content: "\f0ee"; -} -.icon-user-md:before { - content: "\f0f0"; -} -.icon-stethoscope:before { - content: "\f0f1"; -} -.icon-suitcase:before { - content: "\f0f2"; -} -.icon-bell-alt:before { - content: "\f0f3"; -} -.icon-coffee:before { - content: "\f0f4"; -} -.icon-food:before { - content: "\f0f5"; -} -.icon-file-alt:before { - content: "\f0f6"; -} -.icon-building:before { - content: "\f0f7"; -} -.icon-hospital:before { - content: "\f0f8"; -} -.icon-ambulance:before { - content: "\f0f9"; -} -.icon-medkit:before { - content: "\f0fa"; -} -.icon-fighter-jet:before { - content: "\f0fb"; -} -.icon-beer:before { - content: "\f0fc"; -} -.icon-h-sign:before { - content: "\f0fd"; -} -.icon-plus-sign-alt:before { - content: "\f0fe"; -} -.icon-double-angle-left:before { - content: "\f100"; -} -.icon-double-angle-right:before { - content: "\f101"; -} -.icon-double-angle-up:before { - content: "\f102"; -} -.icon-double-angle-down:before { - content: "\f103"; -} -.icon-angle-left:before { - content: "\f104"; -} -.icon-angle-right:before { - content: "\f105"; -} -.icon-angle-up:before { - content: "\f106"; -} -.icon-angle-down:before { - content: "\f107"; -} -.icon-desktop:before { - content: "\f108"; -} -.icon-laptop:before { - content: "\f109"; -} -.icon-tablet:before { - content: "\f10a"; -} -.icon-mobile-phone:before { - content: "\f10b"; -} -.icon-circle-blank:before { - content: "\f10c"; -} -.icon-quote-left:before { - content: "\f10d"; -} -.icon-quote-right:before { - content: "\f10e"; -} -.icon-spinner:before { - content: "\f110"; -} -.icon-circle:before { - content: "\f111"; -} -.icon-reply:before { - content: "\f112"; -} -.icon-github-alt:before { - content: "\f113"; -} -.icon-folder-close-alt:before { - content: "\f114"; -} -.icon-folder-open-alt:before { - content: "\f115"; -} [class^="icon-"], [class*=" icon-"] { display: inline-block; @@ -7963,45 +7141,39 @@ h4 small { .context-info.editing .module-content { margin-top: 0; } -.hero { - background: url("../../../base/images/background-tile.png"); +.homepage [role=main] { padding: 20px 0; min-height: 0; } -.hero > .container { +.homepage .row { position: relative; - padding-bottom: 0; } -.hero .search-giant { - margin-bottom: 10px; -} -.hero .search-giant input { - border-color: #003f52; -} -.hero .page-heading { - font-size: 18px; - margin-bottom: 0; -} -.hero .module-dark { +.homepage .module-search { padding: 5px; - margin-bottom: 0; + margin: 0; color: #ffffff; background: #ffffff; } -.hero .module-dark .module-content { +.homepage .module-search .search-giant { + margin-bottom: 10px; +} +.homepage .module-search .search-giant input { + border-color: #003f52; +} +.homepage .module-search .module-content { -webkit-border-radius: 3px 3px 0 0; -moz-border-radius: 3px 3px 0 0; border-radius: 3px 3px 0 0; background-color: #c14531; border-bottom: none; } -.hero .module-dark .module-content .heading { +.homepage .module-search .module-content .heading { margin-top: 0; margin-bottom: 7px; font-size: 24px; line-height: 40px; } -.hero .tags { +.homepage .module-search .tags { *zoom: 1; padding: 5px 10px 10px 10px; background-color: #983627; @@ -8009,69 +7181,77 @@ h4 small { -moz-border-radius: 0 0 3px 3px; border-radius: 0 0 3px 3px; } -.hero .tags:before, -.hero .tags:after { +.homepage .module-search .tags:before, +.homepage .module-search .tags:after { display: table; content: ""; line-height: 0; } -.hero .tags:after { +.homepage .module-search .tags:after { clear: both; } -.hero .tags h3, -.hero .tags .tag { +.homepage .module-search .tags h3, +.homepage .module-search .tags .tag { display: block; float: left; margin: 5px 10px 0 0; } -.hero .tags h3 { +.homepage .module-search .tags h3 { font-size: 14px; line-height: 20px; padding: 2px 8px; } -.hero-primary, -.hero-secondary { +.homepage .group-list { + margin: 0; +} +.homepage .box .inner { + padding: 20px 25px; +} +.homepage .stats h3 { + margin: 0 0 10px 0; +} +.homepage .stats ul { + margin: 0; + list-style: none; + *zoom: 1; +} +.homepage .stats ul:before, +.homepage .stats ul:after { + display: table; + content: ""; + line-height: 0; +} +.homepage .stats ul:after { + clear: both; +} +.homepage .stats ul li { float: left; - margin-left: 20px; - width: 460px; + width: 25%; + font-weight: 300; } -.hero-primary { - margin-left: 0; - margin-bottom: 0; +.homepage .stats ul li a { + display: block; } -.hero-secondary { +.homepage .stats ul li a b { + display: block; + font-size: 35px; + line-height: 1.5; +} +.homepage .stats ul li a:hover { + text-decoration: none; +} +.homepage.layout-1 .row1 .col2 { position: absolute; bottom: 0; right: 0; } -.hero-secondary .hero-secondary-inner { +.homepage.layout-1 .row1 .col2 .module-search { bottom: 0; left: 0; right: 0; } -.main.homepage { - padding-top: 20px; - padding-bottom: 20px; - border-top: 1px solid #cccccc; -} -.main.homepage .module-heading .media-image { - margin-right: 15px; - max-height: 53px; - -webkit-box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); - -moz-box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); - box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.5); -} -.main.homepage .group-listing .box { - min-height: 275px; -} -.main.homepage .group-list { - margin-bottom: 0; -} -.main.homepage .group-list .dataset-content { - min-height: 70px; -} -.main.homepage .box .module { - margin-top: 0; +.homepage.layout-2 .stats { + margin-top: 20px; } .account-masthead { *zoom: 1; @@ -8920,6 +8100,11 @@ iframe { padding-bottom: 20px !important; margin-bottom: 0 !important; } +.ie8 .lang-dropdown, +.ie7 .lang-dropdown { + position: relative !important; + top: -90px !important; +} .ie7 .alert { position: relative; } From afc80ed59d6173a96980576fc4760702413ac8dc Mon Sep 17 00:00:00 2001 From: John Martin Date: Thu, 3 Oct 2013 10:49:20 +0100 Subject: [PATCH 156/269] [#1259] Fixes a JS error with autocomplete module and improves UX 1. Basically adds an extra check to make sure that 'abort()' is defined 2. Then changes the module to behave a little nicer when inbetween ajax requests --- .../base/javascript/modules/autocomplete.js | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/ckan/public/base/javascript/modules/autocomplete.js b/ckan/public/base/javascript/modules/autocomplete.js index 768a708c9c3..adea1628f49 100644 --- a/ckan/public/base/javascript/modules/autocomplete.js +++ b/ckan/public/base/javascript/modules/autocomplete.js @@ -86,6 +86,8 @@ this.ckan.module('autocomplete', function (jQuery, _) { $('.select2-choice', select2.container).on('click', function() { return false; }); + + this._select2 = select2; }, /* Looks up the completions for the current search term and passes them @@ -140,29 +142,30 @@ this.ckan.module('autocomplete', function (jQuery, _) { // old data. this._lastTerm = string; + // Kills previous timeout + clearTimeout(this._debounced); + + // OK, wipe the dropdown before we start ajaxing the completions + fn({results:[]}); + if (string) { - if (!this._debounced) { - // Set a timer to prevent the search lookup occurring too often. - this._debounced = setTimeout(function () { - var term = module._lastTerm; + // Set a timer to prevent the search lookup occurring too often. + this._debounced = setTimeout(function () { + var term = module._lastTerm; - delete module._debounced; + // Cancel the previous request if it hasn't yet completed. + if (module._last && typeof module._last.abort == 'function') { + module._last.abort(); + } - // Cancel the previous request if it hasn't yet completed. - if (module._last) { - module._last.abort(); - } + module._last = module.getCompletions(term, function (terms) { + fn(terms); + }); + }, this.options.interval); - module._last = module.getCompletions(term, function (terms) { - fn(module._lastResults = terms); - }); - }, this.options.interval); - } else { - // Re-use the last set of terms. - fn(this._lastResults || {results: []}); - } - } else { - fn({results: []}); + // This forces the ajax throbber to appear, because we've called the + // callback already and that hides the throbber + $('.select2-search input', this._select2.dropdown).addClass('select2-active'); } }, From 3d5fe71e8b17067f032784be75b932a352163fc6 Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 7 Oct 2013 10:00:07 +0100 Subject: [PATCH 157/269] [1262] basic image upload functionality --- ckan/config/middleware.py | 6 ++ ckan/controllers/group.py | 28 ++++++++- ckan/lib/dictization/model_dictize.py | 21 +++++++ ckan/lib/helpers.py | 5 ++ ckan/lib/munge.py | 7 +++ ckan/lib/uploader.py | 63 +++++++++++++++++++ ckan/logic/schema.py | 1 + ckan/templates/group/snippets/group_form.html | 11 +++- ckan/templates/group/snippets/group_item.html | 2 +- .../snippets/organization_form.html | 10 ++- .../snippets/organization_item.html | 2 +- ckan/templates/snippets/group.html | 2 +- ckan/templates/snippets/group_item.html | 2 +- ckan/templates/snippets/organization.html | 2 +- .../templates/snippets/organization_item.html | 2 +- 15 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 ckan/lib/uploader.py diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 4df0aa84cb0..63f5568cd5f 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -147,6 +147,12 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): cache_max_age=static_max_age) static_parsers = [static_app, app] + storage_directory = config.get('ckan.storage_path') + if storage_directory: + storage_app = StaticURLParser(storage_directory, + cache_max_age=static_max_age) + static_parsers.insert(0, storage_app) + # Configurable extra static file paths extra_static_parsers = [] for public_path in config.get('extra_public_paths', '').split(','): diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index ccef46bfe78..a11dd03cc40 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -1,14 +1,18 @@ import re +import os import logging import genshi +import cgi import datetime from urllib import urlencode from pylons.i18n import get_lang +from pylons import config import ckan.lib.base as base import ckan.lib.helpers as h import ckan.lib.maintain as maintain +import ckan.lib.uploader as uploader import ckan.lib.navl.dictization_functions as dict_fns import ckan.logic as logic import ckan.lib.search as search @@ -421,6 +425,9 @@ def new(self, data=None, errors=None, error_summary=None): return self._save_new(context, group_type) data = data or {} + if not data.get('image_url', '').startswith('http'): + data.pop('image_url', None) + errors = errors or {} error_summary = error_summary or {} vars = {'data': data, 'errors': errors, @@ -445,10 +452,17 @@ def edit(self, id, data=None, errors=None, error_summary=None): return self._save_edit(id, context) try: + ## only needs removing when form repopulates + if data and not data.get('image_url', '').startswith('http'): + data.pop('image_url', None) old_data = self._action('group_show')(context, data_dict) c.grouptitle = old_data.get('title') c.groupname = old_data.get('name') data = data or old_data + image_url = data.get('image_url') + if image_url and not image_url.startswith('http'): + data['upload_file'] = image_url + data['image_url'] = '' except NotFound: abort(404, _('Group not found')) except NotAuthorized: @@ -456,6 +470,7 @@ def edit(self, id, data=None, errors=None, error_summary=None): group = context.get("group") c.group = group + context.pop('for_edit') c.group_dict = self._action('group_show')(context, data_dict) try: @@ -483,12 +498,17 @@ def _get_group_type(self, id): def _save_new(self, context, group_type=None): try: + upload = uploader.Upload('group') + upload.register_request(request, 'image_url', + 'image_upload', 'clear_upload') + data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) data_dict['type'] = group_type or 'group' context['message'] = data_dict.get('log_message', '') data_dict['users'] = [{'name': c.user, 'capacity': 'admin'}] group = self._action('group_create')(context, data_dict) + upload.upload() # Redirect to the appropriate _read route for the type of group h.redirect_to(group['type'] + '_read', id=group['name']) @@ -514,13 +534,19 @@ def _force_reindex(self, grp): def _save_edit(self, id, context): try: + old_group = self._action('group_show')(context, {"id": id}) + old_image_url = old_group.get('image_url') + upload = uploader.Upload('group', old_image_url) + upload.register_request(request, 'image_url', + 'image_upload', 'clear_upload') + data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) context['message'] = data_dict.get('log_message', '') data_dict['id'] = id context['allow_partial_update'] = True group = self._action('group_update')(context, data_dict) - + upload.upload() if id != group['name']: self._force_reindex(group) diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index b3a83235cd6..98756abc2ec 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -10,6 +10,7 @@ import ckan.lib.dictization as d import ckan.new_authz as new_authz import ckan.lib.search as search +import ckan.lib.munge as munge ## package save @@ -41,6 +42,16 @@ def group_list_dictize(obj_list, context, group_dict['display_name'] = obj.display_name + image_url = group_dict.get('image_url') + group_dict['image_display_url'] = image_url + if image_url and not image_url.startswith('http'): + #munge here should not have an effect only doing it incase + #of potential vulnerability of dodgy api input + image_url = munge.munge_filename(image_url) + group_dict['image_display_url'] = h.url_for_static( + 'uploads/group/%s' % group_dict.get('image_url') + ) + if obj.is_organization: group_dict['packages'] = query.facets['owner_org'].get(obj.id, 0) else: @@ -354,6 +365,16 @@ def group_dictize(group, context): for item in plugins.PluginImplementations(plugin): result_dict = item.before_view(result_dict) + if not context.get('for_edit'): + image_url = result_dict.get('image_url') + result_dict['image_display_url'] = image_url + if image_url and not image_url.startswith('http'): + #munge here should not have an effect only doing it incase + #of potential vulnerability of dodgy api input + image_url = munge.munge_filename(image_url) + result_dict['image_display_url'] = h.url_for_static( + 'uploads/group/%s' % result_dict.get('image_url') + ) return result_dict def tag_list_dictize(tag_list, context): diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index c70d74d7dbc..fceab6f1b6a 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -1663,6 +1663,10 @@ def new_activities(): action = logic.get_action('dashboard_new_activities_count') return action({}, {}) +def uploads_enabled(): + if config.get('ckan.storage_path'): + return True + return False # these are the functions that will end up in `h` template helpers __allowed_functions__ = [ @@ -1761,4 +1765,5 @@ def new_activities(): 'radio', 'submit', 'asbool', + 'uploads_enabled', ] diff --git a/ckan/lib/munge.py b/ckan/lib/munge.py index c943e1b113f..a47c8c02258 100644 --- a/ckan/lib/munge.py +++ b/ckan/lib/munge.py @@ -105,6 +105,13 @@ def munge_tag(tag): tag = _munge_to_length(tag, model.MIN_TAG_LENGTH, model.MAX_TAG_LENGTH) return tag +def munge_filename(filename): + filename = substitute_ascii_equivalents(filename) + filename = filename.lower().strip() + filename = re.sub(r'[^a-zA-Z0-9. ]', '', filename).replace(' ', '-') + filename = _munge_to_length(filename, 3, 100) + return filename + def _munge_to_length(string, min_length, max_length): '''Pad/truncates a string''' if len(string) < min_length: diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py new file mode 100644 index 00000000000..5fc144e3aef --- /dev/null +++ b/ckan/lib/uploader.py @@ -0,0 +1,63 @@ +import os +import cgi +import pylons +import datetime +import ckan.lib.munge as munge + +class Upload(object): + def __init__(self, object_type, old_filename=None): + path = pylons.config.get('ckan.storage_path', '/tmp') + self.storage_path = os.path.join(path, 'uploads', object_type) + try: + os.makedirs(self.storage_path) + except OSError, e: + pass + self.object_type = object_type + self.old_filename = old_filename + if old_filename: + self.old_filepath = os.path.join(self.storage_path, old_filename) + self.filename = None + self.filepath = None + + def register_request(self, request, url_field, file_field, clear_field): + + self.request = request + self.url = request.POST.get(url_field) + self.clear = request.POST.get(clear_field) + self.upload_field_storage = request.POST.pop(file_field, None) + + if isinstance(self.upload_field_storage, cgi.FieldStorage): + self.filename = self.upload_field_storage.filename + self.filename = str(datetime.datetime.utcnow()) + self.filename + self.filename = munge.munge_filename(self.filename) + self.filepath = os.path.join(self.storage_path, self.filename) + request.POST[url_field] = self.filename + self.upload_file = self.upload_field_storage.file + self.tmp_filepath = self.filepath + '~' + ### keep the file if there has been no change + if self.old_filename and not self.url and not self.clear: + request.POST[url_field] = self.old_filename + + + def upload(self): + + if self.filename: + output_file = open(self.tmp_filepath, 'wb') + self.upload_file.seek(0) + while True: + data = self.upload_file.read(2 ** 20) #mb chuncks + if not data: + break + output_file.write(data) + output_file.close() + os.rename(self.tmp_filepath, self.filepath) + self.clear = True + elif self.url: + self.clear = True + + if (self.clear and self.old_filename + and not self.old_filename.startswith('http')): + try: + os.remove(self.old_filepath) + except OSError, e: + pass diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f8fe8df7fbd..90465b7f7ec 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -262,6 +262,7 @@ def default_group_schema(): 'title': [ignore_missing, unicode], 'description': [ignore_missing, unicode], 'image_url': [ignore_missing, unicode], + 'image_display_url': [ignore_missing, unicode], 'type': [ignore_missing, unicode], 'state': [ignore_not_group_admin, ignore_missing], 'created': [ignore], diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html index 6d4d3216b26..1b9cac179c7 100644 --- a/ckan/templates/group/snippets/group_form.html +++ b/ckan/templates/group/snippets/group_form.html @@ -1,6 +1,6 @@ {% import 'macros/form.html' as form %} -
      + {% block error_summary %} {{ form.errors(error_summary) }} {% endblock %} @@ -21,6 +21,15 @@ {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + + {% if h.uploads_enabled() %} + {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} + {% set upload_file = data.get('upload_file', '') %} + {% if upload_file %} + {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% endif %} + {% endif %} + {% endblock %} {% block custom_fields %} diff --git a/ckan/templates/group/snippets/group_item.html b/ckan/templates/group/snippets/group_item.html index 9e32ce86e6a..34688b57c96 100644 --- a/ckan/templates/group/snippets/group_item.html +++ b/ckan/templates/group/snippets/group_item.html @@ -14,7 +14,7 @@ {% set url = h.url_for(group.type ~ '_read', action='read', id=group.name) %}
    • {% block image %} - {{ group.name }} + {{ group.name }} {% endblock %} {% block title %}

      {{ group.display_name }}

      diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 7b05e7d1542..7aee9ded32f 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -1,6 +1,6 @@ {% import 'macros/form.html' as form %} - + {% block error_summary %} {{ form.errors(error_summary) }} {% endblock %} @@ -21,6 +21,14 @@ {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {% if h.uploads_enabled() %} + {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} + {% set upload_file = data.get('upload_file', '') %} + {% if upload_file %} + {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% endif %} + {% endif %} + {% endblock %} {% block custom_fields %} diff --git a/ckan/templates/organization/snippets/organization_item.html b/ckan/templates/organization/snippets/organization_item.html index 818d5cc338a..8118fa9a1c6 100644 --- a/ckan/templates/organization/snippets/organization_item.html +++ b/ckan/templates/organization/snippets/organization_item.html @@ -14,7 +14,7 @@ {% set url = h.url_for(organization.type ~ '_read', action='read', id=organization.name) %}
    • {% block image %} - {{ organization.name }} + {{ organization.name }} {% endblock %} {% block title %}

      {{ organization.display_name }}

      diff --git a/ckan/templates/snippets/group.html b/ckan/templates/snippets/group.html index db549fda75a..db758d9bc96 100644 --- a/ckan/templates/snippets/group.html +++ b/ckan/templates/snippets/group.html @@ -14,7 +14,7 @@
      - {{ group.name }} + {{ group.name }}

      {{ group.title or group.name }}

      diff --git a/ckan/templates/snippets/group_item.html b/ckan/templates/snippets/group_item.html index 74c68fbca28..60f43f63b79 100644 --- a/ckan/templates/snippets/group_item.html +++ b/ckan/templates/snippets/group_item.html @@ -5,7 +5,7 @@ {% set truncate_title = truncate_title or 0 %} {% set title = group.title or group.name %} - {{ group.name }} + {{ group.name }}

      {{ group.title or group.name }}

      {% if group.description %} diff --git a/ckan/templates/snippets/organization.html b/ckan/templates/snippets/organization.html index ef08a9ed440..e0f5417024a 100644 --- a/ckan/templates/snippets/organization.html +++ b/ckan/templates/snippets/organization.html @@ -23,7 +23,7 @@

      {{ _('Organization') }} {% block image %}
      - {{ organization.name }} + {{ organization.name }}
      {% endblock %} diff --git a/ckan/templates/snippets/organization_item.html b/ckan/templates/snippets/organization_item.html index 0213723defa..06579756f32 100644 --- a/ckan/templates/snippets/organization_item.html +++ b/ckan/templates/snippets/organization_item.html @@ -3,7 +3,7 @@ {% set url=h.url_for(controller='organization', action='read', id=organization.name) %} {% set truncate=truncate or 0 %} - {{ organization.name }} + {{ organization.name }}

      {{ organization.title or organization.name }}

      {% if organization.description %} From 70bff16b4408e614933cfaf42fdbf4d2a774ad94 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 7 Oct 2013 15:43:45 +0200 Subject: [PATCH 158/269] [#772] Clarify plugin-loading order and best practice --- doc/configuration.rst | 27 +++++++++++++++++++++++++-- doc/extensions/best-practices.rst | 21 ++++++++++++++++----- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index 8d5e3e56946..2a6e3aa592e 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -508,9 +508,32 @@ Specify which CKAN plugins are to be enabled. Format as a space-separated list of the plugin names. The plugin name is the key in the ``[ckan.plugins]`` section of the extension's ``setup.py``. For more information on plugins and extensions, see :doc:`extensions/index`. -The order that plugin names are specified influences the order that plugins are loaded. Plugins in separate python modules are loaded in exactly the order specified, but plugins in the same module will always be loaded in the order their classes appear in the module. +.. note:: -Plugin load order is important for plugins that add templates directories: Templates found in template directories added earlier will override templates in template directories added later. + The order of the plugin names in the configuration file influences the + order that CKAN will load the plugins in. As long as each plugin class is + implemented in a separate Python module (i.e. in a separate Python source + code file), the plugins will be loaded in the order given in the + configuration file. + + When multiple plugins are implemented in the same Python module, CKAN will + process the plugins in the order that they're given in the config file, but as + soon as it reaches one plugin from a given Python module, CKAN will load all + plugins from that Python module, in the order that the plugin classes are + defined in the module. + + For simplicity, we recommend implementing each plugin class in its own Python + module. + + Plugin loading order can be important, for example for plugins that add custom + template files: templates found in template directories added earlier will + override templates in template directories added later. + + .. todo:: + + Fix CKAN's plugin loading order to simply load all plugins in the order + they're given in the config file, regardless of which Python modules + they're implemented in. .. _ckan.datastore.enabled: diff --git a/doc/extensions/best-practices.rst b/doc/extensions/best-practices.rst index 8d95fe7ba49..ecde15a8152 100644 --- a/doc/extensions/best-practices.rst +++ b/doc/extensions/best-practices.rst @@ -1,14 +1,18 @@ -------------------------------------- +===================================== Best practices for writing extensions -------------------------------------- +===================================== + +------------------------------ Follow CKAN's coding standards -============================== +------------------------------ See :ref:`coding standards`. + +------------------------------------------------- Use the plugins toolkit instead of importing CKAN -================================================= +------------------------------------------------- Try to limit your extension to interacting with CKAN only through CKAN's :doc:`plugin interfaces ` and @@ -17,10 +21,17 @@ extension code separate from CKAN as much as possible, so that internal changes in CKAN from one release to the next don't break your extension. +--------------------------------- Don't edit CKAN's database tables -================================= +--------------------------------- An extension can create its own tables in the CKAN database, but it should *not* write to core CKAN tables directly, add columns to core tables, or use foreign keys against core tables. + +------------------------------------------------------- +Implement each plugin class in a separate Python module +------------------------------------------------------- + +This keeps CKAN's plugin loading order simple, see :ref:`ckan.plugins`. From 4b2c46b59bd6cb5ed1235b60e1555d776c5456b2 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 7 Oct 2013 17:46:18 +0100 Subject: [PATCH 159/269] [#1265] Keep visibility value on form errors The wrong value was checked to choose the selected option. --- ckan/templates/package/snippets/package_basic_fields.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/templates/package/snippets/package_basic_fields.html b/ckan/templates/package/snippets/package_basic_fields.html index 044b9ebe5bc..d1ac7e25577 100644 --- a/ckan/templates/package/snippets/package_basic_fields.html +++ b/ckan/templates/package/snippets/package_basic_fields.html @@ -86,8 +86,8 @@
      From dad667eb4233b00561167db2aff41092afecfd2d Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Mon, 7 Oct 2013 13:35:25 -0700 Subject: [PATCH 160/269] Correct encoding to make coveralls work --- ckan/config/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 49be9b726a6..409f77435a4 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -159,7 +159,7 @@ def find_controller(self, controller): ''' This code is based on Genshi code - Copyright © 2006-2012 Edgewall Software + Copyright © 2006-2012 Edgewall Software All rights reserved. Redistribution and use in source and binary forms, with or From a903ef999d8d67db48118ea8413595707c2e3080 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Mon, 7 Oct 2013 14:20:57 -0700 Subject: [PATCH 161/269] Add test coverage banner to readme --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 93d80e668cc..2eafd662e07 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,10 @@ CKAN: The Open Source Data Portal Software :target: http://travis-ci.org/okfn/ckan :alt: Build Status +.. image:: https://coveralls.io/repos/okfn/ckan/badge.png?branch=coveralls + :target: https://coveralls.io/r/okfn/ckan?branch=coveralls + :alt: Test coverage + **CKAN is the world’s leading open-source data portal platform**. CKAN makes it easy to publish, share and work with data. It's a data management system that provides a powerful platform for cataloging, storing and accessing From 5ad2c9bc8d651c31b302499780b6926b7a798ebc Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Mon, 7 Oct 2013 14:25:57 -0700 Subject: [PATCH 162/269] Don't cover tests files --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 28f34e39273..ed2c3b6bc6f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,3 @@ [run] -omit = /ckan/migration/* +omit = /ckan/migration/*, /ckan/tests/*, */tests/* source = ckan, ckanext From 66105d360e010114751ff0280b07196e6f73445a Mon Sep 17 00:00:00 2001 From: kindly Date: Tue, 8 Oct 2013 03:02:12 +0100 Subject: [PATCH 163/269] [#1262] fix how url field is handled --- ckan/config/middleware.py | 9 ++++++++- ckan/controllers/group.py | 7 ------- ckan/lib/dictization/model_dictize.py | 20 +++++++++---------- ckan/lib/uploader.py | 16 +++++++-------- ckan/templates/group/snippets/group_form.html | 7 +++---- ckan/templates/group/snippets/info.html | 2 +- 6 files changed, 29 insertions(+), 32 deletions(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 63f5568cd5f..7699596138d 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -4,6 +4,7 @@ import logging import json import hashlib +import os import sqlalchemy as sa from beaker.middleware import CacheMiddleware, SessionMiddleware @@ -149,7 +150,13 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): storage_directory = config.get('ckan.storage_path') if storage_directory: - storage_app = StaticURLParser(storage_directory, + path = os.path.join(storage_directory, 'storage') + try: + os.makedirs(path) + except OSError, e: + pass + + storage_app = StaticURLParser(path, cache_max_age=static_max_age) static_parsers.insert(0, storage_app) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index a11dd03cc40..ed12edcb082 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -452,17 +452,10 @@ def edit(self, id, data=None, errors=None, error_summary=None): return self._save_edit(id, context) try: - ## only needs removing when form repopulates - if data and not data.get('image_url', '').startswith('http'): - data.pop('image_url', None) old_data = self._action('group_show')(context, data_dict) c.grouptitle = old_data.get('title') c.groupname = old_data.get('name') data = data or old_data - image_url = data.get('image_url') - if image_url and not image_url.startswith('http'): - data['upload_file'] = image_url - data['image_url'] = '' except NotFound: abort(404, _('Group not found')) except NotAuthorized: diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index 98756abc2ec..019e648140b 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -365,16 +365,16 @@ def group_dictize(group, context): for item in plugins.PluginImplementations(plugin): result_dict = item.before_view(result_dict) - if not context.get('for_edit'): - image_url = result_dict.get('image_url') - result_dict['image_display_url'] = image_url - if image_url and not image_url.startswith('http'): - #munge here should not have an effect only doing it incase - #of potential vulnerability of dodgy api input - image_url = munge.munge_filename(image_url) - result_dict['image_display_url'] = h.url_for_static( - 'uploads/group/%s' % result_dict.get('image_url') - ) + image_url = result_dict.get('image_url') + result_dict['image_display_url'] = image_url + if image_url and not image_url.startswith('http'): + #munge here should not have an effect only doing it incase + #of potential vulnerability of dodgy api input + image_url = munge.munge_filename(image_url) + result_dict['image_display_url'] = h.url_for_static( + 'uploads/group/%s' % result_dict.get('image_url'), + qualified = True + ) return result_dict def tag_list_dictize(tag_list, context): diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index 5fc144e3aef..bcfdd7e9ecc 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -7,7 +7,7 @@ class Upload(object): def __init__(self, object_type, old_filename=None): path = pylons.config.get('ckan.storage_path', '/tmp') - self.storage_path = os.path.join(path, 'uploads', object_type) + self.storage_path = os.path.join(path, 'storage', 'uploads', object_type) try: os.makedirs(self.storage_path) except OSError, e: @@ -20,9 +20,7 @@ def __init__(self, object_type, old_filename=None): self.filepath = None def register_request(self, request, url_field, file_field, clear_field): - - self.request = request - self.url = request.POST.get(url_field) + self.url = request.POST.get(url_field, '') self.clear = request.POST.get(clear_field) self.upload_field_storage = request.POST.pop(file_field, None) @@ -35,12 +33,14 @@ def register_request(self, request, url_field, file_field, clear_field): self.upload_file = self.upload_field_storage.file self.tmp_filepath = self.filepath + '~' ### keep the file if there has been no change - if self.old_filename and not self.url and not self.clear: - request.POST[url_field] = self.old_filename + elif self.old_filename and not self.old_filename.startswith('http'): + if not self.clear: + request.POST[url_field] = self.old_filename + if self.clear and self.url == self.old_filename: + request.POST[url_field] = '' def upload(self): - if self.filename: output_file = open(self.tmp_filepath, 'wb') self.upload_file.seek(0) @@ -52,8 +52,6 @@ def upload(self): output_file.close() os.rename(self.tmp_filepath, self.filepath) self.clear = True - elif self.url: - self.clear = True if (self.clear and self.old_filename and not self.old_filename.startswith('http')): diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html index 1b9cac179c7..322f9ebff8a 100644 --- a/ckan/templates/group/snippets/group_form.html +++ b/ckan/templates/group/snippets/group_form.html @@ -19,14 +19,13 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my group...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} {% if h.uploads_enabled() %} {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% set upload_file = data.get('upload_file', '') %} - {% if upload_file %} - {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% if data.image_url and not data.image_url.startswith('http') %} + {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} {% endif %} {% endif %} diff --git a/ckan/templates/group/snippets/info.html b/ckan/templates/group/snippets/info.html index b7c749ab147..dc86036bb4c 100644 --- a/ckan/templates/group/snippets/info.html +++ b/ckan/templates/group/snippets/info.html @@ -2,7 +2,7 @@
      - {{ group.name }} + {{ group.name }}

      {{ group.display_name }}

      From 1ab6ff449faef790a4f160d30aedd08a82b726fb Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 8 Oct 2013 13:06:11 +0100 Subject: [PATCH 164/269] [#1268] Add 'python setup.py develop' step to source upgrade instructions --- doc/upgrade-source.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/upgrade-source.rst b/doc/upgrade-source.rst index af70ac4ea05..539b96dcb62 100644 --- a/doc/upgrade-source.rst +++ b/doc/upgrade-source.rst @@ -37,6 +37,12 @@ CKAN release you're upgrading to: pip install --upgrade -r requirements.txt +#. Register any new or updated plugins: + + :: + + python setup.py develop + #. If you are upgrading to a new :ref:`major release ` you need to update your Solr schema symlink. From 23cfdbee2fcbc9267f66f06b6ecf9a879c92241f Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 8 Oct 2013 16:44:23 +0100 Subject: [PATCH 165/269] [#1269] First working version of the proper IA for resources --- ckan/config/routing.py | 7 ++-- ckan/templates/package/edit.html | 29 ++------------ ckan/templates/package/edit_base.html | 23 +++++++++++ ckan/templates/package/new_resource.html | 29 ++++---------- ckan/templates/package/read_base.html | 15 +------ ckan/templates/package/resource_edit.html | 25 ++---------- .../templates/package/resource_edit_base.html | 28 +++++++++++++ ckan/templates/package/snippets/info.html | 40 ++++++++----------- 8 files changed, 85 insertions(+), 111 deletions(-) create mode 100644 ckan/templates/package/edit_base.html create mode 100644 ckan/templates/package/resource_edit_base.html diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 0fbb02bb886..33175a35c12 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -222,7 +222,6 @@ def make_map(): ]))) m.connect('/dataset/{action}/{id}', requirements=dict(action='|'.join([ - 'edit', 'new_metadata', 'new_resource', 'history', @@ -234,6 +233,8 @@ def make_map(): 'delete', 'api_data', ]))) + m.connect('dataset_edit', '/dataset/edit/{id}', action='edit', + ckan_icon='edit') m.connect('dataset_followers', '/dataset/followers/{id}', action='followers', ckan_icon='group') m.connect('dataset_activity', '/dataset/activity/{id}', @@ -246,8 +247,8 @@ def make_map(): action='resource_read') m.connect('/dataset/{id}/resource_delete/{resource_id}', action='resource_delete') - m.connect('/dataset/{id}/resource_edit/{resource_id}', - action='resource_edit') + m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}', + action='resource_edit', ckan_icon='edit') m.connect('/dataset/{id}/resource/{resource_id}/download', action='resource_download') m.connect('/dataset/{id}/resource/{resource_id}/embed', diff --git a/ckan/templates/package/edit.html b/ckan/templates/package/edit.html index 4f15985d033..dec84769ac4 100644 --- a/ckan/templates/package/edit.html +++ b/ckan/templates/package/edit.html @@ -1,28 +1,5 @@ -{% extends 'package/base_form_page.html' %} +{% extends 'package/edit_base.html' %} -{% set pkg = c.pkg_dict %} - -{% block breadcrumb_content_selected %}{% endblock %} - -{% block breadcrumb_content %} - {{ super() }} -
    • {% link_for _('Edit'), controller='package', action='edit', id=pkg.name %}
      • -{% endblock %} - -{% block resources_module %} - {% snippet 'package/snippets/resources.html', pkg=pkg, action='resource_edit' %} -{% endblock %} - -{% block content_action %} - {% link_for _('View dataset'), controller='package', action='read', id=pkg.name, class_='btn', icon='eye-open' %} +{% block primary_content_inner %} + {% block form %}{{ c.form | safe }}{% endblock %} {% endblock %} - -{% block secondary_content %} - {% snippet 'package/snippets/info.html', pkg=pkg, action='package_edit' %} -{% endblock %} - -{% block primary_content %} -
      - {{ super() }} -{% endblock %} - diff --git a/ckan/templates/package/edit_base.html b/ckan/templates/package/edit_base.html new file mode 100644 index 00000000000..df1d91dfb02 --- /dev/null +++ b/ckan/templates/package/edit_base.html @@ -0,0 +1,23 @@ +{% extends 'package/base.html' %} + +{% set pkg = c.pkg_dict %} + +{% block breadcrumb_content_selected %}{% endblock %} + +{% block breadcrumb_content %} + {{ super() }} +
    • {% link_for _('Edit'), controller='package', action='edit', id=pkg.name %}
      • +{% endblock %} + +{% block content_action %} + {% link_for _('View dataset'), controller='package', action='read', id=pkg.name, class_='btn', icon='eye-open' %} +{% endblock %} + +{% block content_primary_nav %} + {{ h.build_nav_icon('dataset_edit', _('Edit'), id=pkg.name) }} + {#{{ h.build_nav_icon('dataset_resources', _('Resources'), id=pkg.name) }}#} +{% endblock %} + +{% block secondary_content %} + {% snippet 'package/snippets/info.html', pkg=pkg, hide_follow_button=true %} +{% endblock %} diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html index 13cce195453..49d02387768 100644 --- a/ckan/templates/package/new_resource.html +++ b/ckan/templates/package/new_resource.html @@ -1,10 +1,6 @@ {% extends "package/base_form_page.html" %} -{% if c.userobj %} - {% set logged_in = true %} -{% else %} - {% set logged_in = false %} -{% endif %} +{% set logged_in = true if c.userobj else false %} {% block subtitle %}{{ _('Add data to the dataset') }}{% endblock %} @@ -17,26 +13,15 @@ {% block form %}{% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage, allow_upload=g.ofs_impl and logged_in %}{% endblock %} {% block secondary_content %} - {% if pkg_dict and pkg_dict.state != 'draft' %} - {% snippet 'package/snippets/info.html', pkg=pkg_dict, action='resource_new' %} - {% else %} -
      -

      {{ _('What\'s a resource?') }}

      -
      -

      {{ _('A resource can be any file or link to a file containing useful data.') }}

      -
      -
      - {% endif %} +
      +

      {{ _('What\'s a resource?') }}

      +
      +

      {{ _('A resource can be any file or link to a file containing useful data.') }}

      +
      +
      {% endblock %} {% block scripts %} {{ super() }} {% resource 'vendor/fileupload' %} {% endblock %} - -{% block primary_content %} - {% if pkg_dict and pkg_dict.state != 'draft' %} -
      - {% endif %} - {{ super() }} -{% endblock %} diff --git a/ckan/templates/package/read_base.html b/ckan/templates/package/read_base.html index 9cf73a5c5a1..f0a58a21018 100644 --- a/ckan/templates/package/read_base.html +++ b/ckan/templates/package/read_base.html @@ -50,20 +50,7 @@ {% block secondary_help_content %}{% endblock %} {% block package_info %} -
      -
      -

      {{ pkg.title or pkg.name }}

      -
      -
      -
      {{ _('Followers') }}
      -
      {{ h.SI_number_span(h.get_action('dataset_follower_count', {'id': pkg.id})) }}
      -
      -
      - -
      -
      + {% snippet 'package/snippets/info.html', pkg=pkg %} {% endblock %} {% block package_organization %} diff --git a/ckan/templates/package/resource_edit.html b/ckan/templates/package/resource_edit.html index 52ac2ab4722..ab4b9aaa606 100644 --- a/ckan/templates/package/resource_edit.html +++ b/ckan/templates/package/resource_edit.html @@ -1,24 +1,5 @@ -{% extends "package/new_resource.html" %} +{% extends "package/resource_edit_base.html" %} -{% set pkg_dict = c.pkg_dict %} -{% set res = c.resource %} - -{% block subtitle %}{{ h.dataset_display_name(pkg_dict) }} - {{ h.resource_display_name(res) }}{% endblock %} - -{% block breadcrumb_content_selected %}{% endblock %} - -{% block breadcrumb_content %} - {{ super() }} -
    • {{ _('Edit') }}
      • -{% endblock %} - -{% block content_action %} - {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %} -{% endblock %} - -{# logged_in is defined in new_resource.html #} -{% block form %}{{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg_dict.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }}{% endblock %} - -{% block secondary_content %} - {% snippet 'package/snippets/info.html', pkg=pkg_dict, active=data.id, action='resource_edit' %} +{% block primary_content_inner %} + {{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg_dict.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }} {% endblock %} diff --git a/ckan/templates/package/resource_edit_base.html b/ckan/templates/package/resource_edit_base.html new file mode 100644 index 00000000000..fba67b43c80 --- /dev/null +++ b/ckan/templates/package/resource_edit_base.html @@ -0,0 +1,28 @@ +{% extends "package/base.html" %} + +{% set logged_in = true if c.userobj else false %} +{% set pkg_dict = c.pkg_dict %} +{% set res = c.resource %} + +{% block subtitle %}{{ _('Edit') }} - {{ h.resource_display_name(res) }} - {{ h.dataset_display_name(pkg_dict) }}{% endblock %} + +{% block breadcrumb_content_selected %}{% endblock %} + +{% block breadcrumb_content %} + {{ super() }} +
    • {% link_for h.resource_display_name(res)|truncate(30), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id %}
      • +
    • {{ _('Edit') }}
      • +{% endblock %} + +{% block content_action %} + {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %} +{% endblock %} + +{% block content_primary_nav %} + {{ h.build_nav_icon('resource_edit', _('Edit'), id=pkg.name, resource_id=res.id) }} + {#{{ h.build_nav_icon('dataset_resources', _('Resources'), id=pkg.name) }}#} +{% endblock %} + +{% block secondary_content %} + {% snippet 'package/snippets/info.html', pkg=pkg, hide_follow_button=true %} +{% endblock %} diff --git a/ckan/templates/package/snippets/info.html b/ckan/templates/package/snippets/info.html index 49efa7b8fc7..b12ce3f6998 100644 --- a/ckan/templates/package/snippets/info.html +++ b/ckan/templates/package/snippets/info.html @@ -2,33 +2,25 @@ Displays a sidebard module with information for given package pkg - The package dict that owns the resources. -active - The active resource. -action - The action that this is coming from. Example: {% snippet "package/snippets/info.html", pkg=pkg %} #} -{% if pkg and h.check_access('package_update', {'id':pkg.id }) %} -
      -

      {{ _("Edit Dataset") }}

      -
        -
      • - {% link_for h.dataset_display_name(pkg)|truncate(30), controller='package', action='edit', id=pkg.name %} -
        • -
      - {% set resources = pkg.resources or [] %} -

      {{ _("Edit Resources") }}

      -
        - {% block package_resource_list %} - {% for resource in resources %} -
      • - {% link_for h.resource_display_name(resource)|truncate(30), controller='package', action='resource_edit', id=pkg.name, resource_id=resource.id, inner_span=true %} -
        • - {% endfor %} - {% endblock %} -
      • {{ _('Add New Resource') }}
        • -
      -
      -{% endif %} +
      +
      +

      {{ pkg.title or pkg.name }}

      +
      +
      +
      {{ _('Followers') }}
      +
      {{ h.SI_number_span(h.get_action('dataset_follower_count', {'id': pkg.id})) }}
      +
      +
      + {% if not hide_follow_button %} + + {% endif %} +
      +
      From 1b62423d9d013dbc4481bbddd5a1037b49e52529 Mon Sep 17 00:00:00 2001 From: kindly Date: Wed, 9 Oct 2013 03:12:00 +0100 Subject: [PATCH 166/269] [1262] use logic instead of requests --- ckan/controllers/group.py | 11 ----------- ckan/lib/uploader.py | 14 +++++++------- ckan/logic/action/create.py | 7 ++++++- ckan/logic/action/update.py | 7 +++++++ .../organization/snippets/organization_form.html | 7 +++---- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index ed12edcb082..adbd3d7cacc 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -12,7 +12,6 @@ import ckan.lib.base as base import ckan.lib.helpers as h import ckan.lib.maintain as maintain -import ckan.lib.uploader as uploader import ckan.lib.navl.dictization_functions as dict_fns import ckan.logic as logic import ckan.lib.search as search @@ -491,17 +490,12 @@ def _get_group_type(self, id): def _save_new(self, context, group_type=None): try: - upload = uploader.Upload('group') - upload.register_request(request, 'image_url', - 'image_upload', 'clear_upload') - data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) data_dict['type'] = group_type or 'group' context['message'] = data_dict.get('log_message', '') data_dict['users'] = [{'name': c.user, 'capacity': 'admin'}] group = self._action('group_create')(context, data_dict) - upload.upload() # Redirect to the appropriate _read route for the type of group h.redirect_to(group['type'] + '_read', id=group['name']) @@ -529,17 +523,12 @@ def _save_edit(self, id, context): try: old_group = self._action('group_show')(context, {"id": id}) old_image_url = old_group.get('image_url') - upload = uploader.Upload('group', old_image_url) - upload.register_request(request, 'image_url', - 'image_upload', 'clear_upload') - data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) context['message'] = data_dict.get('log_message', '') data_dict['id'] = id context['allow_partial_update'] = True group = self._action('group_update')(context, data_dict) - upload.upload() if id != group['name']: self._force_reindex(group) diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index bcfdd7e9ecc..9b7cea2922e 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -19,25 +19,25 @@ def __init__(self, object_type, old_filename=None): self.filename = None self.filepath = None - def register_request(self, request, url_field, file_field, clear_field): - self.url = request.POST.get(url_field, '') - self.clear = request.POST.get(clear_field) - self.upload_field_storage = request.POST.pop(file_field, None) + def update_data_dict(self, data_dict, url_field, file_field, clear_field): + self.url = data_dict.get(url_field, '') + self.clear = data_dict.pop(clear_field, None) + self.upload_field_storage = data_dict.pop(file_field, None) if isinstance(self.upload_field_storage, cgi.FieldStorage): self.filename = self.upload_field_storage.filename self.filename = str(datetime.datetime.utcnow()) + self.filename self.filename = munge.munge_filename(self.filename) self.filepath = os.path.join(self.storage_path, self.filename) - request.POST[url_field] = self.filename + data_dict[url_field] = self.filename self.upload_file = self.upload_field_storage.file self.tmp_filepath = self.filepath + '~' ### keep the file if there has been no change elif self.old_filename and not self.old_filename.startswith('http'): if not self.clear: - request.POST[url_field] = self.old_filename + data_dict[url_field] = self.old_filename if self.clear and self.url == self.old_filename: - request.POST[url_field] = '' + data_dict[url_field] = '' def upload(self): diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index baac5a2154c..702a6028d99 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -15,6 +15,7 @@ import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.dictization.model_save as model_save import ckan.lib.navl.dictization_functions +import ckan.lib.uploader as uploader from ckan.common import _ @@ -446,6 +447,7 @@ def member_create(context, data_dict=None): if not obj: raise NotFound('%s was not found.' % obj_type.title()) + # User must be able to update the group to add a member to it _check_access('group_update', context, data_dict) @@ -475,7 +477,9 @@ def _group_or_org_create(context, data_dict, is_org=False): parent = context.get('parent', None) data_dict['is_organization'] = is_org - + upload = uploader.Upload('group') + upload.update_data_dict(data_dict, 'image_url', + 'image_upload', 'clear_upload') # get the schema group_plugin = lib_plugins.lookup_group_plugin( group_type=data_dict.get('type')) @@ -561,6 +565,7 @@ def _group_or_org_create(context, data_dict, is_org=False): logic.get_action('activity_create')(activity_create_context, activity_dict) + upload.upload() if not context.get('defer_commit'): model.repo.commit() context["group"] = group diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 3db2d0d19cf..f8aee9f62fb 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -19,6 +19,7 @@ import ckan.lib.plugins as lib_plugins import ckan.lib.email_notifications as email_notifications import ckan.lib.search as search +import ckan.lib.uploader as uploader from ckan.common import _, request @@ -424,6 +425,10 @@ def _group_or_org_update(context, data_dict, is_org=False): except AttributeError: schema = group_plugin.form_to_db_schema() + upload = uploader.Upload('group', group.image_url) + upload.update_data_dict(data_dict, 'image_url', + 'image_upload', 'clear_upload') + if is_org: _check_access('organization_update', context, data_dict) else: @@ -528,9 +533,11 @@ def _group_or_org_update(context, data_dict, is_org=False): # TODO: Also create an activity detail recording what exactly changed # in the group. + upload.upload() if not context.get('defer_commit'): model.repo.commit() + return model_dictize.group_dictize(group, context) def group_update(context, data_dict): diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 7aee9ded32f..0a2134e89f7 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -19,13 +19,12 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my organization...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} {% if h.uploads_enabled() %} {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% set upload_file = data.get('upload_file', '') %} - {% if upload_file %} - {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% if data.image_url and not data.image_url.startswith('http') %} + {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} {% endif %} {% endif %} From 8f8d2ab1a502100d7da4358ea2947fbf9a5c4adb Mon Sep 17 00:00:00 2001 From: kindly Date: Wed, 9 Oct 2013 11:28:26 +0100 Subject: [PATCH 167/269] 1200 make sure all users can submit job --- ckanext/datapusher/logic/action.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 66c56d22c82..cfbd4620d09 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -71,6 +71,7 @@ def datapusher_submit(context, data_dict): except logic.NotFound: pass + context['ignore_auth'] = True result = p.toolkit.get_action('task_status_update')(context, task) task_id = result['id'] From 48041a4e40e837f3f84ea9bb4f7cd94918daa21a Mon Sep 17 00:00:00 2001 From: John Martin Date: Wed, 9 Oct 2013 12:03:04 +0100 Subject: [PATCH 168/269] [#1269] Adds resource listing page within edit dataset section - Also tidys up the edit dataset templats - Adds a new state to the new resource action that uses a different template when dataset is no in draft mode (e.g. it's not part of the dataset wizard creation any more) --- ckan/config/routing.py | 2 ++ ckan/controllers/package.py | 30 ++++++++++++++++++- ckan/templates/package/edit.html | 2 ++ ckan/templates/package/edit_base.html | 4 +-- .../package/new_resource_not_draft.html | 8 +++++ ckan/templates/package/resource_edit.html | 8 +++-- .../templates/package/resource_edit_base.html | 28 +++++------------ ckan/templates/package/resources.html | 21 +++++++++++++ .../package/snippets/resource_item.html | 15 +++++++++- 9 files changed, 92 insertions(+), 26 deletions(-) create mode 100644 ckan/templates/package/new_resource_not_draft.html create mode 100644 ckan/templates/package/resources.html diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 33175a35c12..9ef4e0895eb 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -241,6 +241,8 @@ def make_map(): action='activity', ckan_icon='time') m.connect('/dataset/activity/{id}/{offset}', action='activity') m.connect('/dataset/{id}.{format}', action='read') + m.connect('dataset_resources', '/dataset/resources/{id}', + action='resources', ckan_icon='reorder') m.connect('dataset_read', '/dataset/{id}', action='read', ckan_icon='sitemap') m.connect('/dataset/{id}/resource/{resource_id}', diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index d1852cd95b7..8468dfa3947 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -301,6 +301,31 @@ def _content_type_from_accept(self): ct, mu, ext = accept.parse_header(request.headers.get('Accept', '')) return ct, ext, (NewTextTemplate, MarkupTemplate)[mu] + def resources(self, id): + package_type = self._get_package_type(id.split('@')[0]) + context = {'model': model, 'session': model.Session, + 'user': c.user or c.author, 'for_view': True, + 'auth_user_obj': c.userobj} + data_dict = {'id': id} + + try: + check_access('package_update', context) + except NotAuthorized, e: + abort(401, _('User %r not authorized to edit %s') % (c.user, id)) + # check if package exists + try: + c.pkg_dict = get_action('package_show')(context, data_dict) + c.pkg = context['package'] + except NotFound: + abort(404, _('Dataset not found')) + except NotAuthorized: + abort(401, _('Unauthorized to read package %s') % id) + + self._setup_template_variables(context, {'id': id}, + package_type=package_type) + + return render('package/resources.html') + def read(self, id, format='html'): if not format == 'html': ctype, extension, loader = \ @@ -666,11 +691,14 @@ def new_resource(self, id, data=None, errors=None, error_summary=None): abort(404, _('The dataset {id} could not be found.').format(id=id)) # required for nav menu vars['pkg_dict'] = pkg_dict + template = 'package/new_resource_not_draft.html' if pkg_dict['state'] == 'draft': vars['stage'] = ['complete', 'active'] + template = 'package/new_resource.html' elif pkg_dict['state'] == 'draft-complete': vars['stage'] = ['complete', 'active', 'complete'] - return render('package/new_resource.html', extra_vars=vars) + template = 'package/new_resource.html' + return render(template, extra_vars=vars) def new_metadata(self, id, data=None, errors=None, error_summary=None): ''' FIXME: This is a temporary action to allow styling of the diff --git a/ckan/templates/package/edit.html b/ckan/templates/package/edit.html index dec84769ac4..565623f7ebd 100644 --- a/ckan/templates/package/edit.html +++ b/ckan/templates/package/edit.html @@ -1,5 +1,7 @@ {% extends 'package/edit_base.html' %} +{% block subtitle %}{{ _('Edit') }} - {{ h.dataset_display_name(pkg) }}{% endblock %} + {% block primary_content_inner %} {% block form %}{{ c.form | safe }}{% endblock %} {% endblock %} diff --git a/ckan/templates/package/edit_base.html b/ckan/templates/package/edit_base.html index df1d91dfb02..e8b785712b3 100644 --- a/ckan/templates/package/edit_base.html +++ b/ckan/templates/package/edit_base.html @@ -14,8 +14,8 @@ {% endblock %} {% block content_primary_nav %} - {{ h.build_nav_icon('dataset_edit', _('Edit'), id=pkg.name) }} - {#{{ h.build_nav_icon('dataset_resources', _('Resources'), id=pkg.name) }}#} + {{ h.build_nav_icon('dataset_edit', _('Edit metadata'), id=pkg.name) }} + {{ h.build_nav_icon('dataset_resources', _('Resources'), id=pkg.name) }} {% endblock %} {% block secondary_content %} diff --git a/ckan/templates/package/new_resource_not_draft.html b/ckan/templates/package/new_resource_not_draft.html new file mode 100644 index 00000000000..a24f37ce459 --- /dev/null +++ b/ckan/templates/package/new_resource_not_draft.html @@ -0,0 +1,8 @@ +{% extends "package/resource_edit_base.html" %} + +{% block subtitle %}{{ _('Add resource') }} - {{ h.dataset_display_name(pkg) }}{% endblock %} +{% block form_title %}{{ _('Add resource') }}{% endblock %} + +{% block form %} + {% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage, allow_upload=g.ofs_impl and logged_in %} +{% endblock %} diff --git a/ckan/templates/package/resource_edit.html b/ckan/templates/package/resource_edit.html index ab4b9aaa606..9756c8f9581 100644 --- a/ckan/templates/package/resource_edit.html +++ b/ckan/templates/package/resource_edit.html @@ -1,5 +1,9 @@ {% extends "package/resource_edit_base.html" %} -{% block primary_content_inner %} - {{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg_dict.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }} +{% set res = c.resource %} + +{% block subtitle %}{{ _('Edit') }} - {{ h.resource_display_name(res) }} - {{ h.dataset_display_name(pkg) }}{% endblock %} + +{% block form %} + {{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }} {% endblock %} diff --git a/ckan/templates/package/resource_edit_base.html b/ckan/templates/package/resource_edit_base.html index fba67b43c80..7c3a8d012ec 100644 --- a/ckan/templates/package/resource_edit_base.html +++ b/ckan/templates/package/resource_edit_base.html @@ -1,28 +1,16 @@ -{% extends "package/base.html" %} +{% extends "package/edit_base.html" %} {% set logged_in = true if c.userobj else false %} -{% set pkg_dict = c.pkg_dict %} -{% set res = c.resource %} - -{% block subtitle %}{{ _('Edit') }} - {{ h.resource_display_name(res) }} - {{ h.dataset_display_name(pkg_dict) }}{% endblock %} {% block breadcrumb_content_selected %}{% endblock %} -{% block breadcrumb_content %} - {{ super() }} -
    • {% link_for h.resource_display_name(res)|truncate(30), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id %}
      • -
    • {{ _('Edit') }}
      • -{% endblock %} - -{% block content_action %} - {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %} +{% block primary_content_inner %} + {% link_for _('View all resources'), controller='package', action='resources', id=pkg.name, class_='btn pull-right', icon='arrow-left' %} +

      {% block form_title %}{{ _('Edit resource') }}{% endblock %}

      + {% block form %}{% endblock %} {% endblock %} -{% block content_primary_nav %} - {{ h.build_nav_icon('resource_edit', _('Edit'), id=pkg.name, resource_id=res.id) }} - {#{{ h.build_nav_icon('dataset_resources', _('Resources'), id=pkg.name) }}#} -{% endblock %} - -{% block secondary_content %} - {% snippet 'package/snippets/info.html', pkg=pkg, hide_follow_button=true %} +{% block scripts %} + {{ super() }} + {% resource 'vendor/fileupload' %} {% endblock %} diff --git a/ckan/templates/package/resources.html b/ckan/templates/package/resources.html new file mode 100644 index 00000000000..e353d03f10b --- /dev/null +++ b/ckan/templates/package/resources.html @@ -0,0 +1,21 @@ +{% extends "package/edit_base.html" %} + +{% block subtitle %}{{ _('Resources') }} - {{ h.dataset_display_name(pkg) }}{% endblock %} + +{% block page_primary_action %} + {% link_for _('Add new resource'), controller='package', action='new_resource', id=c.pkg_dict.name, class_='btn btn-primary', icon='plus' %} +{% endblock %} + +{% block primary_content_inner %} + {% if pkg.resources %} +
        + {% for resource in pkg.resources %} + {% snippet 'package/snippets/resource_item.html', pkg=pkg, res=resource, url_is_edit=true %} + {% endfor %} +
      + {% else %} + {% trans url=h.url_for(controller='package', action='new_resource', id=pkg.name) %} +

      This dataset has no data, why not add some?

      + {% endtrans %} + {% endif %} +{% endblock %} diff --git a/ckan/templates/package/snippets/resource_item.html b/ckan/templates/package/snippets/resource_item.html index fccd94ff4d2..c5b22521d64 100644 --- a/ckan/templates/package/snippets/resource_item.html +++ b/ckan/templates/package/snippets/resource_item.html @@ -1,4 +1,7 @@ -{% set url = h.url_for(controller='package', action='resource_read', id=pkg.name, resource_id=res.id) %} +{% set can_edit = h.check_access('package_update', {'id':pkg.id }) %} +{% set url_action = 'resource_edit' if url_is_edit and can_edit else 'resource_read' %} +{% set url = h.url_for(controller='package', action=url_action, id=pkg.name, resource_id=res.id) %} +
    • {% block resource_item_title %} @@ -14,6 +17,7 @@ {% endif %}

      {% block resource_item_explore %} + {% if not url_is_edit %}
      @@ -44,8 +48,17 @@ {% endif %}
      • + {% if can_edit %} +
    • + + + {{ _('Edit') }} + +
      • + {% endif %} {% endblock %}

    + {% endif %} {% endblock %} From 485f681f6f5d5fe890dc339d57a8563844e2fde2 Mon Sep 17 00:00:00 2001 From: joetsoi Date: Wed, 9 Oct 2013 14:02:39 +0100 Subject: [PATCH 169/269] [#1272] prevent keyerror when looking up 'url_type' in datastore before_show --- ckanext/datastore/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 4477ad70526..04f201f9eb0 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -285,7 +285,7 @@ def before_show(self, resource_dict): ''' Modify the resource url of datastore resources so that they link to the datastore dumps. ''' - if resource_dict['url_type'] == 'datastore': + if resource_dict.get('url_type') == 'datastore': resource_dict['url'] = p.toolkit.url_for( controller='ckanext.datastore.controller:DatastoreController', action='dump', resource_id=resource_dict['id']) From c4d673327c6b83e01ead0091246cecb83875160a Mon Sep 17 00:00:00 2001 From: kindly Date: Thu, 10 Oct 2013 20:40:49 +0100 Subject: [PATCH 170/269] [#1262] fix test failures --- ckan/lib/uploader.py | 12 +++++++----- ckan/tests/lib/test_dictization.py | 2 ++ ckan/tests/lib/test_dictization_schema.py | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index 9b7cea2922e..eb0f28eec54 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -4,10 +4,12 @@ import datetime import ckan.lib.munge as munge + class Upload(object): def __init__(self, object_type, old_filename=None): path = pylons.config.get('ckan.storage_path', '/tmp') - self.storage_path = os.path.join(path, 'storage', 'uploads', object_type) + self.storage_path = os.path.join(path, 'storage', + 'uploads', object_type) try: os.makedirs(self.storage_path) except OSError, e: @@ -22,7 +24,7 @@ def __init__(self, object_type, old_filename=None): def update_data_dict(self, data_dict, url_field, file_field, clear_field): self.url = data_dict.get(url_field, '') self.clear = data_dict.pop(clear_field, None) - self.upload_field_storage = data_dict.pop(file_field, None) + self.upload_field_storage = data_dict.pop(file_field, None) if isinstance(self.upload_field_storage, cgi.FieldStorage): self.filename = self.upload_field_storage.filename @@ -39,13 +41,13 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): if self.clear and self.url == self.old_filename: data_dict[url_field] = '' - def upload(self): if self.filename: output_file = open(self.tmp_filepath, 'wb') self.upload_file.seek(0) while True: - data = self.upload_file.read(2 ** 20) #mb chuncks + # mb chuncks + data = self.upload_file.read(2 ** 20) if not data: break output_file.write(data) @@ -54,7 +56,7 @@ def upload(self): self.clear = True if (self.clear and self.old_filename - and not self.old_filename.startswith('http')): + and not self.old_filename.startswith('http')): try: os.remove(self.old_filepath) except OSError, e: diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py index 57f29963910..dcf955d3cc5 100644 --- a/ckan/tests/lib/test_dictization.py +++ b/ckan/tests/lib/test_dictization.py @@ -923,6 +923,7 @@ def test_16_group_dictized(self): 'capacity' : 'public', 'display_name': u'simple', 'image_url': u'', + 'image_display_url': u'', 'name': u'simple', 'packages': 0, 'state': u'active', @@ -944,6 +945,7 @@ def test_16_group_dictized(self): 'name': u'help', 'display_name': u'help', 'image_url': u'', + 'image_display_url': u'', 'package_count': 2, 'is_organization': False, 'packages': [{'author': None, diff --git a/ckan/tests/lib/test_dictization_schema.py b/ckan/tests/lib/test_dictization_schema.py index 5a94d2e54e8..9507be2806f 100644 --- a/ckan/tests/lib/test_dictization_schema.py +++ b/ckan/tests/lib/test_dictization_schema.py @@ -160,6 +160,7 @@ def test_2_group_schema(self): 'is_organization': False, 'type': u'group', 'image_url': u'', + 'image_display_url': u'', 'packages': sorted([{'id': group_pack[0].id, 'name': group_pack[0].name, 'title': group_pack[0].title}, From fc1097763c8d9bdb3be42bbd43c037aab59f1b6d Mon Sep 17 00:00:00 2001 From: kindly Date: Thu, 10 Oct 2013 20:55:23 +0100 Subject: [PATCH 171/269] [#1267] fix url for by faking routes threadlocal --- ckan/lib/cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 14c217ed096..5d13da6638b 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -10,6 +10,8 @@ import ckan.include.rcssmin as rcssmin import ckan.lib.fanstatic_resources as fanstatic_resources import sqlalchemy as sa +import urlparse +import routes import paste.script from paste.registry import Registry @@ -99,6 +101,12 @@ def _load_config(self): self.translator_obj = MockTranslator() self.registry.register(pylons.translator, self.translator_obj) + ## give routes enough information to run url_for + parsed = urlparse.urlparse(conf.get('ckan.site_url', 'http://0.0.0.0')) + request_config = routes.request_config() + request_config.host = parsed.netloc + parsed.path + request_config.protocol = parsed.scheme + def _setup_app(self): cmd = paste.script.appinstall.SetupCommand('setup-app') cmd.run([self.filename]) From 16924d937af4207a585fd34eaf2be083bcc2d6d2 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 20:23:41 -0300 Subject: [PATCH 172/269] [#1178] Remove backward migrations, as we don't support it --- .../versions/071_add_state_column_to_user_table.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ckan/migration/versions/071_add_state_column_to_user_table.py b/ckan/migration/versions/071_add_state_column_to_user_table.py index 791a046bad2..c4f3f5013aa 100644 --- a/ckan/migration/versions/071_add_state_column_to_user_table.py +++ b/ckan/migration/versions/071_add_state_column_to_user_table.py @@ -7,11 +7,3 @@ def upgrade(migrate_engine): ALTER TABLE "user" ADD COLUMN "state" text NOT NULL DEFAULT '%s' ''' % ckan.model.core.State.ACTIVE ) - - -def downgrade(migrate_engine): - migrate_engine.execute( - ''' - ALTER TABLE "user" DROP COLUMN "state" - ''' - ) From 0bd163e75eacc65fee0546fa356f195f1e109713 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 20:40:46 -0300 Subject: [PATCH 173/269] [#1178] Fix repeated test name --- ckan/tests/lib/test_authenticator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/tests/lib/test_authenticator.py b/ckan/tests/lib/test_authenticator.py index 8e7254d94c7..8b2c6139f0f 100644 --- a/ckan/tests/lib/test_authenticator.py +++ b/ckan/tests/lib/test_authenticator.py @@ -109,7 +109,7 @@ def test_authenticate_fails_if_user_is_deleted(self): assert self.authenticate(environ, identity) is None - def test_authenticate_fails_if_user_is_deleted(self): + def test_authenticate_fails_if_user_is_pending(self): environ = {} openid = 'some-openid-key' user = CreateTestData.create_user('a_user', **{'openid': openid}) From d073ce7f6b56361cf8c5452c2f969e4d45356437 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 20:54:24 -0300 Subject: [PATCH 174/269] [#1178] Use the data returned by validate, instead of the one passed as param --- ckan/logic/action/create.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 066ed7bcc87..f4ab9fa4ba1 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -865,23 +865,23 @@ def user_invite(context, data_dict): 'group_id': [validators.not_empty], 'role': [validators.not_empty], } - _, errors = _validate(data_dict, user_invite_schema, context) + data, errors = _validate(data_dict, user_invite_schema, context) if errors: raise ValidationError(errors) while True: try: - name = _get_random_username_from_email(data_dict['email']) + name = _get_random_username_from_email(data['email']) password = str(random.SystemRandom().random()) - data_dict['name'] = name - data_dict['password'] = password - data_dict['state'] = ckan.model.State.PENDING - user_dict = _get_action('user_create')(context, data_dict) + data['name'] = name + data['password'] = password + data['state'] = ckan.model.State.PENDING + user_dict = _get_action('user_create')(context, data) user = ckan.model.User.get(user_dict['id']) member_dict = { 'username': user.id, - 'id': data_dict['group_id'], - 'role': data_dict['role'] + 'id': data['group_id'], + 'role': data['role'] } _get_action('group_member_create')(context, member_dict) mailer.send_invite(user) From 1da817c40b59d7dfa866abd9e68ed844442f34d4 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 20:58:11 -0300 Subject: [PATCH 175/269] [#1178] Move user_invite_schema to ckan.logic.schema --- ckan/logic/action/create.py | 8 ++------ ckan/logic/schema.py | 8 ++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index f4ab9fa4ba1..e4f13b3baad 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -860,12 +860,8 @@ def user_invite(context, data_dict): ''' _check_access('user_invite', context, data_dict) - user_invite_schema = { - 'email': [validators.not_empty, unicode], - 'group_id': [validators.not_empty], - 'role': [validators.not_empty], - } - data, errors = _validate(data_dict, user_invite_schema, context) + schema = context.get('schema') or ckan.logic.schema.default_user_invite_schema() + data, errors = _validate(data_dict, schema, context) if errors: raise ValidationError(errors) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index a8cd4ce6779..60f36411c7b 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -424,6 +424,14 @@ def default_update_user_schema(): return schema +def default_user_invite_schema(): + schema = { + 'email': [not_empty, unicode], + 'group_id': [not_empty], + 'role': [not_empty], + } + return schema + def default_task_status_schema(): schema = { 'id': [ignore], From c2e4f767eed6485383a7ef2efd40a8e990d0a623 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 22:59:53 -0300 Subject: [PATCH 176/269] [#1178] Refactor user name creation I've isolated the loop where it needs to be. This makes the code clearer, but makes it susceptible to errors. For example, after the random name was generated but before the user is created, another user with the same name might be created, breaking our code. This should be rare enough to bother us. --- ckan/logic/action/create.py | 43 +++++++++++++++------------------ ckan/tests/logic/test_action.py | 7 +++--- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index e4f13b3baad..9dca2940773 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -865,34 +865,31 @@ def user_invite(context, data_dict): if errors: raise ValidationError(errors) - while True: - try: - name = _get_random_username_from_email(data['email']) - password = str(random.SystemRandom().random()) - data['name'] = name - data['password'] = password - data['state'] = ckan.model.State.PENDING - user_dict = _get_action('user_create')(context, data) - user = ckan.model.User.get(user_dict['id']) - member_dict = { - 'username': user.id, - 'id': data['group_id'], - 'role': data['role'] - } - _get_action('group_member_create')(context, member_dict) - mailer.send_invite(user) - return model_dictize.user_dictize(user, context) - except ValidationError as e: - if 'name' not in e.error_dict: - raise e + name = _get_random_username_from_email(data['email']) + password = str(random.SystemRandom().random()) + data['name'] = name + data['password'] = password + data['state'] = ckan.model.State.PENDING + user_dict = _get_action('user_create')(context, data) + user = ckan.model.User.get(user_dict['id']) + member_dict = { + 'username': user.id, + 'id': data['group_id'], + 'role': data['role'] + } + _get_action('group_member_create')(context, member_dict) + mailer.send_invite(user) + return model_dictize.user_dictize(user, context) def _get_random_username_from_email(email): localpart = email.split('@')[0] cleaned_localpart = re.sub(r'[^\w]', '-', localpart) - random_number = random.SystemRandom().random() * 10000 - name = '%s-%d' % (cleaned_localpart, random_number) - return name + while True: + random_number = random.SystemRandom().random() * 10000 + name = '%s-%d' % (cleaned_localpart, random_number) + if not ckan.model.User.get(name): + return name ## Modifications for rest api diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 1fb96e83ca1..d5a043c0b9a 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -613,8 +613,8 @@ def test_user_invite_without_email_raises_error(self, mail_user): assert res_obj['success'] is False, res_obj assert 'email' in res_obj['error'], res_obj - @mock.patch('ckan.logic.action.create._get_random_username_from_email') - def test_user_invite_should_work_even_if_tried_username_already_exists(self, random_username_mock): + @mock.patch('random.SystemRandom') + def test_user_invite_should_work_even_if_tried_username_already_exists(self, system_random_mock): patcher = mock.patch('ckan.lib.mailer.mail_user') patcher.start() email = 'invited_user@email.com' @@ -628,8 +628,7 @@ def test_user_invite_should_work_even_if_tried_username_already_exists(self, ran postparams = '%s=1' % json.dumps(params) extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} - usernames = ['first', 'first', 'second'] - random_username_mock.side_effect = lambda email: usernames.pop(0) + system_random_mock.return_value.random.side_effect = [1000, 1000, 2000, 3000] for _ in range(2): res = self.app.post('/api/action/user_invite', params=postparams, From dfd8ae9fb07ff551707bdc08c6e23bd424c7335c Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 11 Oct 2013 23:07:25 -0300 Subject: [PATCH 177/269] [#1178] Simplify "if" expression, removing redundancy --- ckan/controllers/group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index c8775168c8f..66bd533fd43 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -614,7 +614,7 @@ def member_new(self, id): data_dict['id'] = id email = data_dict.get('email') - if email and email != '': + if email: user_data_dict = { 'email': email, 'group_id': data_dict['id'], From bae4d47c3227bd5f7581ac5859edcb9badd027c0 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Sat, 12 Oct 2013 16:52:19 -0300 Subject: [PATCH 178/269] [#1178] Use model.State instead of core.State --- ckan/logic/action/get.py | 6 +++--- .../versions/071_add_state_column_to_user_table.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 57c62a2916c..0621f55cf9a 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -15,7 +15,7 @@ import ckan.logic.schema import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.navl.dictization_functions -import ckan.model.core as core +import ckan.model as model import ckan.model.misc as misc import ckan.plugins as plugins import ckan.lib.search as search @@ -690,7 +690,7 @@ def user_list(context, data_dict): ) # Filter deleted users - query = query.filter(model.User.state != core.State.DELETED) + query = query.filter(model.User.state != model.State.DELETED) ## hack for pagination if context.get('return_query'): @@ -1246,7 +1246,7 @@ def user_autocomplete(context, data_dict): limit = data_dict.get('limit', 20) query = model.User.search(q) - query = query.filter(model.User.state != core.State.DELETED) + query = query.filter(model.User.state != model.State.DELETED) query = query.limit(limit) user_list = [] diff --git a/ckan/migration/versions/071_add_state_column_to_user_table.py b/ckan/migration/versions/071_add_state_column_to_user_table.py index c4f3f5013aa..828ebfb5f44 100644 --- a/ckan/migration/versions/071_add_state_column_to_user_table.py +++ b/ckan/migration/versions/071_add_state_column_to_user_table.py @@ -1,9 +1,9 @@ -import ckan.model.core +import ckan.model def upgrade(migrate_engine): migrate_engine.execute( ''' ALTER TABLE "user" ADD COLUMN "state" text NOT NULL DEFAULT '%s' - ''' % ckan.model.core.State.ACTIVE + ''' % ckan.model.State.ACTIVE ) From 5023e0c3354ca03475b06ee11a3e47f867adedaf Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Mon, 14 Oct 2013 10:51:01 +0530 Subject: [PATCH 179/269] Don't exempt sysadmins --- ckan/logic/auth/create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index ec38483bd0d..ca3a2d70b6d 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -104,6 +104,7 @@ def rating_create(context, data_dict): return {'success': True} +@logic.auth_sysadmins_check def user_create(context, data_dict=None): using_api = 'api_version' in context create_user_via_api = new_authz.check_config_permission( From 878772cfc37932805cca58439150d702506fe043 Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Mon, 14 Oct 2013 12:22:29 +0530 Subject: [PATCH 180/269] [#1226] Add tests --- ckan/tests/functional/api/test_user.py | 190 ++++++++++++++++++++++++- ckan/tests/logic/test_auth.py | 3 +- 2 files changed, 189 insertions(+), 4 deletions(-) diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py index 06dd35dd7fc..26565810c2f 100644 --- a/ckan/tests/functional/api/test_user.py +++ b/ckan/tests/functional/api/test_user.py @@ -1,20 +1,26 @@ +import paste +from pylons import config from nose.tools import assert_equal import ckan.logic as logic +import ckan.new_authz as new_authz from ckan import model from ckan.lib.create_test_data import CreateTestData from ckan.tests import TestController as ControllerTestCase +from ckan.tests.pylons_controller import PylonsTestCase from ckan.tests import url_for +import ckan.config.middleware +from ckan.common import json class TestUserApi(ControllerTestCase): @classmethod def setup_class(cls): CreateTestData.create() - + @classmethod def teardown_class(cls): model.repo.rebuild_db() - + def test_autocomplete(self): response = self.app.get( url=url_for(controller='api', action='user_autocomplete', ver=2), @@ -51,8 +57,186 @@ def test_autocomplete_limit(self): print response.json assert_equal(len(response.json), 1) + +class TestCreateUserApiDisabled(PylonsTestCase): + ''' + Tests for the creating user when create_user_via_api is disabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + cls.sysadmin_user = model.User.get('testsysadmin') + PylonsTestCase.setup_class() + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled_sysadmin(self): + params = { + 'name': 'testinganewusersysadmin', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + def test_user_create_api_disabled_anon(self): + params = { + 'name': 'testinganewuseranon', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + +class TestCreateUserApiEnabled(PylonsTestCase): + ''' + Tests for the creating user when create_user_via_api is enabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + config['ckan.auth.create_user_via_api'] = True + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + PylonsTestCase.setup_class() + cls.sysadmin_user = model.User.get('testsysadmin') + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled_sysadmin(self): + params = { + 'name': 'testinganewusersysadmin', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': + str(self.sysadmin_user.apikey)}) + res_dict = res.json + assert res_dict['success'] is True + + def test_user_create_api_disabled_anon(self): + params = { + 'name': 'testinganewuseranon', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is True + + +class TestCreateUserWebDisabled(PylonsTestCase): + ''' + Tests for the creating user by create_user_via_web is disabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + config['ckan.auth.create_user_via_web'] = False + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + cls.sysadmin_user = model.User.get('testsysadmin') + PylonsTestCase.setup_class() + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled(self): + params = { + 'name': 'testinganewuser', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + +class TestCreateUserWebEnabled(PylonsTestCase): + ''' + Tests for the creating user by create_user_via_web is enabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + config['ckan.auth.create_user_via_web'] = True + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + cls.sysadmin_user = model.User.get('testsysadmin') + PylonsTestCase.setup_class() + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled(self): + params = { + 'name': 'testinganewuser', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + class TestUserActions(object): - + @classmethod def setup_class(cls): CreateTestData.create() diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 3ea9f2db39f..b73834d2657 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -11,8 +11,9 @@ 'user_create_organizations': False, 'user_delete_groups': False, 'user_delete_organizations': False, - 'create_user_via_api': False, 'create_unowned_dataset': False, + 'create_user_via_api': False, + 'create_user_via_web': True, } From 3273ebf7d0e06dbf3d854a25f701e651fe88bb0b Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Mon, 14 Oct 2013 15:34:07 +0530 Subject: [PATCH 181/269] [#1226] Return the old behaviour for create_user --- ckan/logic/auth/create.py | 1 - ckan/tests/functional/api/test_user.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 13e02125855..53c8ae153ab 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -105,7 +105,6 @@ def rating_create(context, data_dict): @logic.auth_allow_anonymous_access -@logic.auth_sysadmins_check def user_create(context, data_dict=None): using_api = 'api_version' in context create_user_via_api = new_authz.check_config_permission( diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py index ba67b97455e..cdbe039d012 100644 --- a/ckan/tests/functional/api/test_user.py +++ b/ckan/tests/functional/api/test_user.py @@ -83,7 +83,7 @@ def teardown_class(cls): model.repo.rebuild_db() - def test_user_create_api_disabled_sysadmin(self): + def test_user_create_api_enabled_sysadmin(self): params = { 'name': 'testinganewusersysadmin', 'email': 'testinganewuser@ckan.org', @@ -93,7 +93,7 @@ def test_user_create_api_disabled_sysadmin(self): extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, expect_errors=True) res_dict = res.json - assert res_dict['success'] is False + assert res_dict['success'] is True def test_user_create_api_disabled_anon(self): params = { @@ -190,7 +190,6 @@ def test_user_create_api_disabled(self): 'password': 'random', } res = self.app.post('/api/3/action/user_create', json.dumps(params), - extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, expect_errors=True) res_dict = res.json assert res_dict['success'] is False @@ -229,7 +228,6 @@ def test_user_create_api_disabled(self): 'password': 'random', } res = self.app.post('/api/3/action/user_create', json.dumps(params), - extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, expect_errors=True) res_dict = res.json assert res_dict['success'] is False From 3218b581443ad07cd09a93d364bf5085fb0f7884 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 14 Oct 2013 17:31:04 +0100 Subject: [PATCH 182/269] [#1237] Fix docstring --- ckan/logic/action/get.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a810cfa3cbe..384026a64e8 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1289,8 +1289,8 @@ def package_search(context, data_dict): :param facet.mincount: the minimum counts for facet fields should be included in the results. :type facet.mincount: int - :param facet.limit: the maximum number of values the facet fields. A - negative value means unlimited. This can be set instance-wide with + :param facet.limit: the maximum number of values the facet fields return. + A negative value means unlimited. This can be set instance-wide with the :ref:`search.facets.limit` config option. Default is 50. :type facet.limit: int :param facet.field: the fields to facet upon. Default empty. If empty, From bb6cb7d0ff417b72f002dd372bdfd09ba560a766 Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 14 Oct 2013 21:46:39 +0100 Subject: [PATCH 183/269] [#1261] add filesize limit, make storage folder backwards compatable, document a bit more --- ckan/lib/helpers.py | 3 +- ckan/lib/uploader.py | 69 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index fceab6f1b6a..3aa73ad3a8b 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -37,6 +37,7 @@ import ckan.lib.maintain as maintain import ckan.lib.datapreview as datapreview import ckan.logic as logic +import ckan.lib.uploader as uploader from ckan.common import ( _, ungettext, g, c, request, session, json, OrderedDict @@ -1664,7 +1665,7 @@ def new_activities(): return action({}, {}) def uploads_enabled(): - if config.get('ckan.storage_path'): + if uploader.get_storage_path(): return True return False diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index eb0f28eec54..880488b8161 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -3,11 +3,51 @@ import pylons import datetime import ckan.lib.munge as munge +import logging +import ckan.logic as logic + +log = logging.getLogger(__name__) + +_storage_path = None + +def get_storage_path(): + '''Function to cache storage path''' + global _storage_path + + #None means it has not been set. False means not in config. + if _storage_path is None: + storage_path = pylons.config.get('ckan.storage_path') + ofs_impl = pylons.config.get('ofs.impl') + ofs_storage_dir = pylons.config.get('ofs.storage_dir') + if storage_path: + _storage_path = storage_path + elif ofs_impl == 'pairtree' and ofs_storage_dir: + log.warn('''Please use config option ckan.storage_path instaed of + ofs.storage_path''') + _storage_path = ofs_storage_dir + return _storage_path + elif ofs_impl: + log.critical('''We only support local file storage form version 2.2 of ckan + please specify ckan.storage_path in your config for your uploads''') + _storage_path = False + else: + log.critical('''Please specify a ckan.storage_path in your config for your uploads''') + _storage_path = False + + return _storage_path + class Upload(object): def __init__(self, object_type, old_filename=None): - path = pylons.config.get('ckan.storage_path', '/tmp') + ''' Setup upload by creating a subdirectory of the storage directory of name + object_type. old_filename is the name of the file in the url field last time''' + self.storage_path = None + self.filename = None + self.filepath = None + path = get_storage_path() + if not path: + return self.storage_path = os.path.join(path, 'storage', 'uploads', object_type) try: @@ -18,12 +58,20 @@ def __init__(self, object_type, old_filename=None): self.old_filename = old_filename if old_filename: self.old_filepath = os.path.join(self.storage_path, old_filename) - self.filename = None - self.filepath = None def update_data_dict(self, data_dict, url_field, file_field, clear_field): + ''' Manipulate data from the data_dict. url_field is the name of the field + where the upload is going to be. file_field is name of the key where the + FieldStorage is kept (i.e the field where the file data actually is). clear_field + is the name of a boolean field which requests the upload to be deleted. This needs + to be called before it reaches any validators''' + + if not self.storage_path: + return + self.url = data_dict.get(url_field, '') self.clear = data_dict.pop(clear_field, None) + self.file_field = file_field self.upload_field_storage = data_dict.pop(file_field, None) if isinstance(self.upload_field_storage, cgi.FieldStorage): @@ -41,16 +89,27 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): if self.clear and self.url == self.old_filename: data_dict[url_field] = '' - def upload(self): + def upload(self, max_size=2): + ''' Actually upload the file. This should happen just before a commit but after + the data has been validated and flushed to the db. This is so we do not store anything + unless the request is actually good. Max_size is size in MB maximum of the file''' + if self.filename: output_file = open(self.tmp_filepath, 'wb') self.upload_file.seek(0) + current_size = 0 while True: - # mb chuncks + current_size = current_size + 1 + # MB chuncks data = self.upload_file.read(2 ** 20) if not data: break output_file.write(data) + if current_size > max_size: + os.remove(self.tmp_filepath) + raise logic.ValidationError( + {self.file_field: ['File upload too large']} + ) output_file.close() os.rename(self.tmp_filepath, self.filepath) self.clear = True From f5d168cd9018d68868644188dded4dd2b6b01f36 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 15 Oct 2013 13:35:08 +0100 Subject: [PATCH 184/269] [#1256] Don't load the synchronous_search plugin on cli It is already loaded on startup --- ckan/lib/cli.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 14c217ed096..c74c25e85e2 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -842,12 +842,10 @@ def show(self, dataset_ref): pprint.pprint(dataset.as_dict()) def delete(self, dataset_ref): - from ckan import plugins import ckan.model as model dataset = self._get_dataset(dataset_ref) old_state = dataset.state - plugins.load('synchronous_search') rev = model.repo.new_revision() dataset.delete() model.repo.commit_and_remove() @@ -855,12 +853,10 @@ def delete(self, dataset_ref): print '%s %s -> %s' % (dataset.name, old_state, dataset.state) def purge(self, dataset_ref): - from ckan import plugins import ckan.model as model dataset = self._get_dataset(dataset_ref) name = dataset.name - plugins.load('synchronous_search') rev = model.repo.new_revision() dataset.purge() model.repo.commit_and_remove() @@ -1286,8 +1282,6 @@ class CreateTestDataCommand(CkanCommand): def command(self): self._load_config() self._setup_app() - from ckan import plugins - plugins.load('synchronous_search') # so packages get indexed from create_test_data import CreateTestData if self.args: From 6431ba3dc6a117926101a97ae859132a906cf0f2 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 15 Oct 2013 16:00:30 +0200 Subject: [PATCH 185/269] [#1178] Add auth_user_obj back to context in User controller methods. --- ckan/controllers/user.py | 48 ++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 98033d9a0ab..1c6fefff344 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -38,7 +38,8 @@ class UserController(base.BaseController): def __before__(self, action, **env): base.BaseController.__before__(self, action, **env) try: - context = {'model': model, 'user': c.user or c.author} + context = {'model': model, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} check_access('site_read', context) except NotAuthorized: if c.action not in ('login', 'request_reset', 'perform_reset',): @@ -89,7 +90,8 @@ def index(self): c.q = request.params.get('q', '') c.order_by = request.params.get('order_by', 'name') - context = {'return_query': True} + context = {'return_query': True, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'q': c.q, 'order_by': c.order_by} @@ -111,7 +113,8 @@ def index(self): def read(self, id=None): context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, 'for_view': True} + 'user': c.user or c.author, 'auth_user_obj': c.userobj, + 'for_view': True} data_dict = {'id': id, 'user_obj': c.userobj} try: @@ -140,7 +143,8 @@ def me(self, locale=None): id=user_ref) def register(self, data=None, errors=None, error_summary=None): - context = {'model': model, 'session': model.Session, 'user': c.user} + context = {'model': model, 'session': model.Session, 'user': c.user, + 'auth_user_obj': c.userobj} try: check_access('user_create', context) except NotAuthorized: @@ -154,6 +158,7 @@ def new(self, data=None, errors=None, error_summary=None): ''' context = {'model': model, 'session': model.Session, 'user': c.user or c.author, + 'auth_user_obj': c.userobj, 'schema': self._new_form_to_db_schema(), 'save': 'save' in request.params} @@ -182,7 +187,8 @@ def delete(self, id): '''Delete user with id passed as parameter''' context = {'model': model, 'session': model.Session, - 'user': c.user} + 'user': c.user, + 'auth_user_obj': c.userobj} data_dict = {'id': id} try: @@ -234,7 +240,7 @@ def edit(self, id=None, data=None, errors=None, error_summary=None): context = {'save': 'save' in request.params, 'schema': self._edit_form_to_db_schema(), 'model': model, 'session': model.Session, - 'user': c.user, + 'user': c.user, 'auth_user_obj': c.userobj } if id is None: if c.userobj: @@ -391,7 +397,8 @@ def logged_out_page(self): return render('user/logout.html') def request_reset(self): - context = {'model': model, 'session': model.Session, 'user': c.user} + context = {'model': model, 'session': model.Session, 'user': c.user, + 'auth_user_obj': c.userobj} data_dict = {'id': request.params.get('user')} try: check_access('request_reset', context) @@ -511,7 +518,8 @@ def _get_form_password(self): raise ValueError(_('You must provide a password')) def followers(self, id=None): - context = {'for_view': True} + context = {'for_view': True, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'id': id, 'user_obj': c.userobj} self._setup_template_variables(context, data_dict) f = get_action('user_follower_list') @@ -525,7 +533,8 @@ def activity(self, id, offset=0): '''Render this user's public activity stream page.''' context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, 'for_view': True} + 'user': c.user or c.author, 'auth_user_obj': c.userobj, + 'for_view': True} data_dict = {'id': id, 'user_obj': c.userobj} try: check_access('user_show', context, data_dict) @@ -553,7 +562,8 @@ def display_name(followee): if (filter_type and filter_id): context = { 'model': model, 'session': model.Session, - 'user': c.user or c.author, 'for_view': True + 'user': c.user or c.author, 'auth_user_obj': c.userobj, + 'for_view': True } data_dict = {'id': filter_id} followee = None @@ -595,7 +605,8 @@ def display_name(followee): def dashboard(self, id=None, offset=0): context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, 'for_view': True} + 'user': c.user or c.author, 'auth_user_obj': c.userobj, + 'for_view': True} data_dict = {'id': id, 'user_obj': c.userobj, 'offset': offset} self._setup_template_variables(context, data_dict) @@ -618,19 +629,22 @@ def dashboard(self, id=None, offset=0): return render('user/dashboard.html') def dashboard_datasets(self): - context = {'for_view': True} + context = {'for_view': True, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'user_obj': c.userobj} self._setup_template_variables(context, data_dict) return render('user/dashboard_datasets.html') def dashboard_organizations(self): - context = {'for_view': True} + context = {'for_view': True, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'user_obj': c.userobj} self._setup_template_variables(context, data_dict) return render('user/dashboard_organizations.html') def dashboard_groups(self): - context = {'for_view': True} + context = {'for_view': True, 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'user_obj': c.userobj} self._setup_template_variables(context, data_dict) return render('user/dashboard_groups.html') @@ -639,7 +653,8 @@ def follow(self, id): '''Start following this user.''' context = {'model': model, 'session': model.Session, - 'user': c.user or c.author} + 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'id': id} try: get_action('follow_user')(context, data_dict) @@ -658,7 +673,8 @@ def unfollow(self, id): '''Stop following this user.''' context = {'model': model, 'session': model.Session, - 'user': c.user or c.author} + 'user': c.user or c.author, + 'auth_user_obj': c.userobj} data_dict = {'id': id} try: get_action('unfollow_user')(context, data_dict) From 3dee85226f1fa6a29a03d598c8c11be1c5dba4ca Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:10:09 +0100 Subject: [PATCH 186/269] [#1262] First working version of the JS module for image uploading --- .../base/javascript/modules/image-upload.js | 147 ++++++++++++++++++ ckan/public/base/javascript/resource.config | 1 + ckan/public/base/less/forms.less | 34 ++++ ckan/templates/group/snippets/group_form.html | 10 +- ckan/templates/macros/form.html | 27 ++++ .../snippets/organization_form.html | 9 +- 6 files changed, 211 insertions(+), 17 deletions(-) create mode 100644 ckan/public/base/javascript/modules/image-upload.js diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js new file mode 100644 index 00000000000..69ca8f93c05 --- /dev/null +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -0,0 +1,147 @@ +/* Image Upload + * + * + * Options + * - id: what's the id of the context? + */ +this.ckan.module('image-upload', function($, _) { + return { + /* options object can be extended using data-module-* attributes */ + options: { + is_url: true, + has_image: false, + i18n: { + upload: _('From computer'), + url: _('From web'), + remove: _('Remove'), + label: _('Upload image'), + label_url: _('Image URL'), + remove_tooltip: _('Reset this') + }, + template: [ + '' + ].join("\n") + }, + + state: { + attached: 1, + blank: 2, + web: 3 + }, + + /* Initialises the module setting up elements and event listeners. + * + * Returns nothing. + */ + initialize: function () { + $.proxyAll(this, /_on/); + var options = this.options; + + // firstly setup the fields + this.input = $('input[name="image_upload"]', this.el); + this.field_url = $('input[name="image_url"]', this.el).parents('.control-group'); + this.field_image = this.input.parents('.control-group'); + + var checkbox = $('input[name="clear_upload"]', this.el); + if (checkbox.length > 0) { + options.has_image = true; + checkbox.parents('.control-group').remove(); + } + + this.field_clear = $('') + .appendTo(this.el); + + this.button_url = $(' '+this.i18n('url')+'') + .on('click', this._onFromWeb) + .insertAfter(this.input); + + this.button_upload = $(''+this.i18n('upload')+'') + .insertAfter(this.input); + + this.button_remove = $('') + .text(this.i18n('remove')) + .on('click', this._onRemove) + .insertAfter(this.button_upload); + + $('') + .prop('title', this.i18n('remove_tooltip')) + .on('click', this._onRemove) + .insertBefore($('input', this.field_url)); + + $('label[for="field-image-upload"]').text(this.i18n('label')); + + this.input + .on('mouseover', this._onInputMouseOver) + .on('mouseout', this._onInputMouseOut) + .on('change', this._onInputChange) + .css('width', this.button_upload.outerWidth()) + .hide(); + + this.fields = $('') + .add(this.button_remove) + .add(this.button_upload) + .add(this.button_url) + .add(this.input) + .add(this.field_url) + .add(this.field_image); + + if (options.is_url) { + this.changeState(this.state.web); + } else if (options.has_image) { + this.changeState(this.state.attached); + } else { + this.changeState(this.state.blank); + } + + }, + + changeState: function(state) { + this.fields.hide(); + if (state == this.state.blank) { + this.button_upload + .add(this.field_image) + .add(this.button_url) + .add(this.input) + .show(); + } else if (state == this.state.attached) { + this.button_remove + .add(this.field_image) + .show(); + } else if (state == this.state.web) { + this.field_url + .show(); + } + }, + + _onFromWeb: function() { + this.changeState(this.state.web); + $('input', this.field_url).focus(); + if (this.options.has_image) { + this.field_clear.val('true'); + } + }, + + _onRemove: function() { + this.changeState(this.state.blank); + $('input', this.field_url).val(''); + if (this.options.has_image) { + this.field_clear.val('true'); + } + }, + + _onInputChange: function() { + this.file_name = this.input.val(); + this.field_clear.val(''); + this.changeState(this.state.attached); + }, + + _onInputMouseOver: function() { + this.button_upload.addClass('hover'); + }, + + _onInputMouseOut: function() { + this.button_upload.removeClass('hover'); + } + + } +}); diff --git a/ckan/public/base/javascript/resource.config b/ckan/public/base/javascript/resource.config index b1a7302d9e8..564da811a6e 100644 --- a/ckan/public/base/javascript/resource.config +++ b/ckan/public/base/javascript/resource.config @@ -38,6 +38,7 @@ ckan = modules/table-toggle-more.js modules/dataset-visibility.js modules/media-grid.js + modules/image-upload.js main = apply_html_class diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 289439c291a..2bfc1a2d492 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -670,3 +670,37 @@ textarea { .box-shadow(none); } } + +.js .image-upload { + #field-image-upload { + cursor: pointer; + position: absolute; + z-index: 1; + .opacity(0); + } + .controls { + position: relative; + } + .btn { + position: relative; + top: 0; + margin-right: 10px; + &.hover { + color: @grayDark; + text-decoration: none; + background-position: 0 -15px; + .transition(background-position .1s linear); + } + } + .btn-remove-url { + position: absolute; + margin-right: 0; + top: 4px; + right: 5px; + padding: 0 4px; + .border-radius(100px); + .icon-remove { + margin-right: 0; + } + } +} diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html index 322f9ebff8a..cd19912d66b 100644 --- a/ckan/templates/group/snippets/group_form.html +++ b/ckan/templates/group/snippets/group_form.html @@ -19,15 +19,7 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my group...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} - - - {% if h.uploads_enabled() %} - {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% if data.image_url and not data.image_url.startswith('http') %} - {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} - {% endif %} - {% endif %} + {{ form.image_upload(data, errors, image_url=c.group_dict.image_display_url, is_upload_enabled=h.uploads_enabled()) }} {% endblock %} diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html index 497dea6bf02..7dc044ccca2 100644 --- a/ckan/templates/macros/form.html +++ b/ckan/templates/macros/form.html @@ -395,3 +395,30 @@

    {% endmacro %} +{# +Builds a file upload for input + +Example + {% import 'macros/form.html' as form %} + {{ form.image_upload(data, errors, is_upload_enabled=true) }} + +#} +{% macro image_upload(data, errors, image_url=false, is_upload_enabled=false, placeholder=false) %} + {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %} + {% set has_uploaded_data = data.image_url and not data.image_url.startswith('http') %} + {% set is_url = data.image_url and data.image_url.startswith('http') %} + + {% if is_upload_enabled %}
    {% endif %} + + {{ input('image_url', label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }} + + {% if is_upload_enabled %} + {{ input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} + {% if has_uploaded_data %} + {{ checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% endif %} + {% endif %} + + {% if is_upload_enabled %}
    {% endif %} + +{% endmacro %} diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 0a2134e89f7..0b97b958d38 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -19,14 +19,7 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my organization...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} - - {% if h.uploads_enabled() %} - {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% if data.image_url and not data.image_url.startswith('http') %} - {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} - {% endif %} - {% endif %} + {{ form.image_upload(data, errors, image_url=c.group_dict.image_display_url, is_upload_enabled=h.uploads_enabled()) }} {% endblock %} From 0f74db8a964451e5cba1360b4f15ea3a84a56544 Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:21:14 +0100 Subject: [PATCH 187/269] [#1262] Adds in doc blocks into the image_upload JS module --- .../base/javascript/modules/image-upload.js | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js index 69ca8f93c05..dbb83ade15a 100644 --- a/ckan/public/base/javascript/modules/image-upload.js +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -1,8 +1,5 @@ /* Image Upload * - * - * Options - * - id: what's the id of the context? */ this.ckan.module('image-upload', function($, _) { return { @@ -42,41 +39,49 @@ this.ckan.module('image-upload', function($, _) { this.field_url = $('input[name="image_url"]', this.el).parents('.control-group'); this.field_image = this.input.parents('.control-group'); + // Is there a clear checkbox on the form already? var checkbox = $('input[name="clear_upload"]', this.el); if (checkbox.length > 0) { options.has_image = true; checkbox.parents('.control-group').remove(); } + // Adds the hidden clear input to the form this.field_clear = $('') .appendTo(this.el); + // Button to set the field to be a URL this.button_url = $(' '+this.i18n('url')+'') .on('click', this._onFromWeb) .insertAfter(this.input); + // Button to attach local file to the form this.button_upload = $(''+this.i18n('upload')+'') .insertAfter(this.input); + // Button to reset the form back to the first from when there is a image uploaded this.button_remove = $('') .text(this.i18n('remove')) .on('click', this._onRemove) .insertAfter(this.button_upload); + // Button for resetting the form when there is a URL set $('') .prop('title', this.i18n('remove_tooltip')) .on('click', this._onRemove) .insertBefore($('input', this.field_url)); + // Update the main label $('label[for="field-image-upload"]').text(this.i18n('label')); + // Setup the file input this.input .on('mouseover', this._onInputMouseOver) .on('mouseout', this._onInputMouseOut) .on('change', this._onInputChange) - .css('width', this.button_upload.outerWidth()) - .hide(); + .css('width', this.button_upload.outerWidth()); + // Fields storage. Used in this.changeState this.fields = $('') .add(this.button_remove) .add(this.button_upload) @@ -85,6 +90,7 @@ this.ckan.module('image-upload', function($, _) { .add(this.field_url) .add(this.field_image); + // Setup the initial state if (options.is_url) { this.changeState(this.state.web); } else if (options.has_image) { @@ -95,6 +101,16 @@ this.ckan.module('image-upload', function($, _) { }, + /* Method to change the display state of the image fields + * + * state - Pseudo constant for passing the state we should be in now + * + * Examples + * + * this.changeState(this.state.web); // Sets the state in URL mode + * + * Returns nothing. + */ changeState: function(state) { this.fields.hide(); if (state == this.state.blank) { @@ -113,6 +129,10 @@ this.ckan.module('image-upload', function($, _) { } }, + /* Event listener for when someone sets the field to URL mode + * + * Returns nothing. + */ _onFromWeb: function() { this.changeState(this.state.web); $('input', this.field_url).focus(); @@ -121,6 +141,10 @@ this.ckan.module('image-upload', function($, _) { } }, + /* Event listener for resetting the field back to the blank state + * + * Returns nothing. + */ _onRemove: function() { this.changeState(this.state.blank); $('input', this.field_url).val(''); @@ -129,16 +153,28 @@ this.ckan.module('image-upload', function($, _) { } }, + /* Event listener for when someone chooses a file to upload + * + * Returns nothing. + */ _onInputChange: function() { this.file_name = this.input.val(); this.field_clear.val(''); this.changeState(this.state.attached); }, + /* Event listener for when a user mouseovers the hidden file input + * + * Returns nothing. + */ _onInputMouseOver: function() { this.button_upload.addClass('hover'); }, + /* Event listener for when a user mouseouts the hidden file input + * + * Returns nothing. + */ _onInputMouseOut: function() { this.button_upload.removeClass('hover'); } From 4ac411b61cfae7048cdc6cc5cace42aec20a13e9 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 15 Oct 2013 16:28:12 +0200 Subject: [PATCH 188/269] [#1178] Fix typo in org member_new template --- ckan/templates/organization/member_new.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/templates/organization/member_new.html b/ckan/templates/organization/member_new.html index 352cb9f3790..a121549dab4 100644 --- a/ckan/templates/organization/member_new.html +++ b/ckan/templates/organization/member_new.html @@ -19,7 +19,7 @@

    {{ _('Existing User') }} - {{ _('If you wish to add an existing user, search for her username below.') }} + {{ _('If you wish to add an existing user, search for their username below.') }}
    {% if user %} @@ -41,7 +41,7 @@

    {{ _('New User') }} - {{ _('If you wish to invite a new user, enter her email address.') }} + {{ _('If you wish to invite a new user, enter their email address.') }}
    From 7d15d591499d5722f60cc0ecfc09abf59dad177f Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:29:45 +0100 Subject: [PATCH 189/269] [#1262] Adds support for different field names for image upload --- ckan/public/base/javascript/modules/image-upload.js | 13 +++++++------ ckan/templates/macros/form.html | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js index dbb83ade15a..96308ed5764 100644 --- a/ckan/public/base/javascript/modules/image-upload.js +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -7,6 +7,9 @@ this.ckan.module('image-upload', function($, _) { options: { is_url: true, has_image: false, + field_upload: 'input[name="image_upload"]', + field_url: 'input[name="image_url"]', + field_clear: 'input[name="clear_upload"]', i18n: { upload: _('From computer'), url: _('From web'), @@ -35,12 +38,12 @@ this.ckan.module('image-upload', function($, _) { var options = this.options; // firstly setup the fields - this.input = $('input[name="image_upload"]', this.el); - this.field_url = $('input[name="image_url"]', this.el).parents('.control-group'); + this.input = $(options.field_upload, this.el); + this.field_url = $(options.field_url, this.el).parents('.control-group'); this.field_image = this.input.parents('.control-group'); // Is there a clear checkbox on the form already? - var checkbox = $('input[name="clear_upload"]', this.el); + var checkbox = $(options.field_clear, this.el); if (checkbox.length > 0) { options.has_image = true; checkbox.parents('.control-group').remove(); @@ -148,9 +151,7 @@ this.ckan.module('image-upload', function($, _) { _onRemove: function() { this.changeState(this.state.blank); $('input', this.field_url).val(''); - if (this.options.has_image) { - this.field_clear.val('true'); - } + this.field_clear.val('true'); }, /* Event listener for when someone chooses a file to upload diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html index 7dc044ccca2..5a6c00a6cb1 100644 --- a/ckan/templates/macros/form.html +++ b/ckan/templates/macros/form.html @@ -403,19 +403,19 @@ {{ form.image_upload(data, errors, is_upload_enabled=true) }} #} -{% macro image_upload(data, errors, image_url=false, is_upload_enabled=false, placeholder=false) %} +{% macro image_upload(data, errors, field_url='image_url', field_upload='image_upload', field_clear='clear_upload', image_url=false, is_upload_enabled=false, placeholder=false) %} {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %} {% set has_uploaded_data = data.image_url and not data.image_url.startswith('http') %} {% set is_url = data.image_url and data.image_url.startswith('http') %} {% if is_upload_enabled %}
    {% endif %} - {{ input('image_url', label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {{ input(field_url, label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }} {% if is_upload_enabled %} - {{ input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} + {{ input(field_upload, label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} {% if has_uploaded_data %} - {{ checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {{ checkbox(field_clear, label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} {% endif %} {% endif %} From 0f8727a31f26069561352017064c7febd9b48b2a Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:44:51 +0100 Subject: [PATCH 190/269] [#1269] Tweaks to the templates to make them better --- ckan/templates/package/resource_edit.html | 2 +- .../templates/package/resource_edit_base.html | 24 ++++++++++++++++--- .../package/snippets/resource_info.html | 21 ++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 ckan/templates/package/snippets/resource_info.html diff --git a/ckan/templates/package/resource_edit.html b/ckan/templates/package/resource_edit.html index 9756c8f9581..3d761185230 100644 --- a/ckan/templates/package/resource_edit.html +++ b/ckan/templates/package/resource_edit.html @@ -5,5 +5,5 @@ {% block subtitle %}{{ _('Edit') }} - {{ h.resource_display_name(res) }} - {{ h.dataset_display_name(pkg) }}{% endblock %} {% block form %} - {{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }} + {{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }} {% endblock %} diff --git a/ckan/templates/package/resource_edit_base.html b/ckan/templates/package/resource_edit_base.html index 7c3a8d012ec..e6c98ad7ca3 100644 --- a/ckan/templates/package/resource_edit_base.html +++ b/ckan/templates/package/resource_edit_base.html @@ -1,12 +1,30 @@ -{% extends "package/edit_base.html" %} +{% extends "package/base.html" %} {% set logged_in = true if c.userobj else false %} {% block breadcrumb_content_selected %}{% endblock %} +{% block breadcrumb_content %} + {{ super() }} +
  • {% link_for h.resource_display_name(res)|truncate(30), controller='package', action='resource_read', id=pkg.name, resource_id=res.id %}
    • +
  • Edit
    • +{% endblock %} + +{% block content_action %} + {% link_for _('All resources'), controller='package', action='resources', id=pkg.name, class_='btn', icon='arrow-left' %} + {% link_for _('View resource'), controller='package', action='resource_read', id=pkg.name, resource_id=res.id, class_='btn', icon='eye-open' %} +{% endblock %} + +{% block content_primary_nav %} + {{ h.build_nav_icon('resource_edit', _('Edit resource'), id=pkg.name, resource_id=res.id) }} +{% endblock %} + +{% block secondary_content %} + {% snippet 'package/snippets/resource_info.html', res=res %} +{% endblock %} + {% block primary_content_inner %} - {% link_for _('View all resources'), controller='package', action='resources', id=pkg.name, class_='btn pull-right', icon='arrow-left' %} -

    {% block form_title %}{{ _('Edit resource') }}{% endblock %}

    +

    {% block form_title %}{{ _('Edit resource') }}{% endblock %}

    {% block form %}{% endblock %} {% endblock %} diff --git a/ckan/templates/package/snippets/resource_info.html b/ckan/templates/package/snippets/resource_info.html new file mode 100644 index 00000000000..d8ada30d83a --- /dev/null +++ b/ckan/templates/package/snippets/resource_info.html @@ -0,0 +1,21 @@ +{# +Displays a sidebard module with information for given resource + +res - The respource dict + +Example: + + {% snippet "package/snippets/resrouce_info.html", res=res %} + +#} +
    +
    +

    {{ res.name or res.id }}

    +
    +
    +
    {{ _('Format') }}
    +
    {{ res.format }}
    +
    +
    +
    +
    From 322149c6934a574589f37ffe8317c444ad0885ff Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 15 Oct 2013 16:51:07 +0200 Subject: [PATCH 191/269] [#1178] Constrain number of attempts to generate a random user name in user_invite --- ckan/logic/action/create.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 9dca2940773..693845398d0 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -860,7 +860,8 @@ def user_invite(context, data_dict): ''' _check_access('user_invite', context, data_dict) - schema = context.get('schema') or ckan.logic.schema.default_user_invite_schema() + schema = context.get('schema', + ckan.logic.schema.default_user_invite_schema()) data, errors = _validate(data_dict, schema, context) if errors: raise ValidationError(errors) @@ -885,7 +886,12 @@ def user_invite(context, data_dict): def _get_random_username_from_email(email): localpart = email.split('@')[0] cleaned_localpart = re.sub(r'[^\w]', '-', localpart) - while True: + + # if we can't create a unique user name within this many attempts + # then something else is probably wrong and we should give up + max_name_creation_attempts = 100 + + for i in range(max_name_creation_attempts): random_number = random.SystemRandom().random() * 10000 name = '%s-%d' % (cleaned_localpart, random_number) if not ckan.model.User.get(name): From a480ace5ae73532627ad9b04cc3416f855373fc6 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 15 Oct 2013 17:09:29 +0200 Subject: [PATCH 192/269] [#1178] handle edge case where _get_random_username_from_email fails by returning the cleaned local part of the email address. This is expected to fail validation in user_create. --- ckan/logic/action/create.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 693845398d0..c7fb98be30a 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -897,6 +897,8 @@ def _get_random_username_from_email(email): if not ckan.model.User.get(name): return name + return cleaned_localpart + ## Modifications for rest api From 971026bf286aee490e8080bab715870d65b84cf7 Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 16:22:49 +0100 Subject: [PATCH 193/269] [#1263] Remove the 'clear all' from the faceter on the left nav --- ckan/public/base/less/iehacks.less | 5 ----- ckan/public/base/less/module.less | 11 ----------- ckan/templates/development/snippets/facet.html | 2 +- ckan/templates/development/snippets/module.html | 2 +- ckan/templates/snippets/facet_list.html | 1 - 5 files changed, 2 insertions(+), 19 deletions(-) diff --git a/ckan/public/base/less/iehacks.less b/ckan/public/base/less/iehacks.less index 232ee72ee9c..7d9bc5cc274 100644 --- a/ckan/public/base/less/iehacks.less +++ b/ckan/public/base/less/iehacks.less @@ -200,11 +200,6 @@ .media-content { position: relative; } - .action { - position: absolute; - top: 9px; - right: 10px; - } .media-image img { float: left; } diff --git a/ckan/public/base/less/module.less b/ckan/public/base/less/module.less index a575abcbda4..aa964cf5ea3 100644 --- a/ckan/public/base/less/module.less +++ b/ckan/public/base/less/module.less @@ -13,17 +13,6 @@ border-bottom: 1px solid @moduleHeadingBorderColor; } -.module-heading .action { - float: right; - color: @moduleHeadingActionTextColor; - font-size: 12px; - line-height: @baseLineHeight; - text-decoration: underline; - &:hover { - color: @layoutTextColor; - } -} - .module-content { padding: 0 @gutterX; margin: 20px 0; diff --git a/ckan/templates/development/snippets/facet.html b/ckan/templates/development/snippets/facet.html index ac9b9df1a0c..7280bf9a3d2 100644 --- a/ckan/templates/development/snippets/facet.html +++ b/ckan/templates/development/snippets/facet.html @@ -1,6 +1,6 @@
    {% with items=(("First", true), ("Second", false), ("Third", true), ("Fourth", false), ("Last", false)) %} -

    Facet List Clear All

    +

    Facet List