From 910ca29a902b265e5788864f533a70d8626cbcf1 Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 14:59:14 +0530 Subject: [PATCH 01/49] New option to disable user creation via web --- ckan/config/deployment.ini_tmpl | 1 + ckan/logic/auth/create.py | 22 +++++++++++++++------- ckan/new_authz.py | 2 ++ ckan/templates/header.html | 4 +++- ckan/templates/user/login.html | 20 +++++++++++--------- test-core.ini | 3 ++- 6 files changed, 34 insertions(+), 18 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index 97c4f9412ba..6a0833d2d9f 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -66,6 +66,7 @@ ckan.auth.user_create_organizations = true ckan.auth.user_delete_groups = true ckan.auth.user_delete_organizations = true ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true ## Search Settings diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index e96b789bcf5..35a9f39d876 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -103,14 +103,22 @@ def rating_create(context, data_dict): # No authz check in the logic function return {'success': True} -def user_create(context, data_dict=None): - user = context['user'] - if ('api_version' in context - and not new_authz.check_config_permission('create_user_via_api')): - return {'success': False, 'msg': _('User %s not authorized to create users') % user} - else: - return {'success': True} +def user_create(context, data_dict=None): + # create_user_via_api is deprecated + using_api = 'api_version' in context + create_user_via_api = new_authz.check_config_permission( + 'create_user_via_api') + create_user_via_web = new_authz.check_config_permission( + 'create_user_via_web') + + if using_api and not create_user_via_api: + return {'success': False, 'msg': _('User {user} not authorized to ' + 'create users via the API').format(user=context.get('user'))} + if not using_api and not create_user_via_web: + return {'success': False, 'msg': _('Not authorized to ' + 'create users')} + return {'success': True} def _check_group_auth(context, data_dict): diff --git a/ckan/new_authz.py b/ckan/new_authz.py index 00a7dfe0e17..03ca209faf4 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -86,6 +86,7 @@ def _build(self): def clear_auth_functions_cache(): _AuthFunctions.clear() + CONFIG_PERMISSIONS.clear() def auth_functions_list(): @@ -319,6 +320,7 @@ def get_user_id_for_username(user_name, allow_none=False): 'user_delete_groups': True, 'user_delete_organizations': True, 'create_user_via_api': False, + 'create_user_via_web': True, } CONFIG_PERMISSIONS = {} diff --git a/ckan/templates/header.html b/ckan/templates/header.html index 5b4ebe225a7..268b370e968 100644 --- a/ckan/templates/header.html +++ b/ckan/templates/header.html @@ -49,7 +49,9 @@ diff --git a/ckan/templates/user/login.html b/ckan/templates/user/login.html index c84e3522748..98c99510570 100644 --- a/ckan/templates/user/login.html +++ b/ckan/templates/user/login.html @@ -18,15 +18,17 @@

{% block page_heading %}{{ _('Login') }}{% endblock %}< {% endblock %} {% block secondary_content %} -
-

{{ _('Need an Account?') }}

-
-

{% trans %}Then sign right up, it only takes a minute.{% endtrans %}

-

- {{ _('Create an Account') }} -

-
-
+ {% if h.check_access('user_create') %} +
+

{{ _('Need an Account?') }}

+
+

{% trans %}Then sign right up, it only takes a minute.{% endtrans %}

+

+ {{ _('Create an Account') }} +

+
+
+ {% endif %}

{{ _('Forgotten your details?') }}

diff --git a/test-core.ini b/test-core.ini index 21dc08cfae9..08aba34ed24 100644 --- a/test-core.ini +++ b/test-core.ini @@ -31,6 +31,7 @@ solr_url = http://127.0.0.1:8983/solr ckan.auth.user_create_organizations = true ckan.auth.user_create_groups = true ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true ckan.auth.create_dataset_if_not_in_organization = true ckan.auth.anon_create_dataset = false ckan.auth.user_delete_groups=true @@ -80,7 +81,7 @@ smtp.mail_from = info@test.ckan.net ckan.locale_default = en ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv -ckan.locales_filtered_out = +ckan.locales_filtered_out = ckan.datastore.enabled = 1 From 6bc0c3ee3b9ea158ad358c5697e26eedd372ba9f Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 14:59:50 +0530 Subject: [PATCH 02/49] Add documentation for new config option --- doc/configuration.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index 0efe9573226..853ffd23939 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -355,6 +355,20 @@ Default value: ``False`` Allow new user accounts to be created via the API. +.. _ckan.auth.create_user_via_web: + +ckan.auth.create_user_via_api +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.auth.create_user_via_web = True + +Default value: ``True`` + + +Allow new user accounts to be created via the Web. + .. end_config-authorization From c75e487ff99765418d5c82e9f683e2d694afc98d Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 16:46:10 +0530 Subject: [PATCH 03/49] Fix docs, remove deprecated comment --- ckan/logic/auth/create.py | 1 - doc/configuration.rst | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 35a9f39d876..ec38483bd0d 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -105,7 +105,6 @@ def rating_create(context, data_dict): def user_create(context, data_dict=None): - # create_user_via_api is deprecated using_api = 'api_version' in context create_user_via_api = new_authz.check_config_permission( 'create_user_via_api') diff --git a/doc/configuration.rst b/doc/configuration.rst index 853ffd23939..29ecbd310d8 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -357,7 +357,7 @@ Allow new user accounts to be created via the API. .. _ckan.auth.create_user_via_web: -ckan.auth.create_user_via_api +ckan.auth.create_user_via_web ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example:: From 3d5fe71e8b17067f032784be75b932a352163fc6 Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 7 Oct 2013 10:00:07 +0100 Subject: [PATCH 04/49] [1262] basic image upload functionality --- ckan/config/middleware.py | 6 ++ ckan/controllers/group.py | 28 ++++++++- ckan/lib/dictization/model_dictize.py | 21 +++++++ ckan/lib/helpers.py | 5 ++ ckan/lib/munge.py | 7 +++ ckan/lib/uploader.py | 63 +++++++++++++++++++ ckan/logic/schema.py | 1 + ckan/templates/group/snippets/group_form.html | 11 +++- ckan/templates/group/snippets/group_item.html | 2 +- .../snippets/organization_form.html | 10 ++- .../snippets/organization_item.html | 2 +- ckan/templates/snippets/group.html | 2 +- ckan/templates/snippets/group_item.html | 2 +- ckan/templates/snippets/organization.html | 2 +- .../templates/snippets/organization_item.html | 2 +- 15 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 ckan/lib/uploader.py diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 4df0aa84cb0..63f5568cd5f 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -147,6 +147,12 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): cache_max_age=static_max_age) static_parsers = [static_app, app] + storage_directory = config.get('ckan.storage_path') + if storage_directory: + storage_app = StaticURLParser(storage_directory, + cache_max_age=static_max_age) + static_parsers.insert(0, storage_app) + # Configurable extra static file paths extra_static_parsers = [] for public_path in config.get('extra_public_paths', '').split(','): diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index ccef46bfe78..a11dd03cc40 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -1,14 +1,18 @@ import re +import os import logging import genshi +import cgi import datetime from urllib import urlencode from pylons.i18n import get_lang +from pylons import config import ckan.lib.base as base import ckan.lib.helpers as h import ckan.lib.maintain as maintain +import ckan.lib.uploader as uploader import ckan.lib.navl.dictization_functions as dict_fns import ckan.logic as logic import ckan.lib.search as search @@ -421,6 +425,9 @@ def new(self, data=None, errors=None, error_summary=None): return self._save_new(context, group_type) data = data or {} + if not data.get('image_url', '').startswith('http'): + data.pop('image_url', None) + errors = errors or {} error_summary = error_summary or {} vars = {'data': data, 'errors': errors, @@ -445,10 +452,17 @@ def edit(self, id, data=None, errors=None, error_summary=None): return self._save_edit(id, context) try: + ## only needs removing when form repopulates + if data and not data.get('image_url', '').startswith('http'): + data.pop('image_url', None) old_data = self._action('group_show')(context, data_dict) c.grouptitle = old_data.get('title') c.groupname = old_data.get('name') data = data or old_data + image_url = data.get('image_url') + if image_url and not image_url.startswith('http'): + data['upload_file'] = image_url + data['image_url'] = '' except NotFound: abort(404, _('Group not found')) except NotAuthorized: @@ -456,6 +470,7 @@ def edit(self, id, data=None, errors=None, error_summary=None): group = context.get("group") c.group = group + context.pop('for_edit') c.group_dict = self._action('group_show')(context, data_dict) try: @@ -483,12 +498,17 @@ def _get_group_type(self, id): def _save_new(self, context, group_type=None): try: + upload = uploader.Upload('group') + upload.register_request(request, 'image_url', + 'image_upload', 'clear_upload') + data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) data_dict['type'] = group_type or 'group' context['message'] = data_dict.get('log_message', '') data_dict['users'] = [{'name': c.user, 'capacity': 'admin'}] group = self._action('group_create')(context, data_dict) + upload.upload() # Redirect to the appropriate _read route for the type of group h.redirect_to(group['type'] + '_read', id=group['name']) @@ -514,13 +534,19 @@ def _force_reindex(self, grp): def _save_edit(self, id, context): try: + old_group = self._action('group_show')(context, {"id": id}) + old_image_url = old_group.get('image_url') + upload = uploader.Upload('group', old_image_url) + upload.register_request(request, 'image_url', + 'image_upload', 'clear_upload') + data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) context['message'] = data_dict.get('log_message', '') data_dict['id'] = id context['allow_partial_update'] = True group = self._action('group_update')(context, data_dict) - + upload.upload() if id != group['name']: self._force_reindex(group) diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index b3a83235cd6..98756abc2ec 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -10,6 +10,7 @@ import ckan.lib.dictization as d import ckan.new_authz as new_authz import ckan.lib.search as search +import ckan.lib.munge as munge ## package save @@ -41,6 +42,16 @@ def group_list_dictize(obj_list, context, group_dict['display_name'] = obj.display_name + image_url = group_dict.get('image_url') + group_dict['image_display_url'] = image_url + if image_url and not image_url.startswith('http'): + #munge here should not have an effect only doing it incase + #of potential vulnerability of dodgy api input + image_url = munge.munge_filename(image_url) + group_dict['image_display_url'] = h.url_for_static( + 'uploads/group/%s' % group_dict.get('image_url') + ) + if obj.is_organization: group_dict['packages'] = query.facets['owner_org'].get(obj.id, 0) else: @@ -354,6 +365,16 @@ def group_dictize(group, context): for item in plugins.PluginImplementations(plugin): result_dict = item.before_view(result_dict) + if not context.get('for_edit'): + image_url = result_dict.get('image_url') + result_dict['image_display_url'] = image_url + if image_url and not image_url.startswith('http'): + #munge here should not have an effect only doing it incase + #of potential vulnerability of dodgy api input + image_url = munge.munge_filename(image_url) + result_dict['image_display_url'] = h.url_for_static( + 'uploads/group/%s' % result_dict.get('image_url') + ) return result_dict def tag_list_dictize(tag_list, context): diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index c70d74d7dbc..fceab6f1b6a 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -1663,6 +1663,10 @@ def new_activities(): action = logic.get_action('dashboard_new_activities_count') return action({}, {}) +def uploads_enabled(): + if config.get('ckan.storage_path'): + return True + return False # these are the functions that will end up in `h` template helpers __allowed_functions__ = [ @@ -1761,4 +1765,5 @@ def new_activities(): 'radio', 'submit', 'asbool', + 'uploads_enabled', ] diff --git a/ckan/lib/munge.py b/ckan/lib/munge.py index c943e1b113f..a47c8c02258 100644 --- a/ckan/lib/munge.py +++ b/ckan/lib/munge.py @@ -105,6 +105,13 @@ def munge_tag(tag): tag = _munge_to_length(tag, model.MIN_TAG_LENGTH, model.MAX_TAG_LENGTH) return tag +def munge_filename(filename): + filename = substitute_ascii_equivalents(filename) + filename = filename.lower().strip() + filename = re.sub(r'[^a-zA-Z0-9. ]', '', filename).replace(' ', '-') + filename = _munge_to_length(filename, 3, 100) + return filename + def _munge_to_length(string, min_length, max_length): '''Pad/truncates a string''' if len(string) < min_length: diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py new file mode 100644 index 00000000000..5fc144e3aef --- /dev/null +++ b/ckan/lib/uploader.py @@ -0,0 +1,63 @@ +import os +import cgi +import pylons +import datetime +import ckan.lib.munge as munge + +class Upload(object): + def __init__(self, object_type, old_filename=None): + path = pylons.config.get('ckan.storage_path', '/tmp') + self.storage_path = os.path.join(path, 'uploads', object_type) + try: + os.makedirs(self.storage_path) + except OSError, e: + pass + self.object_type = object_type + self.old_filename = old_filename + if old_filename: + self.old_filepath = os.path.join(self.storage_path, old_filename) + self.filename = None + self.filepath = None + + def register_request(self, request, url_field, file_field, clear_field): + + self.request = request + self.url = request.POST.get(url_field) + self.clear = request.POST.get(clear_field) + self.upload_field_storage = request.POST.pop(file_field, None) + + if isinstance(self.upload_field_storage, cgi.FieldStorage): + self.filename = self.upload_field_storage.filename + self.filename = str(datetime.datetime.utcnow()) + self.filename + self.filename = munge.munge_filename(self.filename) + self.filepath = os.path.join(self.storage_path, self.filename) + request.POST[url_field] = self.filename + self.upload_file = self.upload_field_storage.file + self.tmp_filepath = self.filepath + '~' + ### keep the file if there has been no change + if self.old_filename and not self.url and not self.clear: + request.POST[url_field] = self.old_filename + + + def upload(self): + + if self.filename: + output_file = open(self.tmp_filepath, 'wb') + self.upload_file.seek(0) + while True: + data = self.upload_file.read(2 ** 20) #mb chuncks + if not data: + break + output_file.write(data) + output_file.close() + os.rename(self.tmp_filepath, self.filepath) + self.clear = True + elif self.url: + self.clear = True + + if (self.clear and self.old_filename + and not self.old_filename.startswith('http')): + try: + os.remove(self.old_filepath) + except OSError, e: + pass diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f8fe8df7fbd..90465b7f7ec 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -262,6 +262,7 @@ def default_group_schema(): 'title': [ignore_missing, unicode], 'description': [ignore_missing, unicode], 'image_url': [ignore_missing, unicode], + 'image_display_url': [ignore_missing, unicode], 'type': [ignore_missing, unicode], 'state': [ignore_not_group_admin, ignore_missing], 'created': [ignore], diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html index 6d4d3216b26..1b9cac179c7 100644 --- a/ckan/templates/group/snippets/group_form.html +++ b/ckan/templates/group/snippets/group_form.html @@ -1,6 +1,6 @@ {% import 'macros/form.html' as form %} -
+ {% block error_summary %} {{ form.errors(error_summary) }} {% endblock %} @@ -21,6 +21,15 @@ {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + + {% if h.uploads_enabled() %} + {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} + {% set upload_file = data.get('upload_file', '') %} + {% if upload_file %} + {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% endif %} + {% endif %} + {% endblock %} {% block custom_fields %} diff --git a/ckan/templates/group/snippets/group_item.html b/ckan/templates/group/snippets/group_item.html index 9e32ce86e6a..34688b57c96 100644 --- a/ckan/templates/group/snippets/group_item.html +++ b/ckan/templates/group/snippets/group_item.html @@ -14,7 +14,7 @@ {% set url = h.url_for(group.type ~ '_read', action='read', id=group.name) %}
  • {% block image %} - {{ group.name }} + {{ group.name }} {% endblock %} {% block title %}

    {{ group.display_name }}

    diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 7b05e7d1542..7aee9ded32f 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -1,6 +1,6 @@ {% import 'macros/form.html' as form %} - + {% block error_summary %} {{ form.errors(error_summary) }} {% endblock %} @@ -21,6 +21,14 @@ {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {% if h.uploads_enabled() %} + {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} + {% set upload_file = data.get('upload_file', '') %} + {% if upload_file %} + {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% endif %} + {% endif %} + {% endblock %} {% block custom_fields %} diff --git a/ckan/templates/organization/snippets/organization_item.html b/ckan/templates/organization/snippets/organization_item.html index 818d5cc338a..8118fa9a1c6 100644 --- a/ckan/templates/organization/snippets/organization_item.html +++ b/ckan/templates/organization/snippets/organization_item.html @@ -14,7 +14,7 @@ {% set url = h.url_for(organization.type ~ '_read', action='read', id=organization.name) %}
  • {% block image %} - {{ organization.name }} + {{ organization.name }} {% endblock %} {% block title %}

    {{ organization.display_name }}

    diff --git a/ckan/templates/snippets/group.html b/ckan/templates/snippets/group.html index db549fda75a..db758d9bc96 100644 --- a/ckan/templates/snippets/group.html +++ b/ckan/templates/snippets/group.html @@ -14,7 +14,7 @@
    - {{ group.name }} + {{ group.name }}

    {{ group.title or group.name }}

    diff --git a/ckan/templates/snippets/group_item.html b/ckan/templates/snippets/group_item.html index 74c68fbca28..60f43f63b79 100644 --- a/ckan/templates/snippets/group_item.html +++ b/ckan/templates/snippets/group_item.html @@ -5,7 +5,7 @@ {% set truncate_title = truncate_title or 0 %} {% set title = group.title or group.name %} - {{ group.name }} + {{ group.name }}

    {{ group.title or group.name }}

    {% if group.description %} diff --git a/ckan/templates/snippets/organization.html b/ckan/templates/snippets/organization.html index ef08a9ed440..e0f5417024a 100644 --- a/ckan/templates/snippets/organization.html +++ b/ckan/templates/snippets/organization.html @@ -23,7 +23,7 @@

    {{ _('Organization') }} {% block image %}
    - {{ organization.name }} + {{ organization.name }}
    {% endblock %} diff --git a/ckan/templates/snippets/organization_item.html b/ckan/templates/snippets/organization_item.html index 0213723defa..06579756f32 100644 --- a/ckan/templates/snippets/organization_item.html +++ b/ckan/templates/snippets/organization_item.html @@ -3,7 +3,7 @@ {% set url=h.url_for(controller='organization', action='read', id=organization.name) %} {% set truncate=truncate or 0 %} - {{ organization.name }} + {{ organization.name }}

    {{ organization.title or organization.name }}

    {% if organization.description %} From 66105d360e010114751ff0280b07196e6f73445a Mon Sep 17 00:00:00 2001 From: kindly Date: Tue, 8 Oct 2013 03:02:12 +0100 Subject: [PATCH 05/49] [#1262] fix how url field is handled --- ckan/config/middleware.py | 9 ++++++++- ckan/controllers/group.py | 7 ------- ckan/lib/dictization/model_dictize.py | 20 +++++++++---------- ckan/lib/uploader.py | 16 +++++++-------- ckan/templates/group/snippets/group_form.html | 7 +++---- ckan/templates/group/snippets/info.html | 2 +- 6 files changed, 29 insertions(+), 32 deletions(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 63f5568cd5f..7699596138d 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -4,6 +4,7 @@ import logging import json import hashlib +import os import sqlalchemy as sa from beaker.middleware import CacheMiddleware, SessionMiddleware @@ -149,7 +150,13 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): storage_directory = config.get('ckan.storage_path') if storage_directory: - storage_app = StaticURLParser(storage_directory, + path = os.path.join(storage_directory, 'storage') + try: + os.makedirs(path) + except OSError, e: + pass + + storage_app = StaticURLParser(path, cache_max_age=static_max_age) static_parsers.insert(0, storage_app) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index a11dd03cc40..ed12edcb082 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -452,17 +452,10 @@ def edit(self, id, data=None, errors=None, error_summary=None): return self._save_edit(id, context) try: - ## only needs removing when form repopulates - if data and not data.get('image_url', '').startswith('http'): - data.pop('image_url', None) old_data = self._action('group_show')(context, data_dict) c.grouptitle = old_data.get('title') c.groupname = old_data.get('name') data = data or old_data - image_url = data.get('image_url') - if image_url and not image_url.startswith('http'): - data['upload_file'] = image_url - data['image_url'] = '' except NotFound: abort(404, _('Group not found')) except NotAuthorized: diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index 98756abc2ec..019e648140b 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -365,16 +365,16 @@ def group_dictize(group, context): for item in plugins.PluginImplementations(plugin): result_dict = item.before_view(result_dict) - if not context.get('for_edit'): - image_url = result_dict.get('image_url') - result_dict['image_display_url'] = image_url - if image_url and not image_url.startswith('http'): - #munge here should not have an effect only doing it incase - #of potential vulnerability of dodgy api input - image_url = munge.munge_filename(image_url) - result_dict['image_display_url'] = h.url_for_static( - 'uploads/group/%s' % result_dict.get('image_url') - ) + image_url = result_dict.get('image_url') + result_dict['image_display_url'] = image_url + if image_url and not image_url.startswith('http'): + #munge here should not have an effect only doing it incase + #of potential vulnerability of dodgy api input + image_url = munge.munge_filename(image_url) + result_dict['image_display_url'] = h.url_for_static( + 'uploads/group/%s' % result_dict.get('image_url'), + qualified = True + ) return result_dict def tag_list_dictize(tag_list, context): diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index 5fc144e3aef..bcfdd7e9ecc 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -7,7 +7,7 @@ class Upload(object): def __init__(self, object_type, old_filename=None): path = pylons.config.get('ckan.storage_path', '/tmp') - self.storage_path = os.path.join(path, 'uploads', object_type) + self.storage_path = os.path.join(path, 'storage', 'uploads', object_type) try: os.makedirs(self.storage_path) except OSError, e: @@ -20,9 +20,7 @@ def __init__(self, object_type, old_filename=None): self.filepath = None def register_request(self, request, url_field, file_field, clear_field): - - self.request = request - self.url = request.POST.get(url_field) + self.url = request.POST.get(url_field, '') self.clear = request.POST.get(clear_field) self.upload_field_storage = request.POST.pop(file_field, None) @@ -35,12 +33,14 @@ def register_request(self, request, url_field, file_field, clear_field): self.upload_file = self.upload_field_storage.file self.tmp_filepath = self.filepath + '~' ### keep the file if there has been no change - if self.old_filename and not self.url and not self.clear: - request.POST[url_field] = self.old_filename + elif self.old_filename and not self.old_filename.startswith('http'): + if not self.clear: + request.POST[url_field] = self.old_filename + if self.clear and self.url == self.old_filename: + request.POST[url_field] = '' def upload(self): - if self.filename: output_file = open(self.tmp_filepath, 'wb') self.upload_file.seek(0) @@ -52,8 +52,6 @@ def upload(self): output_file.close() os.rename(self.tmp_filepath, self.filepath) self.clear = True - elif self.url: - self.clear = True if (self.clear and self.old_filename and not self.old_filename.startswith('http')): diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html index 1b9cac179c7..322f9ebff8a 100644 --- a/ckan/templates/group/snippets/group_form.html +++ b/ckan/templates/group/snippets/group_form.html @@ -19,14 +19,13 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my group...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} {% if h.uploads_enabled() %} {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% set upload_file = data.get('upload_file', '') %} - {% if upload_file %} - {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% if data.image_url and not data.image_url.startswith('http') %} + {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} {% endif %} {% endif %} diff --git a/ckan/templates/group/snippets/info.html b/ckan/templates/group/snippets/info.html index b7c749ab147..dc86036bb4c 100644 --- a/ckan/templates/group/snippets/info.html +++ b/ckan/templates/group/snippets/info.html @@ -2,7 +2,7 @@
    - {{ group.name }} + {{ group.name }}

    {{ group.display_name }}

    From 1b62423d9d013dbc4481bbddd5a1037b49e52529 Mon Sep 17 00:00:00 2001 From: kindly Date: Wed, 9 Oct 2013 03:12:00 +0100 Subject: [PATCH 06/49] [1262] use logic instead of requests --- ckan/controllers/group.py | 11 ----------- ckan/lib/uploader.py | 14 +++++++------- ckan/logic/action/create.py | 7 ++++++- ckan/logic/action/update.py | 7 +++++++ .../organization/snippets/organization_form.html | 7 +++---- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index ed12edcb082..adbd3d7cacc 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -12,7 +12,6 @@ import ckan.lib.base as base import ckan.lib.helpers as h import ckan.lib.maintain as maintain -import ckan.lib.uploader as uploader import ckan.lib.navl.dictization_functions as dict_fns import ckan.logic as logic import ckan.lib.search as search @@ -491,17 +490,12 @@ def _get_group_type(self, id): def _save_new(self, context, group_type=None): try: - upload = uploader.Upload('group') - upload.register_request(request, 'image_url', - 'image_upload', 'clear_upload') - data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) data_dict['type'] = group_type or 'group' context['message'] = data_dict.get('log_message', '') data_dict['users'] = [{'name': c.user, 'capacity': 'admin'}] group = self._action('group_create')(context, data_dict) - upload.upload() # Redirect to the appropriate _read route for the type of group h.redirect_to(group['type'] + '_read', id=group['name']) @@ -529,17 +523,12 @@ def _save_edit(self, id, context): try: old_group = self._action('group_show')(context, {"id": id}) old_image_url = old_group.get('image_url') - upload = uploader.Upload('group', old_image_url) - upload.register_request(request, 'image_url', - 'image_upload', 'clear_upload') - data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) context['message'] = data_dict.get('log_message', '') data_dict['id'] = id context['allow_partial_update'] = True group = self._action('group_update')(context, data_dict) - upload.upload() if id != group['name']: self._force_reindex(group) diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index bcfdd7e9ecc..9b7cea2922e 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -19,25 +19,25 @@ def __init__(self, object_type, old_filename=None): self.filename = None self.filepath = None - def register_request(self, request, url_field, file_field, clear_field): - self.url = request.POST.get(url_field, '') - self.clear = request.POST.get(clear_field) - self.upload_field_storage = request.POST.pop(file_field, None) + def update_data_dict(self, data_dict, url_field, file_field, clear_field): + self.url = data_dict.get(url_field, '') + self.clear = data_dict.pop(clear_field, None) + self.upload_field_storage = data_dict.pop(file_field, None) if isinstance(self.upload_field_storage, cgi.FieldStorage): self.filename = self.upload_field_storage.filename self.filename = str(datetime.datetime.utcnow()) + self.filename self.filename = munge.munge_filename(self.filename) self.filepath = os.path.join(self.storage_path, self.filename) - request.POST[url_field] = self.filename + data_dict[url_field] = self.filename self.upload_file = self.upload_field_storage.file self.tmp_filepath = self.filepath + '~' ### keep the file if there has been no change elif self.old_filename and not self.old_filename.startswith('http'): if not self.clear: - request.POST[url_field] = self.old_filename + data_dict[url_field] = self.old_filename if self.clear and self.url == self.old_filename: - request.POST[url_field] = '' + data_dict[url_field] = '' def upload(self): diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index baac5a2154c..702a6028d99 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -15,6 +15,7 @@ import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.dictization.model_save as model_save import ckan.lib.navl.dictization_functions +import ckan.lib.uploader as uploader from ckan.common import _ @@ -446,6 +447,7 @@ def member_create(context, data_dict=None): if not obj: raise NotFound('%s was not found.' % obj_type.title()) + # User must be able to update the group to add a member to it _check_access('group_update', context, data_dict) @@ -475,7 +477,9 @@ def _group_or_org_create(context, data_dict, is_org=False): parent = context.get('parent', None) data_dict['is_organization'] = is_org - + upload = uploader.Upload('group') + upload.update_data_dict(data_dict, 'image_url', + 'image_upload', 'clear_upload') # get the schema group_plugin = lib_plugins.lookup_group_plugin( group_type=data_dict.get('type')) @@ -561,6 +565,7 @@ def _group_or_org_create(context, data_dict, is_org=False): logic.get_action('activity_create')(activity_create_context, activity_dict) + upload.upload() if not context.get('defer_commit'): model.repo.commit() context["group"] = group diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 3db2d0d19cf..f8aee9f62fb 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -19,6 +19,7 @@ import ckan.lib.plugins as lib_plugins import ckan.lib.email_notifications as email_notifications import ckan.lib.search as search +import ckan.lib.uploader as uploader from ckan.common import _, request @@ -424,6 +425,10 @@ def _group_or_org_update(context, data_dict, is_org=False): except AttributeError: schema = group_plugin.form_to_db_schema() + upload = uploader.Upload('group', group.image_url) + upload.update_data_dict(data_dict, 'image_url', + 'image_upload', 'clear_upload') + if is_org: _check_access('organization_update', context, data_dict) else: @@ -528,9 +533,11 @@ def _group_or_org_update(context, data_dict, is_org=False): # TODO: Also create an activity detail recording what exactly changed # in the group. + upload.upload() if not context.get('defer_commit'): model.repo.commit() + return model_dictize.group_dictize(group, context) def group_update(context, data_dict): diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 7aee9ded32f..0a2134e89f7 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -19,13 +19,12 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my organization...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} {% if h.uploads_enabled() %} {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% set upload_file = data.get('upload_file', '') %} - {% if upload_file %} - {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% if data.image_url and not data.image_url.startswith('http') %} + {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} {% endif %} {% endif %} From c4d673327c6b83e01ead0091246cecb83875160a Mon Sep 17 00:00:00 2001 From: kindly Date: Thu, 10 Oct 2013 20:40:49 +0100 Subject: [PATCH 07/49] [#1262] fix test failures --- ckan/lib/uploader.py | 12 +++++++----- ckan/tests/lib/test_dictization.py | 2 ++ ckan/tests/lib/test_dictization_schema.py | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index 9b7cea2922e..eb0f28eec54 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -4,10 +4,12 @@ import datetime import ckan.lib.munge as munge + class Upload(object): def __init__(self, object_type, old_filename=None): path = pylons.config.get('ckan.storage_path', '/tmp') - self.storage_path = os.path.join(path, 'storage', 'uploads', object_type) + self.storage_path = os.path.join(path, 'storage', + 'uploads', object_type) try: os.makedirs(self.storage_path) except OSError, e: @@ -22,7 +24,7 @@ def __init__(self, object_type, old_filename=None): def update_data_dict(self, data_dict, url_field, file_field, clear_field): self.url = data_dict.get(url_field, '') self.clear = data_dict.pop(clear_field, None) - self.upload_field_storage = data_dict.pop(file_field, None) + self.upload_field_storage = data_dict.pop(file_field, None) if isinstance(self.upload_field_storage, cgi.FieldStorage): self.filename = self.upload_field_storage.filename @@ -39,13 +41,13 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): if self.clear and self.url == self.old_filename: data_dict[url_field] = '' - def upload(self): if self.filename: output_file = open(self.tmp_filepath, 'wb') self.upload_file.seek(0) while True: - data = self.upload_file.read(2 ** 20) #mb chuncks + # mb chuncks + data = self.upload_file.read(2 ** 20) if not data: break output_file.write(data) @@ -54,7 +56,7 @@ def upload(self): self.clear = True if (self.clear and self.old_filename - and not self.old_filename.startswith('http')): + and not self.old_filename.startswith('http')): try: os.remove(self.old_filepath) except OSError, e: diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py index 57f29963910..dcf955d3cc5 100644 --- a/ckan/tests/lib/test_dictization.py +++ b/ckan/tests/lib/test_dictization.py @@ -923,6 +923,7 @@ def test_16_group_dictized(self): 'capacity' : 'public', 'display_name': u'simple', 'image_url': u'', + 'image_display_url': u'', 'name': u'simple', 'packages': 0, 'state': u'active', @@ -944,6 +945,7 @@ def test_16_group_dictized(self): 'name': u'help', 'display_name': u'help', 'image_url': u'', + 'image_display_url': u'', 'package_count': 2, 'is_organization': False, 'packages': [{'author': None, diff --git a/ckan/tests/lib/test_dictization_schema.py b/ckan/tests/lib/test_dictization_schema.py index 5a94d2e54e8..9507be2806f 100644 --- a/ckan/tests/lib/test_dictization_schema.py +++ b/ckan/tests/lib/test_dictization_schema.py @@ -160,6 +160,7 @@ def test_2_group_schema(self): 'is_organization': False, 'type': u'group', 'image_url': u'', + 'image_display_url': u'', 'packages': sorted([{'id': group_pack[0].id, 'name': group_pack[0].name, 'title': group_pack[0].title}, From 5023e0c3354ca03475b06ee11a3e47f867adedaf Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Mon, 14 Oct 2013 10:51:01 +0530 Subject: [PATCH 08/49] Don't exempt sysadmins --- ckan/logic/auth/create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index ec38483bd0d..ca3a2d70b6d 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -104,6 +104,7 @@ def rating_create(context, data_dict): return {'success': True} +@logic.auth_sysadmins_check def user_create(context, data_dict=None): using_api = 'api_version' in context create_user_via_api = new_authz.check_config_permission( From 878772cfc37932805cca58439150d702506fe043 Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Mon, 14 Oct 2013 12:22:29 +0530 Subject: [PATCH 09/49] [#1226] Add tests --- ckan/tests/functional/api/test_user.py | 190 ++++++++++++++++++++++++- ckan/tests/logic/test_auth.py | 3 +- 2 files changed, 189 insertions(+), 4 deletions(-) diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py index 06dd35dd7fc..26565810c2f 100644 --- a/ckan/tests/functional/api/test_user.py +++ b/ckan/tests/functional/api/test_user.py @@ -1,20 +1,26 @@ +import paste +from pylons import config from nose.tools import assert_equal import ckan.logic as logic +import ckan.new_authz as new_authz from ckan import model from ckan.lib.create_test_data import CreateTestData from ckan.tests import TestController as ControllerTestCase +from ckan.tests.pylons_controller import PylonsTestCase from ckan.tests import url_for +import ckan.config.middleware +from ckan.common import json class TestUserApi(ControllerTestCase): @classmethod def setup_class(cls): CreateTestData.create() - + @classmethod def teardown_class(cls): model.repo.rebuild_db() - + def test_autocomplete(self): response = self.app.get( url=url_for(controller='api', action='user_autocomplete', ver=2), @@ -51,8 +57,186 @@ def test_autocomplete_limit(self): print response.json assert_equal(len(response.json), 1) + +class TestCreateUserApiDisabled(PylonsTestCase): + ''' + Tests for the creating user when create_user_via_api is disabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + cls.sysadmin_user = model.User.get('testsysadmin') + PylonsTestCase.setup_class() + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled_sysadmin(self): + params = { + 'name': 'testinganewusersysadmin', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + def test_user_create_api_disabled_anon(self): + params = { + 'name': 'testinganewuseranon', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + +class TestCreateUserApiEnabled(PylonsTestCase): + ''' + Tests for the creating user when create_user_via_api is enabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + config['ckan.auth.create_user_via_api'] = True + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + PylonsTestCase.setup_class() + cls.sysadmin_user = model.User.get('testsysadmin') + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled_sysadmin(self): + params = { + 'name': 'testinganewusersysadmin', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': + str(self.sysadmin_user.apikey)}) + res_dict = res.json + assert res_dict['success'] is True + + def test_user_create_api_disabled_anon(self): + params = { + 'name': 'testinganewuseranon', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is True + + +class TestCreateUserWebDisabled(PylonsTestCase): + ''' + Tests for the creating user by create_user_via_web is disabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + config['ckan.auth.create_user_via_web'] = False + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + cls.sysadmin_user = model.User.get('testsysadmin') + PylonsTestCase.setup_class() + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled(self): + params = { + 'name': 'testinganewuser', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + +class TestCreateUserWebEnabled(PylonsTestCase): + ''' + Tests for the creating user by create_user_via_web is enabled. + ''' + + @classmethod + def setup_class(cls): + CreateTestData.create() + cls._original_config = config.copy() + config['ckan.auth.create_user_via_web'] = True + new_authz.clear_auth_functions_cache() + wsgiapp = ckan.config.middleware.make_app(config['global_conf'], + **config) + cls.app = paste.fixture.TestApp(wsgiapp) + cls.sysadmin_user = model.User.get('testsysadmin') + PylonsTestCase.setup_class() + + @classmethod + def teardown_class(cls): + config.clear() + config.update(cls._original_config) + new_authz.clear_auth_functions_cache() + PylonsTestCase.teardown_class() + + model.repo.rebuild_db() + + def test_user_create_api_disabled(self): + params = { + 'name': 'testinganewuser', + 'email': 'testinganewuser@ckan.org', + 'password': 'random', + } + res = self.app.post('/api/3/action/user_create', json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + expect_errors=True) + res_dict = res.json + assert res_dict['success'] is False + + class TestUserActions(object): - + @classmethod def setup_class(cls): CreateTestData.create() diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 3ea9f2db39f..b73834d2657 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -11,8 +11,9 @@ 'user_create_organizations': False, 'user_delete_groups': False, 'user_delete_organizations': False, - 'create_user_via_api': False, 'create_unowned_dataset': False, + 'create_user_via_api': False, + 'create_user_via_web': True, } From 3273ebf7d0e06dbf3d854a25f701e651fe88bb0b Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Mon, 14 Oct 2013 15:34:07 +0530 Subject: [PATCH 10/49] [#1226] Return the old behaviour for create_user --- ckan/logic/auth/create.py | 1 - ckan/tests/functional/api/test_user.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 13e02125855..53c8ae153ab 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -105,7 +105,6 @@ def rating_create(context, data_dict): @logic.auth_allow_anonymous_access -@logic.auth_sysadmins_check def user_create(context, data_dict=None): using_api = 'api_version' in context create_user_via_api = new_authz.check_config_permission( diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py index ba67b97455e..cdbe039d012 100644 --- a/ckan/tests/functional/api/test_user.py +++ b/ckan/tests/functional/api/test_user.py @@ -83,7 +83,7 @@ def teardown_class(cls): model.repo.rebuild_db() - def test_user_create_api_disabled_sysadmin(self): + def test_user_create_api_enabled_sysadmin(self): params = { 'name': 'testinganewusersysadmin', 'email': 'testinganewuser@ckan.org', @@ -93,7 +93,7 @@ def test_user_create_api_disabled_sysadmin(self): extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, expect_errors=True) res_dict = res.json - assert res_dict['success'] is False + assert res_dict['success'] is True def test_user_create_api_disabled_anon(self): params = { @@ -190,7 +190,6 @@ def test_user_create_api_disabled(self): 'password': 'random', } res = self.app.post('/api/3/action/user_create', json.dumps(params), - extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, expect_errors=True) res_dict = res.json assert res_dict['success'] is False @@ -229,7 +228,6 @@ def test_user_create_api_disabled(self): 'password': 'random', } res = self.app.post('/api/3/action/user_create', json.dumps(params), - extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, expect_errors=True) res_dict = res.json assert res_dict['success'] is False From bb6cb7d0ff417b72f002dd372bdfd09ba560a766 Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 14 Oct 2013 21:46:39 +0100 Subject: [PATCH 11/49] [#1261] add filesize limit, make storage folder backwards compatable, document a bit more --- ckan/lib/helpers.py | 3 +- ckan/lib/uploader.py | 69 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index fceab6f1b6a..3aa73ad3a8b 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -37,6 +37,7 @@ import ckan.lib.maintain as maintain import ckan.lib.datapreview as datapreview import ckan.logic as logic +import ckan.lib.uploader as uploader from ckan.common import ( _, ungettext, g, c, request, session, json, OrderedDict @@ -1664,7 +1665,7 @@ def new_activities(): return action({}, {}) def uploads_enabled(): - if config.get('ckan.storage_path'): + if uploader.get_storage_path(): return True return False diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index eb0f28eec54..880488b8161 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -3,11 +3,51 @@ import pylons import datetime import ckan.lib.munge as munge +import logging +import ckan.logic as logic + +log = logging.getLogger(__name__) + +_storage_path = None + +def get_storage_path(): + '''Function to cache storage path''' + global _storage_path + + #None means it has not been set. False means not in config. + if _storage_path is None: + storage_path = pylons.config.get('ckan.storage_path') + ofs_impl = pylons.config.get('ofs.impl') + ofs_storage_dir = pylons.config.get('ofs.storage_dir') + if storage_path: + _storage_path = storage_path + elif ofs_impl == 'pairtree' and ofs_storage_dir: + log.warn('''Please use config option ckan.storage_path instaed of + ofs.storage_path''') + _storage_path = ofs_storage_dir + return _storage_path + elif ofs_impl: + log.critical('''We only support local file storage form version 2.2 of ckan + please specify ckan.storage_path in your config for your uploads''') + _storage_path = False + else: + log.critical('''Please specify a ckan.storage_path in your config for your uploads''') + _storage_path = False + + return _storage_path + class Upload(object): def __init__(self, object_type, old_filename=None): - path = pylons.config.get('ckan.storage_path', '/tmp') + ''' Setup upload by creating a subdirectory of the storage directory of name + object_type. old_filename is the name of the file in the url field last time''' + self.storage_path = None + self.filename = None + self.filepath = None + path = get_storage_path() + if not path: + return self.storage_path = os.path.join(path, 'storage', 'uploads', object_type) try: @@ -18,12 +58,20 @@ def __init__(self, object_type, old_filename=None): self.old_filename = old_filename if old_filename: self.old_filepath = os.path.join(self.storage_path, old_filename) - self.filename = None - self.filepath = None def update_data_dict(self, data_dict, url_field, file_field, clear_field): + ''' Manipulate data from the data_dict. url_field is the name of the field + where the upload is going to be. file_field is name of the key where the + FieldStorage is kept (i.e the field where the file data actually is). clear_field + is the name of a boolean field which requests the upload to be deleted. This needs + to be called before it reaches any validators''' + + if not self.storage_path: + return + self.url = data_dict.get(url_field, '') self.clear = data_dict.pop(clear_field, None) + self.file_field = file_field self.upload_field_storage = data_dict.pop(file_field, None) if isinstance(self.upload_field_storage, cgi.FieldStorage): @@ -41,16 +89,27 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): if self.clear and self.url == self.old_filename: data_dict[url_field] = '' - def upload(self): + def upload(self, max_size=2): + ''' Actually upload the file. This should happen just before a commit but after + the data has been validated and flushed to the db. This is so we do not store anything + unless the request is actually good. Max_size is size in MB maximum of the file''' + if self.filename: output_file = open(self.tmp_filepath, 'wb') self.upload_file.seek(0) + current_size = 0 while True: - # mb chuncks + current_size = current_size + 1 + # MB chuncks data = self.upload_file.read(2 ** 20) if not data: break output_file.write(data) + if current_size > max_size: + os.remove(self.tmp_filepath) + raise logic.ValidationError( + {self.file_field: ['File upload too large']} + ) output_file.close() os.rename(self.tmp_filepath, self.filepath) self.clear = True From 3dee85226f1fa6a29a03d598c8c11be1c5dba4ca Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:10:09 +0100 Subject: [PATCH 12/49] [#1262] First working version of the JS module for image uploading --- .../base/javascript/modules/image-upload.js | 147 ++++++++++++++++++ ckan/public/base/javascript/resource.config | 1 + ckan/public/base/less/forms.less | 34 ++++ ckan/templates/group/snippets/group_form.html | 10 +- ckan/templates/macros/form.html | 27 ++++ .../snippets/organization_form.html | 9 +- 6 files changed, 211 insertions(+), 17 deletions(-) create mode 100644 ckan/public/base/javascript/modules/image-upload.js diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js new file mode 100644 index 00000000000..69ca8f93c05 --- /dev/null +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -0,0 +1,147 @@ +/* Image Upload + * + * + * Options + * - id: what's the id of the context? + */ +this.ckan.module('image-upload', function($, _) { + return { + /* options object can be extended using data-module-* attributes */ + options: { + is_url: true, + has_image: false, + i18n: { + upload: _('From computer'), + url: _('From web'), + remove: _('Remove'), + label: _('Upload image'), + label_url: _('Image URL'), + remove_tooltip: _('Reset this') + }, + template: [ + '' + ].join("\n") + }, + + state: { + attached: 1, + blank: 2, + web: 3 + }, + + /* Initialises the module setting up elements and event listeners. + * + * Returns nothing. + */ + initialize: function () { + $.proxyAll(this, /_on/); + var options = this.options; + + // firstly setup the fields + this.input = $('input[name="image_upload"]', this.el); + this.field_url = $('input[name="image_url"]', this.el).parents('.control-group'); + this.field_image = this.input.parents('.control-group'); + + var checkbox = $('input[name="clear_upload"]', this.el); + if (checkbox.length > 0) { + options.has_image = true; + checkbox.parents('.control-group').remove(); + } + + this.field_clear = $('') + .appendTo(this.el); + + this.button_url = $(' '+this.i18n('url')+'') + .on('click', this._onFromWeb) + .insertAfter(this.input); + + this.button_upload = $(''+this.i18n('upload')+'') + .insertAfter(this.input); + + this.button_remove = $('') + .text(this.i18n('remove')) + .on('click', this._onRemove) + .insertAfter(this.button_upload); + + $('') + .prop('title', this.i18n('remove_tooltip')) + .on('click', this._onRemove) + .insertBefore($('input', this.field_url)); + + $('label[for="field-image-upload"]').text(this.i18n('label')); + + this.input + .on('mouseover', this._onInputMouseOver) + .on('mouseout', this._onInputMouseOut) + .on('change', this._onInputChange) + .css('width', this.button_upload.outerWidth()) + .hide(); + + this.fields = $('') + .add(this.button_remove) + .add(this.button_upload) + .add(this.button_url) + .add(this.input) + .add(this.field_url) + .add(this.field_image); + + if (options.is_url) { + this.changeState(this.state.web); + } else if (options.has_image) { + this.changeState(this.state.attached); + } else { + this.changeState(this.state.blank); + } + + }, + + changeState: function(state) { + this.fields.hide(); + if (state == this.state.blank) { + this.button_upload + .add(this.field_image) + .add(this.button_url) + .add(this.input) + .show(); + } else if (state == this.state.attached) { + this.button_remove + .add(this.field_image) + .show(); + } else if (state == this.state.web) { + this.field_url + .show(); + } + }, + + _onFromWeb: function() { + this.changeState(this.state.web); + $('input', this.field_url).focus(); + if (this.options.has_image) { + this.field_clear.val('true'); + } + }, + + _onRemove: function() { + this.changeState(this.state.blank); + $('input', this.field_url).val(''); + if (this.options.has_image) { + this.field_clear.val('true'); + } + }, + + _onInputChange: function() { + this.file_name = this.input.val(); + this.field_clear.val(''); + this.changeState(this.state.attached); + }, + + _onInputMouseOver: function() { + this.button_upload.addClass('hover'); + }, + + _onInputMouseOut: function() { + this.button_upload.removeClass('hover'); + } + + } +}); diff --git a/ckan/public/base/javascript/resource.config b/ckan/public/base/javascript/resource.config index b1a7302d9e8..564da811a6e 100644 --- a/ckan/public/base/javascript/resource.config +++ b/ckan/public/base/javascript/resource.config @@ -38,6 +38,7 @@ ckan = modules/table-toggle-more.js modules/dataset-visibility.js modules/media-grid.js + modules/image-upload.js main = apply_html_class diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 289439c291a..2bfc1a2d492 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -670,3 +670,37 @@ textarea { .box-shadow(none); } } + +.js .image-upload { + #field-image-upload { + cursor: pointer; + position: absolute; + z-index: 1; + .opacity(0); + } + .controls { + position: relative; + } + .btn { + position: relative; + top: 0; + margin-right: 10px; + &.hover { + color: @grayDark; + text-decoration: none; + background-position: 0 -15px; + .transition(background-position .1s linear); + } + } + .btn-remove-url { + position: absolute; + margin-right: 0; + top: 4px; + right: 5px; + padding: 0 4px; + .border-radius(100px); + .icon-remove { + margin-right: 0; + } + } +} diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html index 322f9ebff8a..cd19912d66b 100644 --- a/ckan/templates/group/snippets/group_form.html +++ b/ckan/templates/group/snippets/group_form.html @@ -19,15 +19,7 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my group...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} - - - {% if h.uploads_enabled() %} - {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% if data.image_url and not data.image_url.startswith('http') %} - {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} - {% endif %} - {% endif %} + {{ form.image_upload(data, errors, image_url=c.group_dict.image_display_url, is_upload_enabled=h.uploads_enabled()) }} {% endblock %} diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html index 497dea6bf02..7dc044ccca2 100644 --- a/ckan/templates/macros/form.html +++ b/ckan/templates/macros/form.html @@ -395,3 +395,30 @@

    {% endmacro %} +{# +Builds a file upload for input + +Example + {% import 'macros/form.html' as form %} + {{ form.image_upload(data, errors, is_upload_enabled=true) }} + +#} +{% macro image_upload(data, errors, image_url=false, is_upload_enabled=false, placeholder=false) %} + {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %} + {% set has_uploaded_data = data.image_url and not data.image_url.startswith('http') %} + {% set is_url = data.image_url and data.image_url.startswith('http') %} + + {% if is_upload_enabled %}
    {% endif %} + + {{ input('image_url', label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }} + + {% if is_upload_enabled %} + {{ input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} + {% if has_uploaded_data %} + {{ checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {% endif %} + {% endif %} + + {% if is_upload_enabled %}
    {% endif %} + +{% endmacro %} diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 0a2134e89f7..0b97b958d38 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -19,14 +19,7 @@ {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my organization...'), value=data.description, error=errors.description) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} - - {% if h.uploads_enabled() %} - {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }} - {% if data.image_url and not data.image_url.startswith('http') %} - {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} - {% endif %} - {% endif %} + {{ form.image_upload(data, errors, image_url=c.group_dict.image_display_url, is_upload_enabled=h.uploads_enabled()) }} {% endblock %} From 0f74db8a964451e5cba1360b4f15ea3a84a56544 Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:21:14 +0100 Subject: [PATCH 13/49] [#1262] Adds in doc blocks into the image_upload JS module --- .../base/javascript/modules/image-upload.js | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js index 69ca8f93c05..dbb83ade15a 100644 --- a/ckan/public/base/javascript/modules/image-upload.js +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -1,8 +1,5 @@ /* Image Upload * - * - * Options - * - id: what's the id of the context? */ this.ckan.module('image-upload', function($, _) { return { @@ -42,41 +39,49 @@ this.ckan.module('image-upload', function($, _) { this.field_url = $('input[name="image_url"]', this.el).parents('.control-group'); this.field_image = this.input.parents('.control-group'); + // Is there a clear checkbox on the form already? var checkbox = $('input[name="clear_upload"]', this.el); if (checkbox.length > 0) { options.has_image = true; checkbox.parents('.control-group').remove(); } + // Adds the hidden clear input to the form this.field_clear = $('') .appendTo(this.el); + // Button to set the field to be a URL this.button_url = $(' '+this.i18n('url')+'') .on('click', this._onFromWeb) .insertAfter(this.input); + // Button to attach local file to the form this.button_upload = $(''+this.i18n('upload')+'') .insertAfter(this.input); + // Button to reset the form back to the first from when there is a image uploaded this.button_remove = $('') .text(this.i18n('remove')) .on('click', this._onRemove) .insertAfter(this.button_upload); + // Button for resetting the form when there is a URL set $('') .prop('title', this.i18n('remove_tooltip')) .on('click', this._onRemove) .insertBefore($('input', this.field_url)); + // Update the main label $('label[for="field-image-upload"]').text(this.i18n('label')); + // Setup the file input this.input .on('mouseover', this._onInputMouseOver) .on('mouseout', this._onInputMouseOut) .on('change', this._onInputChange) - .css('width', this.button_upload.outerWidth()) - .hide(); + .css('width', this.button_upload.outerWidth()); + // Fields storage. Used in this.changeState this.fields = $('') .add(this.button_remove) .add(this.button_upload) @@ -85,6 +90,7 @@ this.ckan.module('image-upload', function($, _) { .add(this.field_url) .add(this.field_image); + // Setup the initial state if (options.is_url) { this.changeState(this.state.web); } else if (options.has_image) { @@ -95,6 +101,16 @@ this.ckan.module('image-upload', function($, _) { }, + /* Method to change the display state of the image fields + * + * state - Pseudo constant for passing the state we should be in now + * + * Examples + * + * this.changeState(this.state.web); // Sets the state in URL mode + * + * Returns nothing. + */ changeState: function(state) { this.fields.hide(); if (state == this.state.blank) { @@ -113,6 +129,10 @@ this.ckan.module('image-upload', function($, _) { } }, + /* Event listener for when someone sets the field to URL mode + * + * Returns nothing. + */ _onFromWeb: function() { this.changeState(this.state.web); $('input', this.field_url).focus(); @@ -121,6 +141,10 @@ this.ckan.module('image-upload', function($, _) { } }, + /* Event listener for resetting the field back to the blank state + * + * Returns nothing. + */ _onRemove: function() { this.changeState(this.state.blank); $('input', this.field_url).val(''); @@ -129,16 +153,28 @@ this.ckan.module('image-upload', function($, _) { } }, + /* Event listener for when someone chooses a file to upload + * + * Returns nothing. + */ _onInputChange: function() { this.file_name = this.input.val(); this.field_clear.val(''); this.changeState(this.state.attached); }, + /* Event listener for when a user mouseovers the hidden file input + * + * Returns nothing. + */ _onInputMouseOver: function() { this.button_upload.addClass('hover'); }, + /* Event listener for when a user mouseouts the hidden file input + * + * Returns nothing. + */ _onInputMouseOut: function() { this.button_upload.removeClass('hover'); } From 7d15d591499d5722f60cc0ecfc09abf59dad177f Mon Sep 17 00:00:00 2001 From: John Martin Date: Tue, 15 Oct 2013 15:29:45 +0100 Subject: [PATCH 14/49] [#1262] Adds support for different field names for image upload --- ckan/public/base/javascript/modules/image-upload.js | 13 +++++++------ ckan/templates/macros/form.html | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js index dbb83ade15a..96308ed5764 100644 --- a/ckan/public/base/javascript/modules/image-upload.js +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -7,6 +7,9 @@ this.ckan.module('image-upload', function($, _) { options: { is_url: true, has_image: false, + field_upload: 'input[name="image_upload"]', + field_url: 'input[name="image_url"]', + field_clear: 'input[name="clear_upload"]', i18n: { upload: _('From computer'), url: _('From web'), @@ -35,12 +38,12 @@ this.ckan.module('image-upload', function($, _) { var options = this.options; // firstly setup the fields - this.input = $('input[name="image_upload"]', this.el); - this.field_url = $('input[name="image_url"]', this.el).parents('.control-group'); + this.input = $(options.field_upload, this.el); + this.field_url = $(options.field_url, this.el).parents('.control-group'); this.field_image = this.input.parents('.control-group'); // Is there a clear checkbox on the form already? - var checkbox = $('input[name="clear_upload"]', this.el); + var checkbox = $(options.field_clear, this.el); if (checkbox.length > 0) { options.has_image = true; checkbox.parents('.control-group').remove(); @@ -148,9 +151,7 @@ this.ckan.module('image-upload', function($, _) { _onRemove: function() { this.changeState(this.state.blank); $('input', this.field_url).val(''); - if (this.options.has_image) { - this.field_clear.val('true'); - } + this.field_clear.val('true'); }, /* Event listener for when someone chooses a file to upload diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html index 7dc044ccca2..5a6c00a6cb1 100644 --- a/ckan/templates/macros/form.html +++ b/ckan/templates/macros/form.html @@ -403,19 +403,19 @@ {{ form.image_upload(data, errors, is_upload_enabled=true) }} #} -{% macro image_upload(data, errors, image_url=false, is_upload_enabled=false, placeholder=false) %} +{% macro image_upload(data, errors, field_url='image_url', field_upload='image_upload', field_clear='clear_upload', image_url=false, is_upload_enabled=false, placeholder=false) %} {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %} {% set has_uploaded_data = data.image_url and not data.image_url.startswith('http') %} {% set is_url = data.image_url and data.image_url.startswith('http') %} {% if is_upload_enabled %}
    {% endif %} - {{ input('image_url', label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {{ input(field_url, label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }} {% if is_upload_enabled %} - {{ input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} + {{ input(field_upload, label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} {% if has_uploaded_data %} - {{ checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} + {{ checkbox(field_clear, label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }} {% endif %} {% endif %} From 3a60c2e2420350631742f8d0330be2f002d581eb Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Wed, 16 Oct 2013 21:24:26 -0300 Subject: [PATCH 15/49] [#766] Stop Travis build if a dependency install fails To do so, I needed to split our ./bin/travis-build in two files, one with the deps install, and one with the test run. Doing so allows me to use the "install" directive in Travis, and break the build as soon as something fails with it. --- .travis.yml | 3 ++- ...avis-build => travis-install-dependencies} | 26 +++---------------- bin/travis-run-tests | 22 ++++++++++++++++ 3 files changed, 28 insertions(+), 23 deletions(-) rename bin/{travis-build => travis-install-dependencies} (73%) create mode 100755 bin/travis-run-tests diff --git a/.travis.yml b/.travis.yml index af450298b6c..43b6cc3776a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,8 @@ python: env: - PGVERSION=9.1 - PGVERSION=8.4 -script: ./bin/travis-build +install: ./bin/travis-install-dependencies +script: ./bin/travis-run-tests notifications: irc: channels: diff --git a/bin/travis-build b/bin/travis-install-dependencies similarity index 73% rename from bin/travis-build rename to bin/travis-install-dependencies index 51e633781b4..306d6e1f166 100755 --- a/bin/travis-build +++ b/bin/travis-install-dependencies @@ -1,4 +1,7 @@ -#!/bin/sh +#!/bin/bash + +# Exit immediately if any command fails +set -e # Drop Travis' postgres cluster if we're building using a different pg version TRAVIS_PGVERSION='9.1' @@ -51,24 +54,3 @@ else fi cat test-core.ini - -# Run mocha front-end tests -# We need ckan to be running for some tests -paster serve test-core.ini & -sleep 5 # Make sure the server has fully started -mocha-phantomjs http://localhost:5000/base/test/index.html -# Did an error occur? -MOCHA_ERROR=$? -# We are done so kill ckan -killall paster - -# And finally, run the nosetests -nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext -# Did an error occur? -NOSE_ERROR=$? - -[ "0" -ne "$MOCHA_ERROR" ] && echo MOCKA tests have failed -[ "0" -ne "$NOSE_ERROR" ] && echo NOSE tests have failed - -# If an error occurred in our tests make sure travis knows -exit `expr $MOCHA_ERROR + $NOSE_ERROR` diff --git a/bin/travis-run-tests b/bin/travis-run-tests new file mode 100755 index 00000000000..83f6d333e0d --- /dev/null +++ b/bin/travis-run-tests @@ -0,0 +1,22 @@ +#!/bin/bash + +# Run mocha front-end tests +# We need ckan to be running for some tests +paster serve test-core.ini & +sleep 5 # Make sure the server has fully started +mocha-phantomjs http://localhost:5000/base/test/index.html +# Did an error occur? +MOCHA_ERROR=$? +# We are done so kill ckan +killall paster + +# And finally, run the nosetests +nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext +# Did an error occur? +NOSE_ERROR=$? + +[ "0" -ne "$MOCHA_ERROR" ] && echo MOCKA tests have failed +[ "0" -ne "$NOSE_ERROR" ] && echo NOSE tests have failed + +# If an error occurred in our tests make sure travis knows +exit `expr $MOCHA_ERROR + $NOSE_ERROR` From 1aee242081710ddb1283d3cb9b0987d209273b53 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Wed, 16 Oct 2013 22:04:02 -0300 Subject: [PATCH 16/49] [#766] Remove redundant datastore_test psql user creation --- bin/travis-install-dependencies | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/travis-install-dependencies b/bin/travis-install-dependencies index 306d6e1f166..dcbbf09b316 100755 --- a/bin/travis-install-dependencies +++ b/bin/travis-install-dependencies @@ -46,8 +46,7 @@ paster db init -c test-core.ini # If Postgres >= 9.0, we don't need to use datastore's legacy mode. if [ $PGVERSION != '8.4' ] then - psql -c 'CREATE USER datastore_default;' -U postgres - sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default@\/datastore_test/' test-core.ini + sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default:pass@\/datastore_test/' test-core.ini paster datastore set-permissions postgres -c test-core.ini else sed -i -e 's/.*datastore.read_url.*//' test-core.ini From 6cc143f9deec75bf13fe2e517bead4ae90b91b50 Mon Sep 17 00:00:00 2001 From: kindly Date: Thu, 17 Oct 2013 13:47:19 +0100 Subject: [PATCH 17/49] [#1262] make tests pass --- ckan/config/middleware.py | 3 ++- ckan/lib/uploader.py | 41 +++++++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 7699596138d..efde540054a 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -23,6 +23,7 @@ from ckan.plugins import PluginImplementations from ckan.plugins.interfaces import IMiddleware from ckan.lib.i18n import get_locales_from_config +import ckan.lib.uploader as uploader from ckan.config.environment import load_environment import ckan.lib.app_globals as app_globals @@ -148,7 +149,7 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): cache_max_age=static_max_age) static_parsers = [static_app, app] - storage_directory = config.get('ckan.storage_path') + storage_directory = uploader.get_storage_path() if storage_directory: path = os.path.join(storage_directory, 'storage') try: diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index 880488b8161..867e28cfcb0 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -10,6 +10,7 @@ _storage_path = None + def get_storage_path(): '''Function to cache storage path''' global _storage_path @@ -27,21 +28,24 @@ def get_storage_path(): _storage_path = ofs_storage_dir return _storage_path elif ofs_impl: - log.critical('''We only support local file storage form version 2.2 of ckan - please specify ckan.storage_path in your config for your uploads''') + log.critical('''We only support local file storage form version 2.2 + of ckan please specify ckan.storage_path in your + config for your uploads''') _storage_path = False else: - log.critical('''Please specify a ckan.storage_path in your config for your uploads''') + log.critical('''Please specify a ckan.storage_path in your config + for your uploads''') _storage_path = False return _storage_path - class Upload(object): def __init__(self, object_type, old_filename=None): - ''' Setup upload by creating a subdirectory of the storage directory of name - object_type. old_filename is the name of the file in the url field last time''' + ''' Setup upload by creating a subdirectory of the storage directory + of name object_type. old_filename is the name of the file in the url + field last time''' + self.storage_path = None self.filename = None self.filepath = None @@ -60,20 +64,21 @@ def __init__(self, object_type, old_filename=None): self.old_filepath = os.path.join(self.storage_path, old_filename) def update_data_dict(self, data_dict, url_field, file_field, clear_field): - ''' Manipulate data from the data_dict. url_field is the name of the field - where the upload is going to be. file_field is name of the key where the - FieldStorage is kept (i.e the field where the file data actually is). clear_field - is the name of a boolean field which requests the upload to be deleted. This needs - to be called before it reaches any validators''' - - if not self.storage_path: - return + ''' Manipulate data from the data_dict. url_field is the name of the + field where the upload is going to be. file_field is name of the key + where the FieldStorage is kept (i.e the field where the file data + actually is). clear_field is the name of a boolean field which + requests the upload to be deleted. This needs to be called before + it reaches any validators''' self.url = data_dict.get(url_field, '') self.clear = data_dict.pop(clear_field, None) self.file_field = file_field self.upload_field_storage = data_dict.pop(file_field, None) + if not self.storage_path: + return + if isinstance(self.upload_field_storage, cgi.FieldStorage): self.filename = self.upload_field_storage.filename self.filename = str(datetime.datetime.utcnow()) + self.filename @@ -90,9 +95,11 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field): data_dict[url_field] = '' def upload(self, max_size=2): - ''' Actually upload the file. This should happen just before a commit but after - the data has been validated and flushed to the db. This is so we do not store anything - unless the request is actually good. Max_size is size in MB maximum of the file''' + ''' Actually upload the file. + This should happen just before a commit but after the data has + been validated and flushed to the db. This is so we do not store + anything unless the request is actually good. + max_size is size in MB maximum of the file''' if self.filename: output_file = open(self.tmp_filepath, 'wb') From 8b6e56d9d89ba92fb1a67a0febac8334192ddc83 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Fri, 18 Oct 2013 11:23:04 -0300 Subject: [PATCH 18/49] [#1155] Document how to upgrade the dependencies --- CONTRIBUTING.rst | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 7b15f8d5531..79bd8f77b5b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -236,3 +236,65 @@ Merging a Pull Request If you're reviewing a pull request for CKAN, when merging a branch into master: - Use the ``--no-ff`` option in the ``git merge`` command, + + +-------------------------- +Upgrading the dependencies +-------------------------- + +CKAN's dependencies are pinned to specific versions, so we can guarantee that +no matter when you install it, you'll always get the same dependencies' +versions our code was tested with. + +We couldn't simply leave everything in our requirements.txt file, though. +Sometimes, we discover that one dependency releases a new version that's +incompatible with CKAN, so we need a way to enforce that we won't upgrade it +unexpectedly in the future. We also split between dev and non-dev requirements, +so production servers don't need to install dependencies only used for testing, +for example. + +Our dependencies are split in 3 files: ``requirements.in``, +``requirements.txt``, and ``dev-requirements.txt``. + +requirements.in + Contains our direct dependencies (i.e. not our dependencies' dependencies), + with loosely defined versions. For example, ``apachemiddleware>=0.1.1,<0.2``. + +requirements.txt + Contains every dependency, including indirect, pinned to a specific + version. Created with ``pip freeze``. For example, ``simplejson==3.3.1``. + +dev-requirements.txt + Contains our development dependencies, pinned to a specific version. For + example, ``factory-boy==2.1.1``. + +We haven't created a ``dev-requirement.in`` because we have too few dev +dependencies, we don't update them often, and none of them have a known +incompatible version. + +Steps to upgrade +================ + +#. Create a new virtualenv: ``virtualenv --no-site-packages upgrading`` + +#. Install the requirements with unpinned versions: ``pip install -r + requirements.in`` + +#. Save the new dependencies versions: ``pip freeze > requirements.txt``. We + have to do this before installing the other dependencies so we get only what + was in ``requirements.in`` + +#. Install CKAN: ``python setup.py develop`` + +#. Install the development dependencies: ``pip install -r + dev-requirements.txt`` + +#. Run the tests to make sure everything still works (see :doc:`test`). + + - If not, try to fix the problem. If it's too complicated, pinpoint which + dependency's version broke our tests, find an older version that still + works, and add it to ``requirements.in``. Go back to step 1. + +#. Navigate a bit on CKAN to make sure the tests didn't miss anything. Review + the dependencies changes and their changelogs. If everything seems fine, go + ahead and fill a pull request (see :ref:`making a pull request`). From 4c46078e6737062d9b48c30e82afb5203f02c770 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Tue, 22 Oct 2013 17:25:43 -0300 Subject: [PATCH 19/49] [#766] Move solr configuration to travis-run-tests I was unable to run Jetty-Solr's configuration on travis-install-dependencies. Probably something to do with Bash's "set -e", but I was unable to pinpoint exactly, so I left it into travis-run-tests. No big deal. --- bin/travis-install-dependencies | 6 ------ bin/travis-run-tests | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/travis-install-dependencies b/bin/travis-install-dependencies index dcbbf09b316..50b5ebd22cd 100755 --- a/bin/travis-install-dependencies +++ b/bin/travis-install-dependencies @@ -35,12 +35,6 @@ python setup.py develop # Install npm dpes for mocha npm install -g mocha-phantomjs phantomjs -# Configure Solr -echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty -# FIXME the solr schema cannot be hardcoded as it is dependent on the ckan version -sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml -sudo service jetty restart - paster db init -c test-core.ini # If Postgres >= 9.0, we don't need to use datastore's legacy mode. diff --git a/bin/travis-run-tests b/bin/travis-run-tests index 83f6d333e0d..f9a44de6f9e 100755 --- a/bin/travis-run-tests +++ b/bin/travis-run-tests @@ -1,5 +1,11 @@ #!/bin/bash +# Configure Solr +echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty +# FIXME the solr schema cannot be hardcoded as it is dependent on the ckan version +sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml +sudo service jetty restart + # Run mocha front-end tests # We need ckan to be running for some tests paster serve test-core.ini & From 852a23cda23a81ddc2374d53f67556d2a700ca7e Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Tue, 22 Oct 2013 19:02:53 -0300 Subject: [PATCH 20/49] [#766] Use /bin/sh instead of /bin/bash --- bin/travis-run-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/travis-run-tests b/bin/travis-run-tests index f9a44de6f9e..760be3f458f 100755 --- a/bin/travis-run-tests +++ b/bin/travis-run-tests @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Configure Solr echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty From 12b5d386b150b2072dc6680996a11b8b524199fb Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 25 Oct 2013 11:51:56 +0100 Subject: [PATCH 21/49] [#1294] Update docs theme to latest version --- doc/_themes/sphinx-theme-okfn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/_themes/sphinx-theme-okfn b/doc/_themes/sphinx-theme-okfn index 320555990b6..59688a6679f 160000 --- a/doc/_themes/sphinx-theme-okfn +++ b/doc/_themes/sphinx-theme-okfn @@ -1 +1 @@ -Subproject commit 320555990b639c99ac3bcc79024140c9588e2bdf +Subproject commit 59688a6679f3373a57e8d4e60e43f1b249878eb3 From 5550a4a34e481441c8cdc2ac6f40ebcbfad95663 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 25 Oct 2013 11:58:12 +0100 Subject: [PATCH 22/49] [#1294] Fix logo on docs header bar The logo needs to be in a _static folder because the sphinx-theme-okfn hardcodes it there: https://github.com/okfn/sphinx-theme-okfn/blob/master/layout.html#L18 --- doc/_static/ckanlogo.png | Bin 0 -> 1303 bytes doc/conf.py | 4 ++-- doc/images/ckan_logo_box.png | Bin 5246 -> 0 bytes 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 doc/_static/ckanlogo.png delete mode 100644 doc/images/ckan_logo_box.png diff --git a/doc/_static/ckanlogo.png b/doc/_static/ckanlogo.png new file mode 100644 index 0000000000000000000000000000000000000000..a234cc085bb9e9e0fe45df4d8851b43f2e8ed6c1 GIT binary patch literal 1303 zcmeAS@N?(olHy`uVBq!ia0vp^HXzKw1|+Ti+$;i8k|nMYCBgY=CFO}lsSJ)O`AMk? zp1FzXsX?iUDV2pMQ-L~|l`=yjN+NuHtdjF{^%6m9^eS=-fVzQ(*;f=KCTFLXC?ut( zXXe=|z2CiGNg*@ERw>-n*TA>HIW;5GqpB!1xXLdixhgx^GDXSWPQj+a3TQxXYDuC( zMQ%=Bu~mhw64+cTAR8pCucQE0Qj%?}1aVY^Z-9bxeo?A|sh)vuvVoa_f|;S7p|Od% zxw(#lk%6IszJZaxp^>hkxs|bzm4Sf*6et00D@sYT3UYCS+6CmKL-|y0U7xv`NbLe1q#l=rV74^$(eZ|K3Ki4 zua#$BaYa`peok$)sM+T)Z#P)RW_{oBR!ic5Q?x;M?yO}+1)JI~#z$<}^-U_;<$vPASGN|K zmgtLdy-S`f`bFWJcn+g&*IVXnH~*ye15y_qZy0V+D={eJTEi&Y%FF2O=AP_)Q0SuP z4T%jZB}QgqI*fB$eHqVpo;$Xz>AAz=gyI8pGE8#xW-#}59c2h!ba7)u0+$J=m~c<0 z+To;T!wX9yxC;1Y$#}E+Ic`qSJZP9zT3_U@nrVP?od%mgutbOu=HrurX()~iiYHv)=Dq% zV@*J8QydS$uX6BU&WPbKY2Wmg=X@!D5K~Z$- z+Ch^u29oJ<2*;g71Ts5Wcl1yZcUVLvcyw`gd_H_w)Yk5FOd*jL_rz8T6F|&OET2oZ z0g8T%xVW?o)+3_nKvC(ea`11Jfw_6j88hOSvpTrre@1(K+XmUCHFM^rF-4~NoSXmT z2($`zV_1x*S7&#V^SbnTS>Ro&Hf4rYeF%|a?;(W_bV1zamMJalZuKsexvc6Jc|Vubb=dRRg~{Fux$^3X8P8FNO)^pG*Vh9Ngx#`>is5pp5F6SW;#r(Ki{Q}>FP6<3u<1SiPbFVT2NsX(2l~(kZVlvZtJXnB?2WK6@ zh$P|Ye}3uv_1D1|RhJGhYvhZ9;(iT=f3J{t6rB!Obf-E8zkk(#Pt*cA3n9K1HG(t~ z^+aZWyCmJ3R9He$j2#2K9SwPDV%%H}!(d>L1kY~rxg2qZv`t`SPveb08GKYjw2efv+nb1mO)buQoZ zs&#IjfKd4R%wDofMPG_*M=4C^>h!}NwhLUp1M@+I%8H;fVp%3G46YNb1e716x9_qmvL^1_4Ov?v!o94D`(-Q z#JX+1x$A0<(}adM(x4!re|@#KGvDrnCCL4S%u*?;gBE=241&b%nvua@r?fdn8==NDxnmkd|aoi?GdtFuXzSO>l zJ1{rzDpFi}@t55XV%U`>#z@?D`cpW+uBO-h-P3tAM>{Da37dXr+AG2x!aO*vjLQOY z`k+)O!Au-bVf8%Rfec5d6=|Jvp+Fz|3G)A4C*9T>mjn}{(^RJ<*?Tw_H>?qFDepLG z-IXBGOP4kD;|R~gFptxWbvrOSf%W%N>1}=89LLc9B<&z9(raVj8ngaZ#rd^xsFM`t3+m2}?j+<|Ai}_0C?j`j_mi8~ z@3vqrv$Tu9_iohaY3{vGj?rrh z39EVMHfKLv-~p>)of?uc?|L9?Xq(GA*n4X;x4DT<#=l8!i&3(N-K!}y4k5cdy9qbC zrOyAszZ|{4|Hsk~+9Idgy>cig0N2@{pH}~JX-z%fyi^dHw7M^RO0<6z>K$rI&P z=FJ@a4aD^M2f-|zGi0Sq>%QDmpC|g5wFQ$#sgNVry6$ai0|Oe%sMm?g?ggR0Xp3l{ zS%-!cEnSxl&!w><_j5xOK&{h#qj92vDvgtCP)QA}#Iy9UbF-8;s_qpchC)^MHe%71 zufP1|g|?Dys73h2adHyey8Mkg6=LKun<-?Qn z6s_+q6gk>i>+Ek!*4X?+5gfj}dv~FH0>bd*KUfL{%o>117d=%f$2)EN8888qxlq3N; z`w9ph7nBbLc-Gl^Vh6VrHv7lC{?LJuJm?1x1ck-9!oZ|DJQQ7{Qbc9qp1VQn(q@&% zvBsu95Pd8CtZ=)XvT=SxNzzDL(>>y)O%QpGY?Fq(D5bB~3N_@e1*~Q^N|c^Nm=U39 zy_k!y&dB7`PYkQLrX(XM+I__V9OzW2QwxJK0?}_{W4W&`Y#2wu_%$=q7s0BKN|vtC$rIo`6vm zL376x4XmlHT#--q<{NdY?nB_5IuSh#9tFQGQ$Ax5}(+tLNil>P);h?7m4^?2SI za8mRr7S8w#D>7CkB#whTv@>i1p9&`+4`5$SKX~#Pu)xW*(af7b$28I)K;i~kwlx`P zAs8k>?Mqf?2n9*Ej1!348G1D*{w^zZWBv|p5B9O|WedA*r%J`2@hf8d`D`yKi#IfI=kdOh5ZDj>|k zXT|kDPrkXSq=fRjOzK-6=mhFvH>FOPkuAZBTJg{kVX}S{#j;c}n*KzFl4L%=7WjPp zgNivOfJw8)1?UesmXBFC-~1>AlM=ghZ!l4u4Kw-0)twC^OXV9&!NQn&-sBT>K9c9A z!?*k{!ytO|RwN7Ju4XNC@#(fM3*v!hGgR`EDZL54gPj`GDS3`Qi^ad5b+C0y$ME{C z7qd}>&e(4=5bz}jEGCc~PwKiTIrSB{5;`(vx5+eEun~*1dGA{&&rkP6cdRKv`uhTV z90>I}lTB3YO-zKz*5@N=ZwYjH5Cidynj`UQ7!Z-+_6|#@WB`C9ULTy zH{6x(XN($KSv(LG_uKg85?PgUd7_P+^UeNdE<^=1AtAxo!~~m~s`>o6NaW&={H)cn znVJ58A7NV7)?5I^zVA;FHX8f&w`Sk@`SbTpO&kWy=!7@NU+h#b$6FQ{oX4#mGE!2~IiFD_ho?`mI9%7xe6yE4`9ge?|F#7%tw-x$ z*87)T3OK1xxnXb3*GuJV&N(U`}{z~mD)cFB5w zrah(@w&LRA*n2Kt$0jB~P&+&D=H{lcnc0`QIb8z-hMu0D$?0i($(s&SOC2cU-?R19 zS^9lfm*+qndo4xd&jPP6j|s)aJW5Jg1IdEPZ{KPI=~*Ai$|n%ef;pj#rBj2I;eSh& zPc6z#OG-*+YMtnQO;;8b6p*$)Mw*(kZf|b`)_nGsHBu$42lLcpK3rd&$CSG~}3~4CSVxe_D@kk18F%b47`WiUwWm z_1y^E>tTCT^`tXGrn3D?-sZ*SJ%Bh>!ryhr(D_`Pc!{x6AI}GH6InL8ug=!HM5TKD z-7>lFI@7T_l;-T>@^hj<2dKTlv->Dly^@>$cx#Kn=vO4)XiP*GDqozAPRP5zhH$;nU4J#j%-r|aZ;je&d1 zs)>Svf(J+6J(4pr5bEm0MfzEG6L?gP)6d?7GMo1Bj=kT%UHy0F1~4q5axbnF(|Ch?Zzyx0k7Cte)a;(o*B{L1cNZ8-FFH4Tr)@9s4J zc|YFY4x^-`Tv=Vcu{$jLqhI;OLQ7C~yEyAm)uy@#!(l8lBCf%k1Qg(7UvF{4yxQ)i z-CsHX-#5QExT({VEQFuCxwlST2MG(G$;il@`0ac`Gug(%?Hhkr!q@^zwz(*hg z$v0F5Cuz-W*hQ}-LdWi}(ZtUZtnsq#+~b4&A-+voEozsp)Os z?GV>7pQfg!nL1}CqC2gJj6f>?;Z)^v6B%|jAM&U5#*`i{11|&gWx+n*lc$oI4HlV} zxhK&btVRV`q>qoZo}Qkz@qeD9>Jr756FrG0~{v0S%jk)y6^F zBzIg~oc+bg9wv4B4g)`cm^w36Amgotu~4GMIq$f2wf3{EQj8{Fgmtt3UAI{oz^Zv` zuwPD(ne)+yygXF0AX3Dlq8;`QmRknbriN6z&&!jjE*@@v5wmVep;JDIz+kAJKYuPH zA>jzX4#@RCxD1Xc+`penOioMB#WnoVXMLv1Hfp_fWyQg9`0Xm8Tj5VjH&6g!VPV_R zY$eyZ57;-NW`>4_AZSp~gBUvQwf)th!O&gRSU4KWj*aA}@7QkqF@ho{m%K^yPR=Vb zhC4jHtV~QwO6p`U@x|wZQI%0(Necp2l8eC~uJ>!h&iy;MGL+9Y_(ntiw4F-;^;#dg z)4l@W>H$DvmJRWO*@2N}kAq$w9r=f!baR7t=85POcZta9Zqr<-hwwU3S6w}{C0qbU z24*|rgfow{i16;HHV;?oLMS5u(NBPtkYo3eY^5O}ys_u?_`*V5YHBZzUng+4lbrmW z?2D^V_k|YpH`m)z*TB|*nVq$0yF9XNJ6Wbqy<_uQGoM#hHVxQ7{y951I9Oa+sl}{O zQ&VGJ|D45V?eo%ikM@GnQtF~F1MfBB6#w>*jtT-8WS4fNd2|GH5kg0YPeCDM=8tO9 zWyqRshXeB;;_&O48ew9Xym!N^_S5Dj zi-9dc(M}2@ddzC6Y$dN^9aWuFJX8R6{!n_qu*&L`EY}4VKhgr6vb~^d>7oD~uwH1uH=YEL5n2jcb1mH+?% From 500733eeb791b3960e0a7b116499ad8a27d207cf Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 29 Oct 2013 11:21:29 +0000 Subject: [PATCH 23/49] Default API_URL to SITE_URL if missing Otherwise file uploads and other calls where broken when using CKAN in a non-root location. --- ckan/public/base/javascript/client.js | 2 +- ckan/public/base/javascript/main.js | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ckan/public/base/javascript/client.js b/ckan/public/base/javascript/client.js index 45940c0086d..e1c60ccf24e 100644 --- a/ckan/public/base/javascript/client.js +++ b/ckan/public/base/javascript/client.js @@ -370,7 +370,7 @@ }); ckan.sandbox.setup(function (instance) { - instance.client = new Client({endpoint: ckan.API_ROOT}); + instance.client = new Client({endpoint: ckan.SITE_ROOT}); }); ckan.Client = Client; diff --git a/ckan/public/base/javascript/main.js b/ckan/public/base/javascript/main.js index 42c76dfd15f..843dc47442d 100644 --- a/ckan/public/base/javascript/main.js +++ b/ckan/public/base/javascript/main.js @@ -29,7 +29,9 @@ this.ckan = this.ckan || {}; ckan.SITE_ROOT = getRootFromData('siteRoot'); ckan.LOCALE_ROOT = getRootFromData('localeRoot'); - ckan.API_ROOT = getRootFromData('apiRoot'); + + // Not used, kept for backwards compatibility + ckan.API_ROOT = body.data('apiRoot') || ckan.SITE_ROOT; // Load the localisations before instantiating the modules. ckan.sandbox().client.getLocaleData(locale).done(function (data) { From 787cb7ae25ce47bfa28b8543c1a7e59a06186fe4 Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Fri, 1 Nov 2013 13:35:59 +0530 Subject: [PATCH 24/49] Fix a few small mistakes in the tests --- ckan/tests/functional/api/test_user.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py index cdbe039d012..e313e29710c 100644 --- a/ckan/tests/functional/api/test_user.py +++ b/ckan/tests/functional/api/test_user.py @@ -133,7 +133,7 @@ def teardown_class(cls): model.repo.rebuild_db() - def test_user_create_api_disabled_sysadmin(self): + def test_user_create_api_enabled_sysadmin(self): params = { 'name': 'testinganewusersysadmin', 'email': 'testinganewuser@ckan.org', @@ -145,14 +145,13 @@ def test_user_create_api_disabled_sysadmin(self): res_dict = res.json assert res_dict['success'] is True - def test_user_create_api_disabled_anon(self): + def test_user_create_api_enabled_anon(self): params = { 'name': 'testinganewuseranon', 'email': 'testinganewuser@ckan.org', 'password': 'random', } - res = self.app.post('/api/3/action/user_create', json.dumps(params), - expect_errors=True) + res = self.app.post('/api/3/action/user_create', json.dumps(params)) res_dict = res.json assert res_dict['success'] is True From 4f732627fbf515bc7cbe29e84731606409a637b7 Mon Sep 17 00:00:00 2001 From: kindly Date: Fri, 1 Nov 2013 14:47:41 +0000 Subject: [PATCH 25/49] [#1262] only allow file already exists errors to not raise exception --- ckan/lib/uploader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index 867e28cfcb0..cc4e8751d57 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -57,7 +57,9 @@ def __init__(self, object_type, old_filename=None): try: os.makedirs(self.storage_path) except OSError, e: - pass + ## errno 17 is file already exists + if e.errno != 17: + raise self.object_type = object_type self.old_filename = old_filename if old_filename: From 8c7329c6246c03a965f8284c855c3e538d205a9c Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 4 Nov 2013 07:45:40 +0000 Subject: [PATCH 26/49] [#1262] only allow file already exists errors to not raise exception for other case --- ckan/config/middleware.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index efde540054a..5ba02933d46 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -155,7 +155,9 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): try: os.makedirs(path) except OSError, e: - pass + ## errno 17 is file already exists + if e.errno != 17: + raise storage_app = StaticURLParser(path, cache_max_age=static_max_age) From 8c854ad9a0c5b159c1f99488ed44fc77a134a7f5 Mon Sep 17 00:00:00 2001 From: joetsoi Date: Mon, 4 Nov 2013 10:04:43 +0000 Subject: [PATCH 27/49] [#1304] Clear and keep current database during test runs Add option '--reset-db' to drop and reinitialize database during test runs --- ckan/ckan_nose_plugin.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/ckan/ckan_nose_plugin.py b/ckan/ckan_nose_plugin.py index 2ec4d0988b9..877589a2204 100644 --- a/ckan/ckan_nose_plugin.py +++ b/ckan/ckan_nose_plugin.py @@ -19,21 +19,26 @@ def startContext(self, ctx): if 'new_tests' in repr(ctx): # We don't want to do the stuff below for new-style tests. + if not CkanNose.settings.reset_database: + model.repo.tables_created_and_initialised = True return if isclass(ctx): if hasattr(ctx, "no_db") and ctx.no_db: return - if self.is_first_test or CkanNose.settings.ckan_migration: + if (not CkanNose.settings.reset_database + and not CkanNose.settings.ckan_migration): + model.Session.close_all() + model.repo.tables_created_and_initialised = True + model.repo.rebuild_db() + self.is_first_test = False + elif self.is_first_test or CkanNose.settings.ckan_migration: model.Session.close_all() model.repo.clean_db() self.is_first_test = False if CkanNose.settings.ckan_migration: model.Session.close_all() model.repo.upgrade_db() - # init_db is run at the start of every class because - # when you use an in-memory sqlite db, it appears that - # the db is destroyed after every test when you Session.Remove(). ## This is to make sure the configuration is run again. ## Plugins use configure to make their own tables and they @@ -43,6 +48,9 @@ def startContext(self, ctx): for plugin in PluginImplementations(IConfigurable): plugin.configure(config) + # init_db is run at the start of every class because + # when you use an in-memory sqlite db, it appears that + # the db is destroyed after every test when you Session.Remove(). model.repo.init_db() def options(self, parser, env): @@ -66,6 +74,11 @@ def options(self, parser, env): dest='segments', help='A string containing a hex digits that represent which of' 'the 16 test segments to run. i.e 15af will run segments 1,5,a,f') + parser.add_option( + '--reset-db', + action='store_true', + dest='reset_database', + help='drop database and reinitialize before tests are run') def wantClass(self, cls): name = cls.__name__ From 02057230ab99b10fb7b6d7bc2d4b1771c1c3565b Mon Sep 17 00:00:00 2001 From: joetsoi Date: Mon, 4 Nov 2013 10:05:57 +0000 Subject: [PATCH 28/49] [#1304] change clean_db to rebuild_db in new_test/helpers.py Otherwise new_tests will drop and recreate database --- ckan/new_tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/new_tests/helpers.py b/ckan/new_tests/helpers.py index 51640a7e3dd..4e1e1ffd0c7 100644 --- a/ckan/new_tests/helpers.py +++ b/ckan/new_tests/helpers.py @@ -38,7 +38,7 @@ def reset_db(): # This prevents CKAN from hanging waiting for some unclosed connection. model.Session.close_all() - model.repo.clean_db() + model.repo.rebuild_db() def call_action(action_name, context=None, **kwargs): From eb1a9e74b93ab675ec568b4b21342d5bbbf9e369 Mon Sep 17 00:00:00 2001 From: joetsoi Date: Mon, 4 Nov 2013 10:12:09 +0000 Subject: [PATCH 29/49] [#1304] update docs on 'reset-db' test option --- doc/test.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/test.rst b/doc/test.rst index cdd9b928e0a..f9d2f6bc805 100644 --- a/doc/test.rst +++ b/doc/test.rst @@ -65,6 +65,15 @@ The speed of the PostgreSQL tests can be improved by running PostgreSQL in memory and turning off durability, as described `in the PostgreSQL documentation `_. +By default the tests will keep the database between test runs. If you wish to +drop and reinitialize the database before the run you can use the ``reset-db`` +option:: + + nosetests --ckan --reset-db --with-pylons=test-core.ini ckan + +If you are have the ``ckan-migration`` option on the tests will reset the +reset the database before the test run. + .. _migrationtesting: From ed5ecd2c56d343a110d08d66817e2a5c9a8aa98b Mon Sep 17 00:00:00 2001 From: joetsoi Date: Mon, 4 Nov 2013 10:19:22 +0000 Subject: [PATCH 30/49] [#1304] make sure database is rebuilt during travis test runs --- bin/travis-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/travis-build b/bin/travis-build index 63b4f50e339..5e293b26e1b 100755 --- a/bin/travis-build +++ b/bin/travis-build @@ -63,7 +63,7 @@ MOCHA_ERROR=$? killall paster # And finally, run the nosetests -nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext --with-coverage --cover-package=ckanext --cover-package=ckan +nosetests --ckan --reset-db --with-pylons=test-core.ini --nologcapture ckan ckanext --with-coverage --cover-package=ckanext --cover-package=ckan # Did an error occur? NOSE_ERROR=$? From 8b15c7938295f7919d95857975d45282f3daffbe Mon Sep 17 00:00:00 2001 From: kindly Date: Mon, 4 Nov 2013 14:23:02 +0000 Subject: [PATCH 31/49] [#1262] fix pull request issues --- ckan/controllers/group.py | 4 -- ckan/public/base/css/main.css | 78 +++++++++++++++++---------------- ckan/templates/macros/form.html | 6 +-- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index adbd3d7cacc..907bda73c41 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -7,7 +7,6 @@ from urllib import urlencode from pylons.i18n import get_lang -from pylons import config import ckan.lib.base as base import ckan.lib.helpers as h @@ -462,7 +461,6 @@ def edit(self, id, data=None, errors=None, error_summary=None): group = context.get("group") c.group = group - context.pop('for_edit') c.group_dict = self._action('group_show')(context, data_dict) try: @@ -521,8 +519,6 @@ def _force_reindex(self, grp): def _save_edit(self, id, context): try: - old_group = self._action('group_show')(context, {"id": id}) - old_image_url = old_group.get('image_url') data_dict = clean_dict(dict_fns.unflatten( tuplize_dict(parse_params(request.params)))) context['message'] = data_dict.get('log_message', '') diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index 2ed06e1eb12..0fbaa417e14 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -49,16 +49,12 @@ sub { } img { /* Responsive images (ensure images don't scale beyond their parents) */ - max-width: 100%; /* Part 1: Set a maxium relative to the parent */ - width: auto\9; /* IE7-8 need help adjusting responsive images */ - height: auto; /* Part 2: Scale the height according to the width, otherwise you get stretching */ - vertical-align: middle; border: 0; -ms-interpolation-mode: bicubic; @@ -153,7 +149,7 @@ textarea { img { max-width: 100% !important; } - @page { + @page { margin: 0.5cm; } p, @@ -693,7 +689,6 @@ ol.inline > li { display: inline-block; *display: inline; /* IE7 inline-block hack */ - *zoom: 1; padding-left: 5px; padding-right: 5px; @@ -972,7 +967,6 @@ input[type="color"]:focus, outline: 0; outline: thin dotted \9; /* IE6-9 */ - -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6); -moz-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6); box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6); @@ -982,10 +976,8 @@ input[type="checkbox"] { margin: 4px 0 0; *margin-top: 0; /* IE7 */ - margin-top: 1px \9; /* IE8-9 */ - line-height: normal; } input[type="file"], @@ -1001,10 +993,8 @@ select, input[type="file"] { height: 30px; /* In IE7, the height of the select element cannot be changed by height, only font-size */ - *margin-top: 4px; /* For IE7, add top margin to align select with labels */ - line-height: 30px; } select { @@ -1402,7 +1392,6 @@ select:focus:invalid:focus { display: inline-block; *display: inline; /* IE7 inline-block hack */ - *zoom: 1; vertical-align: middle; padding-left: 5px; @@ -1553,7 +1542,6 @@ input.search-query { padding-left: 14px; padding-left: 4px \9; /* IE7-8 doesn't have border-radius, so don't indent the padding */ - margin-bottom: 0; -webkit-border-radius: 15px; -moz-border-radius: 15px; @@ -1610,7 +1598,6 @@ input.search-query { display: inline-block; *display: inline; /* IE7 inline-block hack */ - *zoom: 1; margin-bottom: 0; vertical-align: middle; @@ -2220,7 +2207,6 @@ button.close { display: inline-block; *display: inline; /* IE7 inline-block hack */ - *zoom: 1; padding: 4px 12px; margin-bottom: 0; @@ -2243,7 +2229,6 @@ button.close { border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); *background-color: #eaeaea; /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); border: 1px solid #cccccc; *border: 0; @@ -2379,7 +2364,6 @@ input[type="button"].btn-block { border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); *background-color: #085871; /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); } .btn-primary:hover, @@ -2411,7 +2395,6 @@ input[type="button"].btn-block { border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); *background-color: #f89406; /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); } .btn-warning:hover, @@ -2443,7 +2426,6 @@ input[type="button"].btn-block { border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); *background-color: #bd362f; /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); } .btn-danger:hover, @@ -2475,7 +2457,6 @@ input[type="button"].btn-block { border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); *background-color: #51a351; /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); } .btn-success:hover, @@ -2507,7 +2488,6 @@ input[type="button"].btn-block { border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); *background-color: #2f96b4; /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); } .btn-info:hover, @@ -2539,7 +2519,6 @@ input[type="button"].btn-block { border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); *background-color: #222222; /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); } .btn-inverse:hover, @@ -2614,7 +2593,6 @@ input[type="submit"].btn.btn-mini { display: inline-block; *display: inline; /* IE7 inline-block hack */ - *zoom: 1; font-size: 0; vertical-align: middle; @@ -2790,7 +2768,6 @@ input[type="submit"].btn.btn-mini { display: inline-block; *display: inline; /* IE7 inline-block hack */ - *zoom: 1; } .btn-group-vertical > .btn { @@ -3487,7 +3464,6 @@ input[type="submit"].btn.btn-mini { border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); *background-color: #e5e5e5; /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075); -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075); @@ -3721,7 +3697,6 @@ input[type="submit"].btn.btn-mini { border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); *background-color: #040404; /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); } .navbar-inverse .btn-navbar:hover, @@ -3751,7 +3726,6 @@ input[type="submit"].btn.btn-mini { display: inline-block; *display: inline; /* IE7 inline-block hack */ - *zoom: 1; text-shadow: 0 1px 0 #ffffff; } @@ -3769,7 +3743,6 @@ input[type="submit"].btn.btn-mini { display: inline-block; *display: inline; /* IE7 inline-block hack */ - *zoom: 1; margin-left: 0; margin-bottom: 0; @@ -3970,7 +3943,6 @@ input[type="submit"].btn.btn-mini { border: 1px solid rgba(0, 0, 0, 0.3); *border: 1px solid #999; /* IE6-7 */ - -webkit-border-radius: 6px; -moz-border-radius: 6px; border-radius: 6px; @@ -5828,7 +5800,6 @@ textarea { border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); *background-color: #085871; /* Darken IE7 buttons by default so they stand out more given they won't have borders */ - filter: progid:DXImageTransform.Microsoft.gradient(enabled = false); } .control-custom.disabled .checkbox.btn:hover, @@ -6107,7 +6078,6 @@ textarea { outline: 0; outline: thin dotted \9; /* IE6-9 */ - -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6); -moz-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6); box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6); @@ -6130,6 +6100,43 @@ textarea { -moz-box-shadow: none; box-shadow: none; } +.js .image-upload #field-image-upload { + cursor: pointer; + position: absolute; + z-index: 1; + opacity: 0; + filter: alpha(opacity=0); +} +.js .image-upload .controls { + position: relative; +} +.js .image-upload .btn { + position: relative; + top: 0; + margin-right: 10px; +} +.js .image-upload .btn.hover { + color: #333333; + text-decoration: none; + background-position: 0 -15px; + -webkit-transition: background-position 0.1s linear; + -moz-transition: background-position 0.1s linear; + -o-transition: background-position 0.1s linear; + transition: background-position 0.1s linear; +} +.js .image-upload .btn-remove-url { + position: absolute; + margin-right: 0; + top: 4px; + right: 5px; + padding: 0 4px; + -webkit-border-radius: 100px; + -moz-border-radius: 100px; + border-radius: 100px; +} +.js .image-upload .btn-remove-url .icon-remove { + margin-right: 0; +} .dataset-item { border-bottom: 1px dotted #dddddd; padding-bottom: 20px; @@ -6516,7 +6523,6 @@ textarea { margin-right: 5px; *display: inline; /* IE7 inline-block hack */ - *zoom: 1; } .actions li:last-of-type { @@ -7925,9 +7931,9 @@ h4 small { -webkit-border-radius: 100px; -moz-border-radius: 100px; border-radius: 100px; - -webkit-box-shadow: inset 0 1px 2 x rgba(0, 0, 0, 0.2); - -moz-box-shadow: inset 0 1px 2 x rgba(0, 0, 0, 0.2); - box-shadow: inset 0 1px 2 x rgba(0, 0, 0, 0.2); + -webkit-box-shadow: inset 0 1px 2x rgba(0, 0, 0, 0.2); + -moz-box-shadow: inset 0 1px 2x rgba(0, 0, 0, 0.2); + box-shadow: inset 0 1px 2x rgba(0, 0, 0, 0.2); } .popover-followee .nav li a:hover i { background-color: #000; @@ -8162,7 +8168,6 @@ iframe { .ie7 .control-custom .checkbox { *display: inline; /* IE7 inline-block hack */ - *zoom: 1; } .ie7 .stages { @@ -8196,7 +8201,6 @@ iframe { .ie7 .masthead nav { *display: inline; /* IE7 inline-block hack */ - *zoom: 1; } .ie7 .masthead .header-image { diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html index 5a6c00a6cb1..db93f046961 100644 --- a/ckan/templates/macros/form.html +++ b/ckan/templates/macros/form.html @@ -405,12 +405,12 @@ #} {% macro image_upload(data, errors, field_url='image_url', field_upload='image_upload', field_clear='clear_upload', image_url=false, is_upload_enabled=false, placeholder=false) %} {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %} - {% set has_uploaded_data = data.image_url and not data.image_url.startswith('http') %} - {% set is_url = data.image_url and data.image_url.startswith('http') %} + {% set has_uploaded_data = data.get(field_url) and not data[field_url].startswith('http') %} + {% set is_url = data.get(field_url) and data[field_url].startswith('http') %} {% if is_upload_enabled %}
    {% endif %} - {{ input(field_url, label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }} + {{ input(field_url, label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.get(field_url), error=errors.get(field_url), classes=['control-full']) }} {% if is_upload_enabled %} {{ input(field_upload, label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }} From 04d65785c97a6b8d44d70c8f6bdff4de50caf3b7 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 5 Nov 2013 16:03:23 +0000 Subject: [PATCH 32/49] [#960] Remove ckan.api_url completely on 2.2 --- CHANGELOG.rst | 2 ++ ckan/lib/app_globals.py | 1 - ckan/public/base/javascript/main.js | 3 --- ckan/templates/base.html | 2 +- doc/configuration.rst | 18 ------------------ doc/legacy-api.rst | 6 ------ 6 files changed, 3 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b97798a4522..17dfb5069c2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ v2.2 API changes and deprecations: + +* The `ckan.api_url` has been completely removed and it can no longer be used * The edit() and after_update() methods of IPackageController plugins are now called when updating a resource using the web frontend or the resource_update API action [#1052] diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index 38c777f7555..38138e87815 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -44,7 +44,6 @@ 'ckan.template_footer_end': {}, 'ckan.dumps_url': {}, 'ckan.dumps_format': {}, - 'ckan.api_url': {}, 'ofs.impl': {'name': 'ofs_impl'}, 'ckan.homepage_style': {'default': '1'}, diff --git a/ckan/public/base/javascript/main.js b/ckan/public/base/javascript/main.js index 843dc47442d..c5f36ef7176 100644 --- a/ckan/public/base/javascript/main.js +++ b/ckan/public/base/javascript/main.js @@ -30,9 +30,6 @@ this.ckan = this.ckan || {}; ckan.SITE_ROOT = getRootFromData('siteRoot'); ckan.LOCALE_ROOT = getRootFromData('localeRoot'); - // Not used, kept for backwards compatibility - ckan.API_ROOT = body.data('apiRoot') || ckan.SITE_ROOT; - // Load the localisations before instantiating the modules. ckan.sandbox().client.getLocaleData(locale).done(function (data) { ckan.i18n.load(data); diff --git a/ckan/templates/base.html b/ckan/templates/base.html index 6012bf65928..b6290af0ad0 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -85,7 +85,7 @@ {# Allows custom attributes to be added to the tag #} - + {# The page block allows you to add content to the page. Most of the time it is diff --git a/doc/configuration.rst b/doc/configuration.rst index 0b623c5e656..c0768e148ab 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -130,24 +130,6 @@ site use this setting. This setting should not have a trailing / on the end. -.. _ckan.api_url: - -ckan.api_url -^^^^^^^^^^^^ - -.. deprecated:: 2 - No longer used. - -Example:: - - ckan.api_url = http://scotdata.ckan.net/api - -Default value: ``/api`` - -The URL that resolves to the CKAN API part of the site. This is useful if the -API is hosted on a different domain, for example when a third-party site uses -the forms API. - .. _apikey_header_name: apikey_header_name diff --git a/doc/legacy-api.rst b/doc/legacy-api.rst index b10841fc976..7ab990dc371 100644 --- a/doc/legacy-api.rst +++ b/doc/legacy-api.rst @@ -434,12 +434,6 @@ front-end javascript. All Util APIs are read-only. The response format is JSON. Javascript calls may want to use the JSONP formatting. -.. Note:: - - Some CKAN deployments have the API deployed at a different domain to the main CKAN website. To make sure that the AJAX calls in the Web UI work, you'll need to configue the ckan.api_url. e.g.:: - - ckan.api_url = http://api.example.com/ - dataset autocomplete ```````````````````` From 4d0184ae6ee1af126c3103a4ac4c407c417fa824 Mon Sep 17 00:00:00 2001 From: kindly Date: Tue, 5 Nov 2013 17:27:27 +0000 Subject: [PATCH 33/49] [#1262] fix required field in wrong place --- ckan/templates/organization/snippets/organization_form.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 0b97b958d38..0d62e36cac1 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -71,8 +71,8 @@ {% endblock %} #} -
    {{ form.required_message() }} +
    {% block delete_button %} {% if h.check_access('organization_delete', {'id': data.id}) %} {% set locale = h.dump_json({'content': _('Are you sure you want to delete this Organization? This will delete all the public and private datasets belonging to this organization.')}) %} From ff7e38e56842d4e20b6846717e61625fbcbe47e2 Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 5 Nov 2013 18:38:22 +0100 Subject: [PATCH 34/49] [#1226] PEP8 --- ckan/tests/functional/api/test_user.py | 38 ++++++++++++++------------ 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py index e313e29710c..875e8a668ab 100644 --- a/ckan/tests/functional/api/test_user.py +++ b/ckan/tests/functional/api/test_user.py @@ -12,6 +12,7 @@ import ckan.config.middleware from ckan.common import json + class TestUserApi(ControllerTestCase): @classmethod def setup_class(cls): @@ -68,8 +69,8 @@ def setup_class(cls): CreateTestData.create() cls._original_config = config.copy() new_authz.clear_auth_functions_cache() - wsgiapp = ckan.config.middleware.make_app(config['global_conf'], - **config) + wsgiapp = ckan.config.middleware.make_app( + config['global_conf'], **config) cls.app = paste.fixture.TestApp(wsgiapp) cls.sysadmin_user = model.User.get('testsysadmin') PylonsTestCase.setup_class() @@ -89,9 +90,11 @@ def test_user_create_api_enabled_sysadmin(self): 'email': 'testinganewuser@ckan.org', 'password': 'random', } - res = self.app.post('/api/3/action/user_create', json.dumps(params), - extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, - expect_errors=True) + res = self.app.post( + '/api/3/action/user_create', + json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, + expect_errors=True) res_dict = res.json assert res_dict['success'] is True @@ -102,7 +105,7 @@ def test_user_create_api_disabled_anon(self): 'password': 'random', } res = self.app.post('/api/3/action/user_create', json.dumps(params), - expect_errors=True) + expect_errors=True) res_dict = res.json assert res_dict['success'] is False @@ -118,8 +121,8 @@ def setup_class(cls): cls._original_config = config.copy() config['ckan.auth.create_user_via_api'] = True new_authz.clear_auth_functions_cache() - wsgiapp = ckan.config.middleware.make_app(config['global_conf'], - **config) + wsgiapp = ckan.config.middleware.make_app( + config['global_conf'], **config) cls.app = paste.fixture.TestApp(wsgiapp) PylonsTestCase.setup_class() cls.sysadmin_user = model.User.get('testsysadmin') @@ -139,9 +142,10 @@ def test_user_create_api_enabled_sysadmin(self): 'email': 'testinganewuser@ckan.org', 'password': 'random', } - res = self.app.post('/api/3/action/user_create', json.dumps(params), - extra_environ={'Authorization': - str(self.sysadmin_user.apikey)}) + res = self.app.post( + '/api/3/action/user_create', + json.dumps(params), + extra_environ={'Authorization': str(self.sysadmin_user.apikey)}) res_dict = res.json assert res_dict['success'] is True @@ -167,8 +171,8 @@ def setup_class(cls): cls._original_config = config.copy() config['ckan.auth.create_user_via_web'] = False new_authz.clear_auth_functions_cache() - wsgiapp = ckan.config.middleware.make_app(config['global_conf'], - **config) + wsgiapp = ckan.config.middleware.make_app( + config['global_conf'], **config) cls.app = paste.fixture.TestApp(wsgiapp) cls.sysadmin_user = model.User.get('testsysadmin') PylonsTestCase.setup_class() @@ -189,7 +193,7 @@ def test_user_create_api_disabled(self): 'password': 'random', } res = self.app.post('/api/3/action/user_create', json.dumps(params), - expect_errors=True) + expect_errors=True) res_dict = res.json assert res_dict['success'] is False @@ -205,8 +209,8 @@ def setup_class(cls): cls._original_config = config.copy() config['ckan.auth.create_user_via_web'] = True new_authz.clear_auth_functions_cache() - wsgiapp = ckan.config.middleware.make_app(config['global_conf'], - **config) + wsgiapp = ckan.config.middleware.make_app( + config['global_conf'], **config) cls.app = paste.fixture.TestApp(wsgiapp) cls.sysadmin_user = model.User.get('testsysadmin') PylonsTestCase.setup_class() @@ -227,7 +231,7 @@ def test_user_create_api_disabled(self): 'password': 'random', } res = self.app.post('/api/3/action/user_create', json.dumps(params), - expect_errors=True) + expect_errors=True) res_dict = res.json assert res_dict['success'] is False From 85737e8e2d1e1b4184849f278c7e24e501e5e510 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 7 Nov 2013 11:41:45 +0000 Subject: [PATCH 35/49] [#1307] Minor docs correction --- ckan/logic/action/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index c7fb98be30a..4ead6b6cb9d 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -90,7 +90,7 @@ def package_create(context, data_dict): :param extras: the dataset's extras (optional), extras are arbitrary (key: value) metadata items that can be added to datasets, each extra dictionary should have keys ``'key'`` (a string), ``'value'`` (a - string), and optionally ``'deleted'`` + string) :type extras: list of dataset extra dictionaries :param relationships_as_object: see ``package_relationship_create()`` for the format of relationship dictionaries (optional) From 7f1327fe010debaee0eed9a5d3c87c119aede52c Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 7 Nov 2013 13:45:55 +0000 Subject: [PATCH 36/49] [#1068] Show 404 instead of login page on user not found Remove unnecessary auth check, fix redirect --- ckan/controllers/user.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 1c6fefff344..28f62b85e01 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -68,7 +68,7 @@ def _setup_template_variables(self, context, data_dict): try: user_dict = get_action('user_show')(context, data_dict) except NotFound: - h.redirect_to(controller='user', action='login', id=None) + abort(404, _('User not found')) except NotAuthorized: abort(401, _('Not authorized to see this page')) c.user_dict = user_dict @@ -117,10 +117,6 @@ def read(self, id=None): 'for_view': True} data_dict = {'id': id, 'user_obj': c.userobj} - try: - check_access('user_show', context, data_dict) - except NotAuthorized: - abort(401, _('Not authorized to see this page')) context['with_related'] = True From c698cdd9565fdfcf7752f1b9d274d0574abab7b1 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 7 Nov 2013 18:02:52 +0000 Subject: [PATCH 37/49] [#1309] Fix placeholders in non-root locations --- ckan/templates/group/snippets/group_item.html | 2 +- ckan/templates/organization/snippets/organization_item.html | 2 +- ckan/templates/related/snippets/related_item.html | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ckan/templates/group/snippets/group_item.html b/ckan/templates/group/snippets/group_item.html index 9e32ce86e6a..ccb072bfc44 100644 --- a/ckan/templates/group/snippets/group_item.html +++ b/ckan/templates/group/snippets/group_item.html @@ -14,7 +14,7 @@ {% set url = h.url_for(group.type ~ '_read', action='read', id=group.name) %}
  • {% block image %} - {{ group.name }} + {{ group.name }} {% endblock %} {% block title %}

    {{ group.display_name }}

    diff --git a/ckan/templates/organization/snippets/organization_item.html b/ckan/templates/organization/snippets/organization_item.html index 818d5cc338a..3020620b5f6 100644 --- a/ckan/templates/organization/snippets/organization_item.html +++ b/ckan/templates/organization/snippets/organization_item.html @@ -14,7 +14,7 @@ {% set url = h.url_for(organization.type ~ '_read', action='read', id=organization.name) %}
  • {% block image %} - {{ organization.name }} + {{ organization.name }} {% endblock %} {% block title %}

    {{ organization.display_name }}

    diff --git a/ckan/templates/related/snippets/related_item.html b/ckan/templates/related/snippets/related_item.html index 39231a7a217..df1c1300025 100644 --- a/ckan/templates/related/snippets/related_item.html +++ b/ckan/templates/related/snippets/related_item.html @@ -11,11 +11,11 @@ #} {% set placeholder_map = { -'application':'/base/images/placeholder-application.png' +'application': h.url_for_static('/base/images/placeholder-application.png') } %} {% set tooltip = _('Go to {related_item_type}').format(related_item_type=related.type|replace('_', ' ')|title) %}
  • - {{ related.title }} + {{ related.title }}

    {{ related.title }}

    {% if related.description %}
    {{ h.render_markdown(related.description) }}
    From 596c9504745acccdd7d7e21c0ba05ad4f3dd13e5 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 8 Nov 2013 10:52:02 +0000 Subject: [PATCH 38/49] Update CHANGELOG --- CHANGELOG.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b97798a4522..95a57a6f501 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,22 @@ API changes and deprecations: called when updating a resource using the web frontend or the resource_update API action [#1052] +v2.1.1 2013-11-8 +================ + +Bug fixes: + * Fix errors on preview on non-root locations (#960) + * Fix place-holder images on non-root locations (#1309) + * Don't accept invalid URLs in resource proxy (#1106) + * Make sure came_from url is local (#1039) + * Fix logout redirect in non-root locations (#1025) + * Wrong auth checks for sysadmins on package_create (#1184) + * Don't return private datasets on package_list (#1295) + * Stop tracking failing when no lang/encoding headers (#1192) + * Fix for paster db clean command getting frozen + * Fix organization not set when editing a dataset (#1199) + * Fix PDF previews (#1194) + * Fix preview failing on private datastore resources (#1221) v2.1 2013-08-13 =============== @@ -96,6 +112,18 @@ Known issues: * Under certain authorization setups the frontend for the groups functionality may not work as expected (See #1176 #1175). +v2.0.3 2013-11-8 +================ + +Bug fixes: + * Fix errors on preview on non-root locations (#960) + * Don't accept invalid URLs in resource proxy (#1106) + * Make sure came_from url is local (#1039) + * Fix logout redirect in non-root locations (#1025) + * Don't return private datasets on package_list (#1295) + * Stop tracking failing when no lang/encoding headers (#1192) + * Fix for paster db clean command getting frozen + v2.0.2 2013-08-13 ================= From d0d18a238ae3938e880699df6db2e49ee952ff30 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 11 Nov 2013 18:38:54 +0100 Subject: [PATCH 39/49] [#1155] Tweak upgrading dependencies docs --- CONTRIBUTING.rst | 64 ++-------------------------------- doc/upgrading-dependencies.rst | 57 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 62 deletions(-) create mode 100644 doc/upgrading-dependencies.rst diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 79bd8f77b5b..b2e324ac204 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -61,6 +61,7 @@ When writing code for CKAN, try to respect our coding standards: css-coding-standards javascript-coding-standards testing-coding-standards + upgrading-dependencies * `CKAN coding standards `_ * `Python coding standards `_ @@ -68,6 +69,7 @@ When writing code for CKAN, try to respect our coding standards: * `CSS coding standards `_ * `JavaScript coding standards `_ * `Testing coding standards `_ +* `Upgrading CKAN's dependencies `_ --------------- @@ -236,65 +238,3 @@ Merging a Pull Request If you're reviewing a pull request for CKAN, when merging a branch into master: - Use the ``--no-ff`` option in the ``git merge`` command, - - --------------------------- -Upgrading the dependencies --------------------------- - -CKAN's dependencies are pinned to specific versions, so we can guarantee that -no matter when you install it, you'll always get the same dependencies' -versions our code was tested with. - -We couldn't simply leave everything in our requirements.txt file, though. -Sometimes, we discover that one dependency releases a new version that's -incompatible with CKAN, so we need a way to enforce that we won't upgrade it -unexpectedly in the future. We also split between dev and non-dev requirements, -so production servers don't need to install dependencies only used for testing, -for example. - -Our dependencies are split in 3 files: ``requirements.in``, -``requirements.txt``, and ``dev-requirements.txt``. - -requirements.in - Contains our direct dependencies (i.e. not our dependencies' dependencies), - with loosely defined versions. For example, ``apachemiddleware>=0.1.1,<0.2``. - -requirements.txt - Contains every dependency, including indirect, pinned to a specific - version. Created with ``pip freeze``. For example, ``simplejson==3.3.1``. - -dev-requirements.txt - Contains our development dependencies, pinned to a specific version. For - example, ``factory-boy==2.1.1``. - -We haven't created a ``dev-requirement.in`` because we have too few dev -dependencies, we don't update them often, and none of them have a known -incompatible version. - -Steps to upgrade -================ - -#. Create a new virtualenv: ``virtualenv --no-site-packages upgrading`` - -#. Install the requirements with unpinned versions: ``pip install -r - requirements.in`` - -#. Save the new dependencies versions: ``pip freeze > requirements.txt``. We - have to do this before installing the other dependencies so we get only what - was in ``requirements.in`` - -#. Install CKAN: ``python setup.py develop`` - -#. Install the development dependencies: ``pip install -r - dev-requirements.txt`` - -#. Run the tests to make sure everything still works (see :doc:`test`). - - - If not, try to fix the problem. If it's too complicated, pinpoint which - dependency's version broke our tests, find an older version that still - works, and add it to ``requirements.in``. Go back to step 1. - -#. Navigate a bit on CKAN to make sure the tests didn't miss anything. Review - the dependencies changes and their changelogs. If everything seems fine, go - ahead and fill a pull request (see :ref:`making a pull request`). diff --git a/doc/upgrading-dependencies.rst b/doc/upgrading-dependencies.rst new file mode 100644 index 00000000000..7fa698cdb68 --- /dev/null +++ b/doc/upgrading-dependencies.rst @@ -0,0 +1,57 @@ +-------------------------- +Upgrading the dependencies +-------------------------- + +The Python modules that CKAN depends on are pinned to specific versions, so we +can guarantee that whenever anyone installs CKAN, they'll always get the same +versions of the Python modules in their virtual environment. + +Our dependencies are defined in three files: + +requirements.in + This file is only used to create a new version of the ``requirements.txt`` + file when upgrading the dependencies. + Contains our direct dependencies only (not dependencies of dependencies) + with loosely defined versions. For example, ``apachemiddleware>=0.1.1,<0.2``. + +requirements.txt + This is the file that people actually use to install CKAN's dependencies into + their virtualenvs. It contains every dependency, including dependencies of + dependencies, each pinned to a specific version. + For example, ``simplejson==3.3.1``. + +dev-requirements.txt + Contains those dependencies only needed by developers, not needed for + production sites. These are pinned to a specific version. For example, + ``factory-boy==2.1.1``. + +We haven't created a ``dev-requirements.in`` file because we have too few dev +dependencies, we don't update them often, and none of them have a known +incompatible version. + +Steps to upgrade +================ + +#. Create a new virtualenv: ``virtualenv --no-site-packages upgrading`` + +#. Install the requirements with unpinned versions: ``pip install -r + requirements.in`` + +#. Save the new dependencies versions: ``pip freeze > requirements.txt``. We + have to do this before installing the other dependencies so we get only what + was in ``requirements.in`` + +#. Install CKAN: ``python setup.py develop`` + +#. Install the development dependencies: ``pip install -r + dev-requirements.txt`` + +#. Run the tests to make sure everything still works (see :doc:`test`). + + - If not, try to fix the problem. If it's too complicated, pinpoint which + dependency's version broke our tests, find an older version that still + works, and add it to ``requirements.in``. Go back to step 1. + +#. Navigate a bit on CKAN to make sure the tests didn't miss anything. Review + the dependencies changes and their changelogs. If everything seems fine, go + ahead and make a pull request (see :ref:`making a pull request`). From 1a188b04f91116beb201eadeb14fe65470e23123 Mon Sep 17 00:00:00 2001 From: Vitor Baptista Date: Mon, 11 Nov 2013 17:19:35 -0300 Subject: [PATCH 40/49] [#1155] Explain how to pin the latest uncompatible version in requirements.in --- doc/upgrading-dependencies.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/upgrading-dependencies.rst b/doc/upgrading-dependencies.rst index 7fa698cdb68..d6c65ca317b 100644 --- a/doc/upgrading-dependencies.rst +++ b/doc/upgrading-dependencies.rst @@ -50,7 +50,9 @@ Steps to upgrade - If not, try to fix the problem. If it's too complicated, pinpoint which dependency's version broke our tests, find an older version that still - works, and add it to ``requirements.in``. Go back to step 1. + works, and add it to ``requirements.in`` (i.e., if ``python-dateutil`` + 2.0.0 broke CKAN, you'd add ``python-dateutil>=1.5.0,<2.0.0``). Go back to + step 1. #. Navigate a bit on CKAN to make sure the tests didn't miss anything. Review the dependencies changes and their changelogs. If everything seems fine, go From 4c05807fa10a946d3f2278e3c85db4290cbbb4bf Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 12 Nov 2013 15:48:28 +0100 Subject: [PATCH 41/49] [#1155] Add a small clarification to the docs --- doc/upgrading-dependencies.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/upgrading-dependencies.rst b/doc/upgrading-dependencies.rst index d6c65ca317b..6d55201d7fb 100644 --- a/doc/upgrading-dependencies.rst +++ b/doc/upgrading-dependencies.rst @@ -32,6 +32,9 @@ incompatible version. Steps to upgrade ================ +These steps will upgrade all of CKAN's dependencies to the latest versions that +work with CKAN: + #. Create a new virtualenv: ``virtualenv --no-site-packages upgrading`` #. Install the requirements with unpinned versions: ``pip install -r From 7eb2b4faef5d346bb39127dbcf977ce9072c7079 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 13 Nov 2013 15:34:00 +0100 Subject: [PATCH 42/49] [#1294] Show table of contents in docs sidebar on front page --- doc/conf.py | 5 +---- doc/index.rst | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 4a61c0f08e6..320548dbaa0 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -157,10 +157,7 @@ } html_sidebars = { - '**': ['relations.html', 'globaltoc.html'], - # There's no point in showing the table of contents in the sidebar on the - # table of contents page! So: - 'index': ['relations.html'], + '**': ['globaltoc.html'], } # The style sheet to use for HTML and HTML Help pages. A file of that name diff --git a/doc/index.rst b/doc/index.rst index d5be74de521..d03a669d749 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -33,6 +33,7 @@ advanced documentation last: ---- .. toctree:: + :hidden: :maxdepth: 2 installing From 829f3e4607e3d550d27c1997c355cd2c9ac1ef31 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 13 Nov 2013 16:24:51 +0100 Subject: [PATCH 43/49] Remove an unnecessary
    from a docs page --- doc/index.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index d03a669d749..e0167318ecb 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -30,8 +30,6 @@ advanced documentation last: covering CKAN's config file options and the differences between CKAN releases. ----- - .. toctree:: :hidden: :maxdepth: 2 From 39ee86d14f9c36a3c8db84d3bee14885298c1dfd Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 13 Nov 2013 17:42:47 +0100 Subject: [PATCH 44/49] Update Sphinx theme --- doc/_themes/sphinx-theme-okfn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/_themes/sphinx-theme-okfn b/doc/_themes/sphinx-theme-okfn index 59688a6679f..4628f26abf4 160000 --- a/doc/_themes/sphinx-theme-okfn +++ b/doc/_themes/sphinx-theme-okfn @@ -1 +1 @@ -Subproject commit 59688a6679f3373a57e8d4e60e43f1b249878eb3 +Subproject commit 4628f26abf401fdb63ec099384bff44a27dcda4c From 35e5729a2444b748830f136ecae0a52e1358e02a Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 13 Nov 2013 17:58:49 +0100 Subject: [PATCH 45/49] Fix documentation title --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 320548dbaa0..22e1049600a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -80,7 +80,7 @@ master_doc = 'index' # General information about the project. -project = u'CKAN Documentation' +project = u'CKAN' project_short_name = u'CKAN' copyright = u'''© 2009-2013, Open Knowledge Foundation. Licensed under Date: Fri, 15 Nov 2013 13:51:07 +0000 Subject: [PATCH 46/49] Install extra package needed by Jetty See http://askubuntu.com/questions/377021/solr-jetty-on-ubuntu-12-04-3 --- bin/travis-install-dependencies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/travis-install-dependencies b/bin/travis-install-dependencies index 50b5ebd22cd..9c699767555 100755 --- a/bin/travis-install-dependencies +++ b/bin/travis-install-dependencies @@ -16,7 +16,7 @@ fi # We need this ppa so we can install postgres-8.4 sudo add-apt-repository -yy ppa:pitti/postgresql sudo apt-get update -qq -sudo apt-get install solr-jetty postgresql-$PGVERSION +sudo apt-get install postgresql-$PGVERSION solr-jetty libcommons-fileupload-java:amd64=1.2.2-1 sudo service postgresql reload From 4ed553eb2b09a758d678156de745277a4c42e0c9 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 15 Nov 2013 15:04:46 +0100 Subject: [PATCH 47/49] Rename a docs page --- doc/authorization.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/authorization.rst b/doc/authorization.rst index 937a833e29e..900d02e52fb 100644 --- a/doc/authorization.rst +++ b/doc/authorization.rst @@ -1,6 +1,6 @@ -============= -Authorization -============= +=============================== +Organizations and authorization +=============================== .. versionchanged:: 2.0 Previous versions of CKAN used a different authorization system. From c433c0c9b7256a31df7437d7746d8d578cc9db3c Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 15 Nov 2013 17:24:21 +0100 Subject: [PATCH 48/49] Fix a Sphinx warning --- doc/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index ec113ea9d4a..fc80fd8ac08 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1097,7 +1097,7 @@ DataPusher Settings .. _ckan.datapusher.formats: ckan.datapusher.formats -^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^ Example:: ckan.datapusher.formats = csv xls xlsx From 0666cf391065337e6f03ae4ed53642696dd02acc Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 15 Nov 2013 16:45:31 +0000 Subject: [PATCH 49/49] Rebuild main.css --- ckan/public/base/css/main.css | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index 5e4887a0293..e3069b76c89 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -6126,6 +6126,7 @@ textarea { } .js .image-upload .btn-remove-url .icon-remove { margin-right: 0; +} .add-member-form .control-label { width: 100%; text-align: left; @@ -7716,6 +7717,9 @@ h4 small { -moz-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.2); box-shadow: 0 1px 2px rgba(0, 0, 0, 0.2); } +.activity .item.no-avatar p { + margin-left: 40px; +} .activity .load-less { margin-bottom: 15px; } @@ -7733,9 +7737,28 @@ h4 small { float: right; text-decoration: none; } +.popover .popover-content { + font-size: 14px; + line-height: 20px; + color: #444444; + word-break: break-all; +} +.popover .popover-content dl { + margin: 0; +} +.popover .popover-content dl dd { + margin-left: 0; + margin-bottom: 10px; +} .activity .item .icon { background-color: #999999; } +.activity .item.failure .icon { + background-color: #b95252; +} +.activity .item.success .icon { + background-color: #69a67a; +} .activity .item.added-tag .icon { background-color: #6995a6; } @@ -7986,6 +8009,21 @@ h4 small { font-size: 16px; margin: 3px 0; } +.datapusher-status-link:hover { + text-decoration: none; +} +.datapusher-status.status-unknown { + color: #bbb; +} +.datapusher-status.status-pending { + color: #FFCC00; +} +.datapusher-status.status-error { + color: red; +} +.datapusher-status.status-complete { + color: #009900; +} body { background: #005d7a url("../../../base/images/bg.png"); }