Merge branch 'master' into 3229-api-blueprint

.gitignore

@@ -25,6 +25,8 @@ fl_notes.txt
 *.ini
 .noseids
 *~
+.idea
+.vscode
 
 # custom style
 ckan/public/base/less/custom.less

Dockerfile

@@ -31,6 +31,9 @@ RUN ckan-pip install --upgrade -r $CKAN_HOME/src/ckan/requirements.txt
 ADD ./dev-requirements.txt $CKAN_HOME/src/ckan/dev-requirements.txt
 RUN ckan-pip install --upgrade -r $CKAN_HOME/src/ckan/dev-requirements.txt
 
+# TMP-BUGFIX https://github.com/ckan/ckan/issues/3594
+RUN ckan-pip install --upgrade urllib3
+
 # SetUp CKAN
 ADD . $CKAN_HOME/src/ckan/
 RUN ckan-pip install -e $CKAN_HOME/src/ckan/
@@ -45,5 +48,4 @@ ENTRYPOINT ["/ckan-entrypoint.sh"]
 VOLUME ["/etc/ckan/default"]
 VOLUME ["/var/lib/ckan"]
 EXPOSE 5000
-
 CMD ["ckan-paster","serve","/etc/ckan/default/ckan.ini"]

ckan/__init__.py

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-__version__ = '2.7.0a'
+__version__ = '2.8.0a'
 
 __description__ = 'CKAN Software'
 __long_description__ = \

ckan/common.py

@@ -21,6 +21,7 @@
                          ungettext as pylons_ungettext)
 
 from pylons import response
+
 import simplejson as json
 
 try:

ckan/config/environment.py

@@ -69,9 +69,21 @@ def find_controller(self, controller):
 
     # Pylons paths
     root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+    valid_base_public_folder_names = ['public', 'public-bs2']
+    static_files = app_conf.get('ckan.base_public_folder', 'public')
+    app_conf['ckan.base_public_folder'] = static_files
+
+    if static_files not in valid_base_public_folder_names:
+        raise CkanConfigurationException(
+            'You provided an invalid value for ckan.base_public_folder. '
+            'Possible values are: "public" and "public-bs2".'
+        )
+
+    log.info('Loading static files from %s' % static_files)
     paths = dict(root=root,
                  controllers=os.path.join(root, 'controllers'),
-                 static_files=os.path.join(root, 'public'),
+                 static_files=os.path.join(root, static_files),
                  templates=[])
 
     # Initialize main CKAN config object
@@ -95,13 +107,13 @@ def find_controller(self, controller):
     for msg in msgs:
         warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
 
+    # load all CKAN plugins
+    p.load_all()
+
     # Check Redis availability
     if not is_redis_available():
         log.critical('Could not connect to Redis.')
 
-    # load all CKAN plugins
-    p.load_all()
-
     app_globals.reset()
 
     # issue #3260: remove idle transaction
@@ -220,7 +232,19 @@ def update_config():
     helpers.load_plugin_helpers()
     config['pylons.h'] = helpers.helper_functions
 
-    jinja2_templates_path = os.path.join(root, 'templates')
+    # Templates and CSS loading from configuration
+    valid_base_templates_folder_names = ['templates', 'templates-bs2']
+    templates = config.get('ckan.base_templates_folder', 'templates')
+    config['ckan.base_templates_folder'] = templates
+
+    if templates not in valid_base_templates_folder_names:
+        raise CkanConfigurationException(
+            'You provided an invalid value for ckan.base_templates_folder. '
+            'Possible values are: "templates" and "templates-bs2".'
+        )
+
+    jinja2_templates_path = os.path.join(root, templates)
+    log.info('Loading templates from %s' % jinja2_templates_path)
     template_paths = [jinja2_templates_path]
 
     extra_template_paths = config.get('extra_template_paths', '')

ckan/config/middleware/__init__.py

@@ -1,18 +1,19 @@
 # encoding: utf-8
 
 """WSGI app initialization"""
+import urllib
 import urlparse
 import urllib
 
 import webob
-
 from routes import request_config as routes_request_config
 
 from ckan.lib.i18n import get_locales_from_config
 from ckan.config.environment import load_environment
 from ckan.config.middleware.flask_app import make_flask_stack
 from ckan.config.middleware.pylons_app import make_pylons_stack
 from ckan.common import config
+from ckan.lib.i18n import get_locales_from_config
 
 import logging
 log = logging.getLogger(__name__)

ckan/config/middleware/flask_app.py

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 import os
+import re
 import inspect
 import itertools
 import pkgutil
@@ -173,6 +174,10 @@ def hello_world_post():
             'bottom': True,
             'bundle': True,
         }
+    root_path = config.get('ckan.root_path', None)
+    if root_path:
+        root_path = re.sub('/{{LANG}}', '', root_path)
+        fanstatic_config['base_url'] = root_path
     app = Fanstatic(app, **fanstatic_config)
 
     for plugin in PluginImplementations(IMiddleware):

ckan/config/middleware/pylons_app.py

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 import os
+import re
 
 from pylons.wsgiapp import PylonsApp
 
@@ -89,6 +90,10 @@ def make_pylons_stack(conf, full_stack=True, static_files=True,
             'bottom': True,
             'bundle': True,
         }
+    root_path = config.get('ckan.root_path', None)
+    if root_path:
+        root_path = re.sub('/{{LANG}}', '', root_path)
+        fanstatic_config['base_url'] = root_path
     app = Fanstatic(app, **fanstatic_config)
 
     for plugin in PluginImplementations(IMiddleware):

ckan/controllers/feed.py

@@ -411,11 +411,12 @@ def output_feed(self, results, feed_title, feed_description,
                 author_email=pkg.get('author_email', ''),
                 categories=[t['name'] for t in pkg.get('tags', [])],
                 enclosure=webhelpers.feedgenerator.Enclosure(
-                    self.base_url + h.url_for(controller='api',
-                                              register='package',
-                                              action='show',
-                                              id=pkg['name'],
-                                              ver='2'),
+                    h.url_for(controller='api',
+                              register='package',
+                              action='show',
+                              id=pkg['name'],
+                              ver='3',
+                              qualified=True),
                     unicode(len(json.dumps(pkg))),   # TODO fix this
                     u'application/json'),
                 **additional_fields

ckan/controllers/group.py

@@ -170,14 +170,24 @@ def index(self):
             context['user_id'] = c.userobj.id
             context['user_is_admin'] = c.userobj.sysadmin
 
-        data_dict_global_results = {
-            'all_fields': False,
-            'q': q,
-            'sort': sort_by,
-            'type': group_type or 'group',
-        }
-        global_results = self._action('group_list')(context,
-                                                    data_dict_global_results)
+        try:
+            data_dict_global_results = {
+                'all_fields': False,
+                'q': q,
+                'sort': sort_by,
+                'type': group_type or 'group',
+            }
+            global_results = self._action('group_list')(
+                context, data_dict_global_results)
+        except ValidationError as e:
+            if e.error_dict and e.error_dict.get('message'):
+                msg = e.error_dict['message']
+            else:
+                msg = str(e)
+            h.flash_error(msg)
+            c.page = h.Page([], 0)
+            return render(self._index_template(group_type),
+                          extra_vars={'group_type': group_type})
 
         data_dict_page_results = {
             'all_fields': True,
@@ -186,6 +196,7 @@ def index(self):
             'type': group_type or 'group',
             'limit': items_per_page,
             'offset': items_per_page * (page - 1),
+            'include_extras': True
         }
         page_results = self._action('group_list')(context,
                                                   data_dict_page_results)
@@ -659,8 +670,11 @@ def member_new(self, id):
 
         context = {'model': model, 'session': model.Session,
                    'user': c.user}
+        try:
+            self._check_access('group_member_create', context, {'id': id})
+        except NotAuthorized:
+            abort(403, _('Unauthorized to create group %s members') % '')
 
-        # self._check_access('group_delete', context, {'id': id})
         try:
             data_dict = {'id': id}
             data_dict['include_datasets'] = False

ckan/controllers/package.py

@@ -546,17 +546,23 @@ def new(self, data=None, errors=None, error_summary=None):
     def resource_edit(self, id, resource_id, data=None, errors=None,
                       error_summary=None):
 
+        context = {'model': model, 'session': model.Session,
+                   'api_version': 3, 'for_edit': True,
+                   'user': c.user, 'auth_user_obj': c.userobj}
+        data_dict = {'id': id}
+
+        try:
+            check_access('package_update', context, data_dict)
+        except NotAuthorized:
+            abort(403, _('User %r not authorized to edit %s') % (c.user, id))
+
         if request.method == 'POST' and not data:
             data = data or \
                 clean_dict(dict_fns.unflatten(tuplize_dict(parse_params(
                                                            request.POST))))
             # we don't want to include save as it is part of the form
             del data['save']
 
-            context = {'model': model, 'session': model.Session,
-                       'api_version': 3, 'for_edit': True,
-                       'user': c.user, 'auth_user_obj': c.userobj}
-
             data['package_id'] = id
             try:
                 if resource_id:
@@ -574,9 +580,6 @@ def resource_edit(self, id, resource_id, data=None, errors=None,
             h.redirect_to(controller='package', action='resource_read', id=id,
                           resource_id=resource_id)
 
-        context = {'model': model, 'session': model.Session,
-                   'api_version': 3, 'for_edit': True,
-                   'user': c.user, 'auth_user_obj': c.userobj}
         pkg_dict = get_action('package_show')(context, {'id': id})
         if pkg_dict['state'].startswith('draft'):
             # dataset has not yet been fully created

ckan/controllers/user.py

@@ -531,8 +531,11 @@ def perform_reset(self, id):
         if request.method == 'POST':
             try:
                 context['reset_password'] = True
+                user_state = user_dict['state']
                 new_password = self._get_form_password()
-                user_dict['password'] = new_password
+                username = request.params.get('name')
+                if (username is not None and username != ''):
+                    user_dict['name'] = username
                 user_dict['reset_key'] = c.reset_key
                 user_dict['state'] = model.State.ACTIVE
                 user = get_action('user_update')(context, user_dict)
@@ -550,6 +553,7 @@ def perform_reset(self, id):
                 h.flash_error(u'%r' % e.error_dict)
             except ValueError, ve:
                 h.flash_error(unicode(ve))
+            user_dict['state'] = user_state
 
         c.user_dict = user_dict
         return render('user/perform_reset.html')

ckan/lib/activity_streams_session_extension.py

@@ -102,7 +102,8 @@ def before_commit(self, session):
                     # Object has no id; skipping
                     continue
 
-                if activity_type == "new" and obj.id in activities:
+                if (activity_type in ('new', 'changed') and
+                        obj.id in activities):
                     # This object was already logged as a new package
                     continue