Merge branch 'master' into 3229-api-blueprint

.circleci-matrix.yml

@@ -1,6 +1,6 @@
 env:
-- SEGMENTS="0123"
-- SEGMENTS="4567"
+- SEGMENTS="01234"
+- SEGMENTS="567"
 - SEGMENTS="89ab"
 - SEGMENTS="cdef"
 

.travis.yml

@@ -0,0 +1,35 @@
+sudo: required
+
+language: python
+
+services:
+  - docker
+
+cache:
+  directories:
+     - ~/docker
+
+before_install:
+  - if [[ -e ~/docker/postgresql.tar ]]; then docker load -i ~/docker/postgresql.tar; fi
+  - if [[ -e ~/docker/solr.tar ]]; then docker load -i ~/docker/solr.tar; fi
+  - if [[ -e ~/docker/redis.tar ]]; then docker load -i ~/docker/redis.tar; fi
+  - if [[ -e ~/docker/ckan.tar ]]; then docker load -i ~/docker/ckan.tar; fi
+
+  - docker build --rm=false -t postgresql ./contrib/docker/postgresql/
+  - docker build --rm=false -t solr ./contrib/docker/solr/
+  - docker pull redis:latest
+  - docker build --rm=false -t ckan .
+
+  - mkdir -p ~/docker; docker save postgresql > ~/docker/postgresql.tar
+  - mkdir -p ~/docker; docker save solr > ~/docker/solr.tar
+  - mkdir -p ~/docker; docker save redis:latest > ~/docker/redis.tar
+  - mkdir -p ~/docker; docker save ckan > ~/docker/ckan.tar
+
+install:
+  - docker run -d --name db postgresql
+  - docker run -d --name solr solr
+  - docker run -d --name redis redis:latest
+  - docker run -d --name ckan -p 5000:5000 --link db:db --link redis:redis --link solr:solr ckan
+
+script:
+  - docker ps -a

CHANGELOG.rst

@@ -15,6 +15,9 @@ API changes and deprecations:
    helper) now return all organizations a user belongs to regardless of
    capacity (Admin, Editor or Member), not just the ones where she is an
    administrator (#2457)
+ * ``organization_list_for_user`` (and the ``h.organizations_available()``
+   helper) now default to not include package_count. Pass
+   include_dataset_count=True if you need the package_count values.
  * ``resource['size']`` will change from string to long integer (#3205)
 
 v2.6.0 2016-11-02

Dockerfile

@@ -1,63 +1,49 @@
-FROM phusion/baseimage:0.9.15
-MAINTAINER Open Knowledge
+# docker build . -t ckan && docker run -d -p 80:5000 --link db:db --link redis:redis --link solr:solr ckan
 
-# Disable SSH
-RUN rm -rf /etc/service/sshd /etc/my_init.d/00_regen_ssh_host_keys.sh
+FROM debian:jessie
+MAINTAINER Open Knowledge
 
-ENV HOME /root
 ENV CKAN_HOME /usr/lib/ckan/default
 ENV CKAN_CONFIG /etc/ckan/default
-ENV CKAN_DATA /var/lib/ckan
+ENV CKAN_STORAGE_PATH /var/lib/ckan
+ENV CKAN_SITE_URL http://localhost:5000
 
 # Install required packages
-RUN apt-get -q -y update && \
-    DEBIAN_FRONTEND=noninteractive apt-get -q -y install \
-        python-minimal \
-        python-dev \
+RUN apt-get -q -y update && apt-get -q -y upgrade && DEBIAN_FRONTEND=noninteractive apt-get -q -y install \
+		python-dev \
+        python-pip \
         python-virtualenv \
-        libevent-dev \
         libpq-dev \
-        nginx-light \
-        apache2 \
-        libapache2-mod-wsgi \
-        postfix \
-        build-essential && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install CKAN
+        git-core \
+	&& apt-get -q clean
+
+# SetUp Virtual Environment CKAN
+RUN mkdir -p $CKAN_HOME $CKAN_CONFIG $CKAN_STORAGE_PATH
 RUN virtualenv $CKAN_HOME
-RUN mkdir -p $CKAN_HOME $CKAN_CONFIG $CKAN_DATA
-RUN chown www-data:www-data $CKAN_DATA
+RUN ln -s $CKAN_HOME/bin/pip /usr/local/bin/ckan-pip
+RUN ln -s $CKAN_HOME/bin/paster /usr/local/bin/ckan-paster
 
+# SetUp Requirements
 ADD ./requirements.txt $CKAN_HOME/src/ckan/requirements.txt
-RUN $CKAN_HOME/bin/pip install -r $CKAN_HOME/src/ckan/requirements.txt
-ADD . $CKAN_HOME/src/ckan/
-RUN $CKAN_HOME/bin/pip install -e $CKAN_HOME/src/ckan/
-RUN ln -s $CKAN_HOME/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini
-ADD ./contrib/docker/apache.wsgi $CKAN_CONFIG/apache.wsgi
-
-# Configure apache
-ADD ./contrib/docker/apache.conf /etc/apache2/sites-available/ckan_default.conf
-RUN echo "Listen 8080" > /etc/apache2/ports.conf
-RUN a2ensite ckan_default
-RUN a2dissite 000-default
+RUN ckan-pip install --upgrade -r $CKAN_HOME/src/ckan/requirements.txt
 
-# Configure nginx
-ADD ./contrib/docker/nginx.conf /etc/nginx/nginx.conf
-RUN mkdir /var/cache/nginx
+# TMP-BUGFIX https://github.com/ckan/ckan/issues/3388
+ADD ./dev-requirements.txt $CKAN_HOME/src/ckan/dev-requirements.txt
+RUN ckan-pip install --upgrade -r $CKAN_HOME/src/ckan/dev-requirements.txt
 
-# Configure postfix
-ADD ./contrib/docker/main.cf /etc/postfix/main.cf
+# SetUp CKAN
+ADD . $CKAN_HOME/src/ckan/
+RUN ckan-pip install -e $CKAN_HOME/src/ckan/
+RUN ln -s $CKAN_HOME/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini
 
-# Configure runit
-ADD ./contrib/docker/my_init.d /etc/my_init.d
-ADD ./contrib/docker/svc /etc/service
-CMD ["/sbin/my_init"]
+# SetUp EntryPoint
+COPY ./contrib/docker/ckan-entrypoint.sh /
+RUN chmod +x /ckan-entrypoint.sh
+ENTRYPOINT ["/ckan-entrypoint.sh"]
 
 # Volumes
 VOLUME ["/etc/ckan/default"]
 VOLUME ["/var/lib/ckan"]
-EXPOSE 80
+EXPOSE 5000
 
-RUN rm -rf /tmp/* /var/tmp/*
+CMD ["ckan-paster","serve","/etc/ckan/default/ckan.ini"]

ckan/ckan_nose_plugin.py

@@ -83,17 +83,14 @@ def options(self, parser, env):
             help='drop database and reinitialize before tests are run')
 
     def wantClass(self, cls):
-        name = cls.__name__
-
-        wanted = (not cls.__name__.startswith('_')
-                  and (issubclass(cls, unittest.TestCase)
-                       or re.search('(?:^|[\b_\./-])[Tt]est', name)
-                      ))
-
-        if self.segments and str(hashlib.md5(name).hexdigest())[0] not in self.segments:
+        if self.segments and str(hashlib.md5(
+                cls.__name__).hexdigest())[0] not in self.segments:
             return False
 
-        return wanted
+    def wantFunction(self, fn):
+        if self.segments and hashlib.md5(
+                fn.__name__).hexdigest()[0] not in self.segments:
+            return False
 
     def finalize(self, report):
         if self.segments:

ckan/controllers/admin.py

@@ -50,7 +50,8 @@ def _get_config_form_items(self):
             {'name': 'ckan.site_title', 'control': 'input', 'label': _('Site Title'), 'placeholder': ''},
             {'name': 'ckan.main_css', 'control': 'select', 'options': styles, 'label': _('Style'), 'placeholder': ''},
             {'name': 'ckan.site_description', 'control': 'input', 'label': _('Site Tag Line'), 'placeholder': ''},
-            {'name': 'ckan.site_logo', 'control': 'input', 'label': _('Site Tag Logo'), 'placeholder': ''},
+            {'name': 'ckan.site_logo', 'control': 'image_upload', 'label': _('Site Tag Logo'), 'placeholder': '', 'upload_enabled':h.uploads_enabled(),
+                'field_url': 'ckan.site_logo', 'field_upload': 'logo_upload', 'field_clear': 'clear_logo_upload'},
             {'name': 'ckan.site_about', 'control': 'markdown', 'label': _('About'), 'placeholder': _('About page text')},
             {'name': 'ckan.site_intro_text', 'control': 'markdown', 'label': _('Intro Text'), 'placeholder': _('Text on home page')},
             {'name': 'ckan.site_custom_css', 'control': 'textarea', 'label': _('Custom CSS'), 'placeholder': _('Customisable css inserted into the page header')},

ckan/i18n/fr/LC_MESSAGES/ckan.po

@@ -924,7 +924,7 @@ msgstr[1] "Il y a plus de {years} ans"
 
 #: ckan/lib/formatters.py:148
 msgid "{month} {day}, {year}, {hour:02}:{min:02} ({timezone})"
-msgstr "{month} {day}, {year}, {heure:02}:{min:02} ({timezone})"
+msgstr "{month} {day}, {year}, {hour:02}:{min:02} ({timezone})"
 
 #: ckan/lib/formatters.py:153
 msgid "{month} {day}, {year}"

ckan/lib/dictization/model_dictize.py

@@ -350,7 +350,7 @@ def group_dictize(group, context,
     like tags are included unless you specify it in the params.
 
     :param packages_field: determines the format of the `packages` field - can
-    be `datasets` or None.
+    be `datasets`, `dataset_count` or None.
     '''
     assert packages_field in ('datasets', 'dataset_count', None)
     if packages_field == 'dataset_count':

ckan/lib/email_notifications.py

@@ -192,7 +192,7 @@ def get_and_send_notifications_for_user(user):
             'ckan.email_notifications_since', '2 days')
     email_notifications_since = string_to_timedelta(
             email_notifications_since)
-    email_notifications_since = (datetime.datetime.now()
+    email_notifications_since = (datetime.datetime.utcnow()
             - email_notifications_since)
 
     # FIXME: We are accessing model from lib here but I'm not sure what
@@ -214,7 +214,7 @@ def get_and_send_notifications_for_user(user):
     # else to do unless we add a update_email_last_sent()
     # logic function which would only be needed by this lib.
     dash = model.Dashboard.get(user['id'])
-    dash.email_last_sent = datetime.datetime.now()
+    dash.email_last_sent = datetime.datetime.utcnow()
     model.repo.commit()
 
 

ckan/lib/helpers.py

@@ -1781,12 +1781,15 @@ def groups_available(am_member=False):
 
 
 @core_helper
-def organizations_available(permission='manage_group'):
+def organizations_available(
+        permission='manage_group', include_dataset_count=False):
     '''Return a list of organizations that the current user has the specified
     permission for.
     '''
     context = {'user': c.user}
-    data_dict = {'permission': permission}
+    data_dict = {
+        'permission': permission,
+        'include_dataset_count': include_dataset_count}
     return logic.get_action('organization_list_for_user')(context, data_dict)
 
 

ckan/lib/jobs.py

@@ -207,6 +207,14 @@ def test_job(*args):
 class Worker(rq.Worker):
     u'''
     CKAN-specific worker.
+
+    Note that starting an instance of this class (via the ``work``
+    method) disposes the currently active database engine and the
+    associated session. This is necessary to prevent their corruption by
+    the forked worker process. Both the engine and the session
+    automatically re-initialize afterwards once they are used. However,
+    non-committed changes are rolled back and instance variables bound
+    to the old session have to be re-fetched from the database.
     '''
     def __init__(self, queues=None, *args, **kwargs):
         u'''
@@ -234,12 +242,27 @@ def register_birth(self, *args, **kwargs):
         return result
 
     def execute_job(self, job, *args, **kwargs):
+        # We shut down all database connections and the engine to make sure
+        # that they are not shared with the child process and closed there
+        # while still being in use in the main process, see
+        #
+        #   https://github.com/ckan/ckan/issues/3365
+        #
+        # Note that this rolls back any non-committed changes in the session.
+        # Both `Session` and `engine` automatically re-initialize themselve
+        # when they are used the next time.
+        log.debug(u'Disposing database engine before fork')
+        meta.Session.remove()
+        meta.engine.dispose()
+
+        # The original implementation performs the actual fork
         queue = remove_queue_name_prefix(job.origin)
-        log.info(u'Worker {} has started job {} from queue "{}"'.format(
+        log.info(u'Worker {} starts job {} from queue "{}"'.format(
                  self.key, job.id, queue))
         result = super(Worker, self).execute_job(job, *args, **kwargs)
         log.info(u'Worker {} has finished job {} from queue "{}"'.format(
                  self.key, job.id, queue))
+
         return result
 
     def register_death(self, *args, **kwargs):

ckan/logic/action/delete.py

@@ -45,14 +45,14 @@ def user_delete(context, data_dict):
     user_id = _get_or_bust(data_dict, 'id')
     user = model.User.get(user_id)
 
+    if user is None:
+        raise NotFound('User "{id}" was not found.'.format(id=user_id))
+
     # New revision, needed by the member table
     rev = model.repo.new_revision()
     rev.author = context['user']
     rev.message = _(u' Delete User: {0}').format(user.name)
 
-    if user is None:
-        raise NotFound('User "{id}" was not found.'.format(id=user_id))
-
     user.delete()
 
     user_memberships = model.Session.query(model.Member).filter(

ckan/logic/action/get.py

@@ -454,6 +454,9 @@ def group_list(context, data_dict):
         packages in the `package_count` property.
         (optional, default: ``False``)
     :type all_fields: boolean
+    :param include_dataset_count: if all_fields, include the full package_count
+        (optional, default: ``True``)
+    :type include_dataset_count: boolean
     :param include_extras: if all_fields, include the group extra fields
         (optional, default: ``False``)
     :type include_extras: boolean
@@ -467,9 +470,7 @@ def group_list(context, data_dict):
         (optional, default: ``False``).
     :type include_users: boolean
 
-
     :rtype: list of strings
-
     '''
     _check_access('group_list', context, data_dict)
     return _group_or_org_list(context, data_dict)
@@ -503,6 +504,9 @@ def organization_list(context, data_dict):
         packages in the `package_count` property.
         (optional, default: ``False``)
     :type all_fields: boolean
+    :param include_dataset_count: if all_fields, include the full package_count
+        (optional, default: ``True``)
+    :type include_dataset_count: boolean
     :param include_extras: if all_fields, include the organization extra fields
         (optional, default: ``False``)
     :type include_extras: boolean
@@ -624,6 +628,9 @@ def organization_list_for_user(context, data_dict):
         returned organizations, for example ``"read"`` or ``"create_dataset"``
         (optional, default: ``"manage_group"``)
     :type permission: string
+    :param include_dataset_count: include the package_count in each org
+        (optional, default: ``False``)
+    :type include_dataset_count: boolean
 
     :returns: list of organizations that the user has the given permission for
     :rtype: list of dicts
@@ -692,7 +699,8 @@ def organization_list_for_user(context, data_dict):
             (org, group_ids_to_capacities[org.id]) for org in orgs_q.all()]
 
     context['with_capacity'] = True
-    orgs_list = model_dictize.group_list_dictize(orgs_and_capacities, context)
+    orgs_list = model_dictize.group_list_dictize(orgs_and_capacities, context,
+        with_package_counts=asbool(data_dict.get('include_dataset_count')))
     return orgs_list
 
 
@@ -1210,8 +1218,12 @@ def _group_or_org_show(context, data_dict, is_org=False):
     group = model.Group.get(id)
     context['group'] = group
 
-    include_datasets = asbool(data_dict.get('include_datasets', False))
-    packages_field = 'datasets' if include_datasets else 'dataset_count'
+    if asbool(data_dict.get('include_datasets', False)):
+        packages_field = 'datasets'
+    elif asbool(data_dict.get('include_dataset_count', True)):
+        packages_field = 'dataset_count'
+    else:
+        packages_field = None
 
     include_tags = asbool(data_dict.get('include_tags', True))
     include_users = asbool(data_dict.get('include_users', True))
@@ -1275,9 +1287,12 @@ def group_show(context, data_dict):
 
     :param id: the id or name of the group
     :type id: string
-    :param include_datasets: include a list of the group's datasets
+    :param include_datasets: include a truncated list of the group's datasets
          (optional, default: ``False``)
-    :type id: boolean
+    :type include_datasets: boolean
+    :param include_dataset_count: include the full package_count
+         (optional, default: ``True``)
+    :type include_dataset_count: boolean
     :param include_extras: include the group's extra fields
          (optional, default: ``True``)
     :type id: boolean
@@ -1307,9 +1322,12 @@ def organization_show(context, data_dict):
 
     :param id: the id or name of the organization
     :type id: string
-    :param include_datasets: include a list of the organization's datasets
+    :param include_datasets: include a truncated list of the org's datasets
          (optional, default: ``False``)
-    :type id: boolean
+    :type include_datasets: boolean
+    :param include_dataset_count: include the full package_count
+         (optional, default: ``True``)
+    :type include_dataset_count: boolean
     :param include_extras: include the organization's extra fields
          (optional, default: ``True``)
     :type id: boolean