Merge remote-tracking branch 'upstream/master' into datastore-dump-ui

.gitignore

@@ -42,3 +42,6 @@ ckan_deb/DEBIAN/prerm
 
 # node.js
 node_modules/
+
+# docker
+contrib/docker/.env

Dockerfile

@@ -1,54 +1,57 @@
-# docker build . -t ckan && docker run -d -p 80:5000 --link db:db --link redis:redis --link solr:solr ckan
-
+# See CKAN docs on installation from Docker Compose on usage
 FROM debian:jessie
 MAINTAINER Open Knowledge
 
-ENV CKAN_HOME /usr/lib/ckan/default
-ENV CKAN_CONFIG /etc/ckan/default
-ENV CKAN_STORAGE_PATH /var/lib/ckan
-ENV CKAN_SITE_URL http://localhost:5000
-
-# Install required packages
-RUN apt-get -q -y update && apt-get -q -y upgrade && DEBIAN_FRONTEND=noninteractive apt-get -q -y install \
+# Install required system packages
+RUN apt-get -q -y update && apt-get -q -y upgrade && \
+        DEBIAN_FRONTEND=noninteractive apt-get -q -y install \
 		python-dev \
         python-pip \
         python-virtualenv \
         libpq-dev \
-        git-core \
-        build-essential \
+        libxml2-dev \
+        libxslt-dev \
+        libgeos-dev \
         libssl-dev \
         libffi-dev \
+        postgresql-client \
+        build-essential \
+        git-core \
+        vim \
+        wget \
 	&& apt-get -q clean
 
-# SetUp Virtual Environment CKAN
-RUN mkdir -p $CKAN_HOME $CKAN_CONFIG $CKAN_STORAGE_PATH
-RUN virtualenv $CKAN_HOME
-RUN ln -s $CKAN_HOME/bin/pip /usr/local/bin/ckan-pip
-RUN ln -s $CKAN_HOME/bin/paster /usr/local/bin/ckan-paster
+# Define environment variables
+ENV CKAN_HOME /usr/lib/ckan
+ENV CKAN_VENV $CKAN_HOME/venv
+ENV CKAN_CONFIG /etc/ckan
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
+
+# Build-time variables specified by docker-compose.yml / .env
+ARG CKAN_SITE_URL
+
+# Create ckan user
+RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan
+
+# Setup virtual environment for CKAN
+RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \
+    virtualenv $CKAN_VENV && \
+    ln -s $CKAN_VENV/bin/pip /usr/local/bin/ckan-pip &&\
+    ln -s $CKAN_VENV/bin/paster /usr/local/bin/ckan-paster
+
+# Setup CKAN
+ADD . $CKAN_VENV/src/ckan/
+RUN ckan-pip install --upgrade -r $CKAN_VENV/src/ckan/requirements.txt && \
+    ckan-pip install -e $CKAN_VENV/src/ckan/ && \
+    ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \
+    cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \
+    chmod +x /ckan-entrypoint.sh && \
+    chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH
 
-# SetUp Requirements
-ADD ./requirements.txt $CKAN_HOME/src/ckan/requirements.txt
-RUN ckan-pip install --upgrade -r $CKAN_HOME/src/ckan/requirements.txt
-
-# TMP-BUGFIX https://github.com/ckan/ckan/issues/3388
-ADD ./dev-requirements.txt $CKAN_HOME/src/ckan/dev-requirements.txt
-RUN ckan-pip install --upgrade -r $CKAN_HOME/src/ckan/dev-requirements.txt
-
-# TMP-BUGFIX https://github.com/ckan/ckan/issues/3594
-RUN ckan-pip install --upgrade urllib3
-
-# SetUp CKAN
-ADD . $CKAN_HOME/src/ckan/
-RUN ckan-pip install -e $CKAN_HOME/src/ckan/
-RUN ln -s $CKAN_HOME/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini
-
-# SetUp EntryPoint
-COPY ./contrib/docker/ckan-entrypoint.sh /
-RUN chmod +x /ckan-entrypoint.sh
 ENTRYPOINT ["/ckan-entrypoint.sh"]
 
-# Volumes
-VOLUME ["/etc/ckan/default"]
-VOLUME ["/var/lib/ckan"]
+USER ckan
 EXPOSE 5000
-CMD ["ckan-paster","serve","/etc/ckan/default/ckan.ini"]
+
+CMD ["ckan-paster","serve","/etc/ckan/ckan.ini"]
+

ckan/config/middleware/flask_app.py

@@ -162,6 +162,21 @@ def hello_world_post():
         if hasattr(plugin, 'get_blueprint'):
             app.register_extension_blueprint(plugin.get_blueprint())
 
+    # Set flask routes in named_routes
+    for rule in app.url_map.iter_rules():
+        if '.' not in rule.endpoint:
+            continue
+        controller, action = rule.endpoint.split('.')
+        route = {
+            rule.endpoint: {
+                'action': action,
+                'controller': controller,
+                'highlight_actions': action,
+                'needed': list(rule.arguments)
+                }
+            }
+        config['routes.named_routes'].update(route)
+
     # Start other middleware
     for plugin in PluginImplementations(IMiddleware):
         app = plugin.make_middleware(app, config)

ckan/lib/cli.py

@@ -1076,7 +1076,7 @@ def run_(self):
         from ckan.lib.celery_app import celery
         celery_args = []
         if len(self.args) == 2 and self.args[1] == 'concurrency':
-            celery_args.append['--concurrency=1']
+            celery_args.append('--concurrency=1')
         celery.worker_main(argv=['celeryd', '--loglevel=INFO'] + celery_args)
 
     def view(self):

ckan/lib/helpers.py

@@ -685,12 +685,25 @@ def are_there_flash_messages():
 
 def _link_active(kwargs):
     ''' creates classes for the link_to calls '''
+    if is_flask_request():
+        return _link_active_flask(kwargs)
+    else:
+        return _link_active_pylons(kwargs)
+
+
+def _link_active_pylons(kwargs):
     highlight_actions = kwargs.get('highlight_actions',
                                    kwargs.get('action', '')).split()
     return (c.controller == kwargs.get('controller')
             and c.action in highlight_actions)
 
 
+def _link_active_flask(kwargs):
+    blueprint, endpoint = request.url_rule.endpoint.split('.')
+    return(kwargs.get('controller') == blueprint and
+           kwargs.get('action') == endpoint)
+
+
 def _link_to(text, *args, **kwargs):
     '''Common link making code for several helper functions'''
     assert len(args) < 2, 'Too many unnamed arguments'
@@ -731,6 +744,30 @@ def nav_link(text, *args, **kwargs):
     :param condition: if ``False`` then no link is returned
 
     '''
+    if is_flask_request():
+        return nav_link_flask(text, *args, **kwargs)
+    else:
+        return nav_link_pylons(text, *args, **kwargs)
+
+
+def nav_link_flask(text, *args, **kwargs):
+    if len(args) > 1:
+        raise Exception('Too many unnamed parameters supplied')
+    blueprint, endpoint = request.url_rule.endpoint.split('.')
+    if args:
+        kwargs['controller'] = blueprint or None
+    named_route = kwargs.pop('named_route', '')
+    if kwargs.pop('condition', True):
+        if named_route:
+            link = _link_to(text, named_route, **kwargs)
+        else:
+            link = _link_to(text, **kwargs)
+    else:
+        link = ''
+    return link
+
+
+def nav_link_pylons(text, *args, **kwargs):
     if len(args) > 1:
         raise Exception('Too many unnamed parameters supplied')
     if args:
@@ -923,8 +960,9 @@ def get_facet_items_dict(facet, limit=None, exclude_active=False):
             facets.append(dict(active=True, **facet_item))
     # Sort descendingly by count and ascendingly by case-sensitive display name
     facets.sort(key=lambda it: (-it['count'], it['display_name'].lower()))
-    if c.search_facets_limits and limit is None:
-        limit = c.search_facets_limits.get(facet)
+    if hasattr(c, 'search_facets_limits'):
+        if c.search_facets_limits and limit is None:
+            limit = c.search_facets_limits.get(facet)
     # zero treated as infinite for hysterical raisins
     if limit is not None and limit > 0:
         return facets[:limit]

contrib/docker/.env.template

@@ -0,0 +1,43 @@
+# Variables in this file will be substituted into docker-compose.yml
+# Save a copy of this file as .env and insert your own values.
+# Verify correct substitution with "docker-compose config"
+# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
+# docker-compose down
+# docker rmi -f docker_ckan docker_db
+# docker rmi $(docker images -f dangling=true -q)
+# docker-compose build
+# docker-compose up -d
+# docker-compose restart ckan # give the db service time to initialize the db cluster on first run
+
+# Image: ckan
+CKAN_SITE_ID=default
+#
+# On AWS, your CKAN_SITE_URL is the output of:
+# curl -s http://169.254.169.254/latest/meta-data/public-hostname
+# CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com
+# When running locally, CKAN_SITE_URL must contain the port
+CKAN_SITE_URL=http://localhost:5000
+#
+# CKAN_PORT must be available on the host: sudo netstat -na
+# To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan
+CKAN_PORT=5000
+#
+# Email settings
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+#
+# Image: db
+POSTGRES_PASSWORD=ckan
+#
+# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 5432
+# To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build
+POSTGRES_PORT=5432
+#
+# The datastore database will be created in the db container as docs
+# Readwrite user/pass will be ckan:POSTGRES_PASSWORD
+# Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD
+DATASTORE_READONLY_PASSWORD=datastore
+

contrib/docker/apache.wsgi

@@ -1,5 +1,5 @@
 import os
-ckan_home = os.environ.get('CKAN_HOME', '/usr/lib/ckan/default')
+ckan_home = os.environ.get('CKAN_VENV', '/usr/lib/ckan/default')
 activate_this = os.path.join(ckan_home, 'bin/activate_this.py')
 execfile(activate_this, dict(__file__=activate_this))
 

contrib/docker/ckan-entrypoint.sh

@@ -8,6 +8,8 @@ set -e
 : ${CKAN_SOLR_URL:=}
 # URL for redis (required unless linked to a container called 'redis')
 : ${CKAN_REDIS_URL:=}
+# URL for datapusher (required unless linked to a container called 'datapusher')
+: ${CKAN_DATAPUSHER_URL:=}
 
 CONFIG="${CKAN_CONFIG}/ckan.ini"
 
@@ -17,89 +19,49 @@ abort () {
 }
 
 set_environment () {
+  export CKAN_SITE_ID=${CKAN_SITE_ID}
+  export CKAN_SITE_URL=${CKAN_SITE_URL}
   export CKAN_SQLALCHEMY_URL=${CKAN_SQLALCHEMY_URL}
   export CKAN_SOLR_URL=${CKAN_SOLR_URL}
   export CKAN_REDIS_URL=${CKAN_REDIS_URL}
-  export CKAN_STORAGE_PATH=${CKAN_STORAGE_PATH}
-  export CKAN_SITE_URL=${CKAN_SITE_URL}
+  export CKAN_STORAGE_PATH=/var/lib/ckan
+  export CKAN_DATAPUSHER_URL=${CKAN_DATAPUSHER_URL}
+  export CKAN_DATASTORE_WRITE_URL=${CKAN_DATASTORE_WRITE_URL}
+  export CKAN_DATASTORE_READ_URL=${CKAN_DATASTORE_READ_URL}
+  export CKAN_SMTP_SERVER=${CKAN_SMTP_SERVER}
+  export CKAN_SMTP_STARTTLS=${CKAN_SMTP_STARTTLS}
+  export CKAN_SMTP_USER=${CKAN_SMTP_USER}
+  export CKAN_SMTP_PASSWORD=${CKAN_SMTP_PASSWORD}
+  export CKAN_SMTP_MAIL_FROM=${CKAN_SMTP_MAIL_FROM}
 }
 
 write_config () {
-  # Note that this only gets called if there is no config, see below!
   ckan-paster make-config --no-interactive ckan "$CONFIG"
-
-  # The variables above will be used by CKAN, but
-  # in case want to use the config from ckan.ini use this
-  #ckan-paster --plugin=ckan config-tool "$CONFIG" -e \
-  #    "sqlalchemy.url = ${CKAN_SQLALCHEMY_URL}" \
-  #    "solr_url = ${CKAN_SOLR_URL}" \
-  #    "ckan.redis.url = ${CKAN_REDIS_URL}" \
-  #    "ckan.storage_path = ${CKAN_STORAGE_PATH}" \
-  #    "ckan.site_url = ${CKAN_SITE_URL}"
-}
-
-link_postgres_url () {
-  local user=$DB_ENV_POSTGRES_USER
-  local pass=$DB_ENV_POSTGRES_PASSWORD
-  local db=$DB_ENV_POSTGRES_DB
-  local host=$DB_PORT_5432_TCP_ADDR
-  local port=$DB_PORT_5432_TCP_PORT
-  echo "postgresql://${user}:${pass}@${host}:${port}/${db}"
-}
-
-link_solr_url () {
-  local host=$SOLR_PORT_8983_TCP_ADDR
-  local port=$SOLR_PORT_8983_TCP_PORT
-  echo "http://${host}:${port}/solr/ckan"
-}
-
-link_redis_url () {
-  local host=$REDIS_PORT_6379_TCP_ADDR
-  local port=$REDIS_PORT_6379_TCP_PORT
-  echo "redis://${host}:${port}/1"
 }
 
 # If we don't already have a config file, bootstrap
 if [ ! -e "$CONFIG" ]; then
   write_config
 fi
 
-# Set environment variables
+# Get or create CKAN_SQLALCHEMY_URL
 if [ -z "$CKAN_SQLALCHEMY_URL" ]; then
-  if ! CKAN_SQLALCHEMY_URL=$(link_postgres_url); then
-    abort "ERROR: no CKAN_SQLALCHEMY_URL specified and linked container called 'db' was not found"
-  else
-    #If that worked, use the DB details to wait for the DB
-    export PGHOST=${DB_PORT_5432_TCP_ADDR}
-    export PGPORT=${DB_PORT_5432_TCP_PORT}
-    export PGDATABASE=${DB_ENV_POSTGRES_DB}
-    export PGUSER=${DB_ENV_POSTGRES_USER}
-    export PGPASSWORD=${DB_ENV_POSTGRES_PASSWORD}
-
-    # wait for postgres db to be available, immediately after creation
-    # its entrypoint creates the cluster and dbs and this can take a moment
-    for tries in $(seq 30); do
-      psql -c 'SELECT 1;' 2> /dev/null && break
-      sleep 0.3
-    done
-  fi
+  abort "ERROR: no CKAN_SQLALCHEMY_URL specified in docker-compose.yml"
 fi
 
 if [ -z "$CKAN_SOLR_URL" ]; then
-  if ! CKAN_SOLR_URL=$(link_solr_url); then
-    abort "ERROR: no CKAN_SOLR_URL specified and linked container called 'solr' was not found"
-  fi
+    abort "ERROR: no CKAN_SOLR_URL specified in docker-compose.yml"
 fi
-    
+
 if [ -z "$CKAN_REDIS_URL" ]; then
-  if ! CKAN_REDIS_URL=$(link_redis_url); then
-    abort "ERROR: no CKAN_REDIS_URL specified and linked container called 'redis' was not found"
-  fi
+    abort "ERROR: no CKAN_REDIS_URL specified in docker-compose.yml"
 fi
 
-set_environment
+if [ -z "$CKAN_DATAPUSHER_URL" ]; then
+    abort "ERROR: no CKAN_DATAPUSHER_URL specified in docker-compose.yml"
+fi
 
-# Initializes the Database
+set_environment
 ckan-paster --plugin=ckan db init -c "${CKAN_CONFIG}/ckan.ini"
-
 exec "$@"
+