Merge pull request #3692 from parksandwildlife/3649-docker-upgrade

3649 docker upgrade

.gitignore

@@ -42,3 +42,6 @@ ckan_deb/DEBIAN/prerm
 
 # node.js
 node_modules/
+
+# docker
+contrib/docker/.env

Dockerfile

@@ -1,54 +1,57 @@
-# docker build . -t ckan && docker run -d -p 80:5000 --link db:db --link redis:redis --link solr:solr ckan
-
+# See CKAN docs on installation from Docker Compose on usage
 FROM debian:jessie
 MAINTAINER Open Knowledge
 
-ENV CKAN_HOME /usr/lib/ckan/default
-ENV CKAN_CONFIG /etc/ckan/default
-ENV CKAN_STORAGE_PATH /var/lib/ckan
-ENV CKAN_SITE_URL http://localhost:5000
-
-# Install required packages
-RUN apt-get -q -y update && apt-get -q -y upgrade && DEBIAN_FRONTEND=noninteractive apt-get -q -y install \
+# Install required system packages
+RUN apt-get -q -y update && apt-get -q -y upgrade && \
+        DEBIAN_FRONTEND=noninteractive apt-get -q -y install \
 		python-dev \
         python-pip \
         python-virtualenv \
         libpq-dev \
-        git-core \
-        build-essential \
+        libxml2-dev \
+        libxslt-dev \
+        libgeos-dev \
         libssl-dev \
         libffi-dev \
+        postgresql-client \
+        build-essential \
+        git-core \
+        vim \
+        wget \
 	&& apt-get -q clean
 
-# SetUp Virtual Environment CKAN
-RUN mkdir -p $CKAN_HOME $CKAN_CONFIG $CKAN_STORAGE_PATH
-RUN virtualenv $CKAN_HOME
-RUN ln -s $CKAN_HOME/bin/pip /usr/local/bin/ckan-pip
-RUN ln -s $CKAN_HOME/bin/paster /usr/local/bin/ckan-paster
+# Define environment variables
+ENV CKAN_HOME /usr/lib/ckan
+ENV CKAN_VENV $CKAN_HOME/venv
+ENV CKAN_CONFIG /etc/ckan
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
+
+# Build-time variables specified by docker-compose.yml / .env
+ARG CKAN_SITE_URL
+
+# Create ckan user
+RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan
+
+# Setup virtual environment for CKAN
+RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \
+    virtualenv $CKAN_VENV && \
+    ln -s $CKAN_VENV/bin/pip /usr/local/bin/ckan-pip &&\
+    ln -s $CKAN_VENV/bin/paster /usr/local/bin/ckan-paster
+
+# Setup CKAN
+ADD . $CKAN_VENV/src/ckan/
+RUN ckan-pip install --upgrade -r $CKAN_VENV/src/ckan/requirements.txt && \
+    ckan-pip install -e $CKAN_VENV/src/ckan/ && \
+    ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \
+    cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \
+    chmod +x /ckan-entrypoint.sh && \
+    chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH
 
-# SetUp Requirements
-ADD ./requirements.txt $CKAN_HOME/src/ckan/requirements.txt
-RUN ckan-pip install --upgrade -r $CKAN_HOME/src/ckan/requirements.txt
-
-# TMP-BUGFIX https://github.com/ckan/ckan/issues/3388
-ADD ./dev-requirements.txt $CKAN_HOME/src/ckan/dev-requirements.txt
-RUN ckan-pip install --upgrade -r $CKAN_HOME/src/ckan/dev-requirements.txt
-
-# TMP-BUGFIX https://github.com/ckan/ckan/issues/3594
-RUN ckan-pip install --upgrade urllib3
-
-# SetUp CKAN
-ADD . $CKAN_HOME/src/ckan/
-RUN ckan-pip install -e $CKAN_HOME/src/ckan/
-RUN ln -s $CKAN_HOME/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini
-
-# SetUp EntryPoint
-COPY ./contrib/docker/ckan-entrypoint.sh /
-RUN chmod +x /ckan-entrypoint.sh
 ENTRYPOINT ["/ckan-entrypoint.sh"]
 
-# Volumes
-VOLUME ["/etc/ckan/default"]
-VOLUME ["/var/lib/ckan"]
+USER ckan
 EXPOSE 5000
-CMD ["ckan-paster","serve","/etc/ckan/default/ckan.ini"]
+
+CMD ["ckan-paster","serve","/etc/ckan/ckan.ini"]
+

contrib/docker/.env.template

@@ -0,0 +1,43 @@
+# Variables in this file will be substituted into docker-compose.yml
+# Save a copy of this file as .env and insert your own values.
+# Verify correct substitution with "docker-compose config"
+# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
+# docker-compose down
+# docker rmi -f docker_ckan docker_db
+# docker rmi $(docker images -f dangling=true -q)
+# docker-compose build
+# docker-compose up -d
+# docker-compose restart ckan # give the db service time to initialize the db cluster on first run
+
+# Image: ckan
+CKAN_SITE_ID=default
+#
+# On AWS, your CKAN_SITE_URL is the output of:
+# curl -s http://169.254.169.254/latest/meta-data/public-hostname
+# CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com
+# When running locally, CKAN_SITE_URL must contain the port
+CKAN_SITE_URL=http://localhost:5000
+#
+# CKAN_PORT must be available on the host: sudo netstat -na
+# To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan
+CKAN_PORT=5000
+#
+# Email settings
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+#
+# Image: db
+POSTGRES_PASSWORD=ckan
+#
+# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 5432
+# To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build
+POSTGRES_PORT=5432
+#
+# The datastore database will be created in the db container as docs
+# Readwrite user/pass will be ckan:POSTGRES_PASSWORD
+# Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD
+DATASTORE_READONLY_PASSWORD=datastore
+

contrib/docker/apache.wsgi

@@ -1,5 +1,5 @@
 import os
-ckan_home = os.environ.get('CKAN_HOME', '/usr/lib/ckan/default')
+ckan_home = os.environ.get('CKAN_VENV', '/usr/lib/ckan/default')
 activate_this = os.path.join(ckan_home, 'bin/activate_this.py')
 execfile(activate_this, dict(__file__=activate_this))
 

contrib/docker/ckan-entrypoint.sh

@@ -8,6 +8,8 @@ set -e
 : ${CKAN_SOLR_URL:=}
 # URL for redis (required unless linked to a container called 'redis')
 : ${CKAN_REDIS_URL:=}
+# URL for datapusher (required unless linked to a container called 'datapusher')
+: ${CKAN_DATAPUSHER_URL:=}
 
 CONFIG="${CKAN_CONFIG}/ckan.ini"
 
@@ -17,89 +19,49 @@ abort () {
 }
 
 set_environment () {
+  export CKAN_SITE_ID=${CKAN_SITE_ID}
+  export CKAN_SITE_URL=${CKAN_SITE_URL}
   export CKAN_SQLALCHEMY_URL=${CKAN_SQLALCHEMY_URL}
   export CKAN_SOLR_URL=${CKAN_SOLR_URL}
   export CKAN_REDIS_URL=${CKAN_REDIS_URL}
-  export CKAN_STORAGE_PATH=${CKAN_STORAGE_PATH}
-  export CKAN_SITE_URL=${CKAN_SITE_URL}
+  export CKAN_STORAGE_PATH=/var/lib/ckan
+  export CKAN_DATAPUSHER_URL=${CKAN_DATAPUSHER_URL}
+  export CKAN_DATASTORE_WRITE_URL=${CKAN_DATASTORE_WRITE_URL}
+  export CKAN_DATASTORE_READ_URL=${CKAN_DATASTORE_READ_URL}
+  export CKAN_SMTP_SERVER=${CKAN_SMTP_SERVER}
+  export CKAN_SMTP_STARTTLS=${CKAN_SMTP_STARTTLS}
+  export CKAN_SMTP_USER=${CKAN_SMTP_USER}
+  export CKAN_SMTP_PASSWORD=${CKAN_SMTP_PASSWORD}
+  export CKAN_SMTP_MAIL_FROM=${CKAN_SMTP_MAIL_FROM}
 }
 
 write_config () {
-  # Note that this only gets called if there is no config, see below!
   ckan-paster make-config --no-interactive ckan "$CONFIG"
-
-  # The variables above will be used by CKAN, but
-  # in case want to use the config from ckan.ini use this
-  #ckan-paster --plugin=ckan config-tool "$CONFIG" -e \
-  #    "sqlalchemy.url = ${CKAN_SQLALCHEMY_URL}" \
-  #    "solr_url = ${CKAN_SOLR_URL}" \
-  #    "ckan.redis.url = ${CKAN_REDIS_URL}" \
-  #    "ckan.storage_path = ${CKAN_STORAGE_PATH}" \
-  #    "ckan.site_url = ${CKAN_SITE_URL}"
-}
-
-link_postgres_url () {
-  local user=$DB_ENV_POSTGRES_USER
-  local pass=$DB_ENV_POSTGRES_PASSWORD
-  local db=$DB_ENV_POSTGRES_DB
-  local host=$DB_PORT_5432_TCP_ADDR
-  local port=$DB_PORT_5432_TCP_PORT
-  echo "postgresql://${user}:${pass}@${host}:${port}/${db}"
-}
-
-link_solr_url () {
-  local host=$SOLR_PORT_8983_TCP_ADDR
-  local port=$SOLR_PORT_8983_TCP_PORT
-  echo "http://${host}:${port}/solr/ckan"
-}
-
-link_redis_url () {
-  local host=$REDIS_PORT_6379_TCP_ADDR
-  local port=$REDIS_PORT_6379_TCP_PORT
-  echo "redis://${host}:${port}/1"
 }
 
 # If we don't already have a config file, bootstrap
 if [ ! -e "$CONFIG" ]; then
   write_config
 fi
 
-# Set environment variables
+# Get or create CKAN_SQLALCHEMY_URL
 if [ -z "$CKAN_SQLALCHEMY_URL" ]; then
-  if ! CKAN_SQLALCHEMY_URL=$(link_postgres_url); then
-    abort "ERROR: no CKAN_SQLALCHEMY_URL specified and linked container called 'db' was not found"
-  else
-    #If that worked, use the DB details to wait for the DB
-    export PGHOST=${DB_PORT_5432_TCP_ADDR}
-    export PGPORT=${DB_PORT_5432_TCP_PORT}
-    export PGDATABASE=${DB_ENV_POSTGRES_DB}
-    export PGUSER=${DB_ENV_POSTGRES_USER}
-    export PGPASSWORD=${DB_ENV_POSTGRES_PASSWORD}
-
-    # wait for postgres db to be available, immediately after creation
-    # its entrypoint creates the cluster and dbs and this can take a moment
-    for tries in $(seq 30); do
-      psql -c 'SELECT 1;' 2> /dev/null && break
-      sleep 0.3
-    done
-  fi
+  abort "ERROR: no CKAN_SQLALCHEMY_URL specified in docker-compose.yml"
 fi
 
 if [ -z "$CKAN_SOLR_URL" ]; then
-  if ! CKAN_SOLR_URL=$(link_solr_url); then
-    abort "ERROR: no CKAN_SOLR_URL specified and linked container called 'solr' was not found"
-  fi
+    abort "ERROR: no CKAN_SOLR_URL specified in docker-compose.yml"
 fi
-    
+
 if [ -z "$CKAN_REDIS_URL" ]; then
-  if ! CKAN_REDIS_URL=$(link_redis_url); then
-    abort "ERROR: no CKAN_REDIS_URL specified and linked container called 'redis' was not found"
-  fi
+    abort "ERROR: no CKAN_REDIS_URL specified in docker-compose.yml"
 fi
 
-set_environment
+if [ -z "$CKAN_DATAPUSHER_URL" ]; then
+    abort "ERROR: no CKAN_DATAPUSHER_URL specified in docker-compose.yml"
+fi
 
-# Initializes the Database
+set_environment
 ckan-paster --plugin=ckan db init -c "${CKAN_CONFIG}/ckan.ini"
-
 exec "$@"
+

contrib/docker/docker-compose.yml

@@ -1,32 +1,66 @@
-# docker-compose build && docker-compose up
-# If you experience problems with the CKAN container not being
-# able to connect to the DB, then most likely the DB has not
-# started up quickly enough. Just do "docker-compose up ckan"
-# again to retry
-ckan:
-  container_name: ckan
-  # Note, if you are working on CKAN core, change this to
-  # build ../../
-  image: ckan/ckan:latest
-  links:
+# docker-compose build && docker-compose up -d
+# If "docker-compose logs ckan" shows DB not ready, run "docker-compose restart ckan" a few times.
+version: "3"
+
+volumes:
+  ckan_config:
+  ckan_home:
+  ckan_storage:
+  pg_data:
+
+services:
+  ckan:
+    container_name: ckan
+    build:
+      context: ../../
+      args:
+          - CKAN_SITE_URL=${CKAN_SITE_URL}
+    links:
       - db
       - solr
       - redis
-  ports:
-      - "80:5000"
-  environment:
-      - DB_PORT_5432_TCP_ADDR=db
-      - SOLR_PORT_8983_TCP_ADDR=solr
-      - REDIS_PORT_6379_TCP_ADDR=redis
+    ports:
+      - "0.0.0.0:${CKAN_PORT}:5000"
+    environment:
+      # Defaults work with linked containers, change to use own Postgres, SolR, Redis or Datapusher
+      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
+      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
+      - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore
+      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
+      - CKAN_REDIS_URL=redis://redis:6379/1
+      - CKAN_DATAPUSHER_URL=http://datapusher:8800
+      - CKAN_SITE_URL=${CKAN_SITE_URL}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
+
+    volumes:
+      - ckan_config:/etc/ckan
+      - ckan_home:/usr/lib/ckan
+      - ckan_storage:/var/lib/ckan
+
+  datapusher:
+    container_name: datapusher
+    image: clementmouchet/datapusher
+    ports:
+      - "8800:8800"
+
+  db:
+    container_name: db
+    build:
+      context: postgresql/
+      args:
+        - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
+        - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    environment:
+      - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+
+  solr:
+    container_name: solr
+    image: ckan/solr:latest
 
-db:
-  container_name: db
-  build: postgresql/
-
-solr:
-  container_name: solr
-  image: ckan/solr:latest
-
-redis:
-  container_name: redis
-  image: redis:latest
+  redis:
+    container_name: redis
+    image: redis:latest

contrib/docker/postgresql/Dockerfile

@@ -1,10 +1,17 @@
-FROM postgres:9.6
+#FROM postgres:9.6
+FROM mdillon/postgis
 MAINTAINER Open Knowledge
 
 # Allow connections; we don't map out any ports so only linked docker containers can connect
 RUN echo "host all  all    0.0.0.0/0  md5" >> /var/lib/postgresql/data/pg_hba.conf
 
 # Customize default user/pass/db
-ENV POSTGRES_USER ckan
-ENV POSTGRES_PASSWORD ckan
 ENV POSTGRES_DB ckan
+ENV POSTGRES_USER ckan
+ARG POSTGRES_PASSWORD
+ARG DS_RO_PASS
+
+# Include datastore setup scripts
+ADD /docker-entrypoint-initdb.d/00_create_datastore.sql ./00_create_datastore.sql
+ADD /docker-entrypoint-initdb.d/20_postgis_permissions.sql ./20_postgis_permissions.sql
+

contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sql

@@ -0,0 +1,3 @@
+CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS';
+CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';
+GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;

contrib/docker/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql

@@ -0,0 +1,3 @@
+CREATE EXTENSION POSTGIS;
+ALTER VIEW geometry_columns OWNER TO ckan;
+ALTER TABLE spatial_ref_sys OWNER TO ckan;