factor out more config variables

Dockerfile

@@ -1,5 +1,6 @@
 # docker build . -t ckan --build-arg CKAN_SITE_URL=http://localhost:5000
-# docker run -d -p 80:5000 --link db:db --link redis:redis --link solr:solr ckan
+# docker run -d -p 80:5000 --link db:db --link redis:redis --link solr:solr ckan \
+# -v ckan_config:/etc.ckan/default -v ckan_storage:/var/lib/ckan
 
 FROM debian:jessie
 MAINTAINER Open Knowledge
@@ -19,10 +20,9 @@ RUN apt-get -q -y update && apt-get -q -y upgrade && \
 ENV CKAN_HOME /usr/lib/ckan
 ENV CKAN_VENV $CKAN_HOME/default
 ENV CKAN_CONFIG /etc/ckan/default
-ENV CKAN_STORAGE_PATH /var/lib/ckan
+ENV CKAN_STORAGE_PATH=/var/lib/ckan
 
-# Build-time variables specified by docker-compose.yml / .env or
-# docker build . -t ckan --build-arg CKAN_SITE_URL=http://localhost:5000
+# Build-time variables specified by docker-compose.yml / .env
 ARG CKAN_SITE_URL
 
 # Create ckan user
@@ -45,10 +45,6 @@ RUN ckan-pip install --upgrade -r $CKAN_VENV/src/ckan/requirements.txt && \
 
 ENTRYPOINT ["/ckan-entrypoint.sh"]
 
-# Volumes
-VOLUME ["/etc/ckan/default"]
-VOLUME ["/var/lib/ckan"]
-
 USER ckan
 EXPOSE 5000
 

contrib/docker/.env.template

@@ -7,26 +7,36 @@
 # docker rmi $(docker images -f dangling=true -q)
 # docker-compose build
 # docker-compose up -d
-# docker-compose restart (run a few times until ckan starts up nicely)
+# docker-compose restart ckan # give the db service time to initialize the db cluster on first run
 
 # Image: ckan
+CKAN_SITE_ID=default
+#
 # On AWS, your CKAN_SITE_URL is the output of:
 # curl -s http://169.254.169.254/latest/meta-data/public-hostname
 CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com
 # CKAN_SITE_URL=http://localhost
 #
-# CKAN_PORT must be available on the host: sudo netstat -na | grep 80
+# CKAN_PORT must be available on the host: sudo netstat -na
 # To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan
 CKAN_PORT=80
+#
+# Email settings
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
 
 # Image: db
 POSTGRES_PASSWORD=ckan
 #
-# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 54
+# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 5432
 # To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build
 POSTGRES_PORT=5432
 #
-# The datastore database will be created in the db container
+# The datastore database will be created in the db container as docs
 # Readwrite user/pass will be ckan:POSTGRES_PASSWORD
-# Readonly user/pass will be datastore:DATASTORE_READONLY_PASSWORD
+# Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD
 DATASTORE_READONLY_PASSWORD=datastore
+

contrib/docker/ckan-entrypoint.sh

@@ -8,6 +8,8 @@ set -e
 : ${CKAN_SOLR_URL:=}
 # URL for redis (required unless linked to a container called 'redis')
 : ${CKAN_REDIS_URL:=}
+# URL for datapusher (required unless linked to a container called 'datapusher')
+: ${CKAN_DATAPUSHER_URL:=}
 
 CONFIG="${CKAN_CONFIG}/ckan.ini"
 
@@ -17,15 +19,20 @@ abort () {
 }
 
 set_environment () {
+  export CKAN_SITE_ID=${CKAN_SITE_ID}
+  export CKAN_SITE_URL=${CKAN_SITE_URL}
   export CKAN_SQLALCHEMY_URL=${CKAN_SQLALCHEMY_URL}
   export CKAN_SOLR_URL=${CKAN_SOLR_URL}
   export CKAN_REDIS_URL=${CKAN_REDIS_URL}
-  export CKAN_STORAGE_PATH=${CKAN_STORAGE_PATH}
-  export CKAN_SITE_URL=${CKAN_SITE_URL}
-  #ckan.datastore.write_url = postgresql://ckan:ckan@db/datastore
-  #ckan.datastore.read_url = postgresql://datastore:datastore@db/datastore
+  export CKAN_STORAGE_PATH=/var/lib/ckan
+  export CKAN_DATAPUSHER_URL=${CKAN_DATAPUSHER_URL}
   export CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
   export CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DS_RO_PASS}@db/datastore
+  export CKAN_SMTP_SERVER=${CKAN_SMTP_SERVER}
+  export CKAN_SMTP_STARTTLS=${CKAN_SMTP_STARTTLS}
+  export CKAN_SMTP_USER=${CKAN_SMTP_USER}
+  export CKAN_SMTP_PASSWORD=${CKAN_SMTP_PASSWORD}
+  export CKAN_SMTP_MAIL_FROM=${CKAN_SMTP_MAIL_FROM}
 }
 
 
@@ -64,15 +71,19 @@ link_redis_url () {
   echo "redis://${host}:${port}/1"
 }
 
+link_datapusher_url() {
+  local host=$DATAPUSHER_PORT_8800_ADDR
+  local port=$DATAPUSHER_PORT_8800_PORT
+  echo "http://${host}:${port}"
+
+}
+
 # If we don't already have a config file, bootstrap
 if [ ! -e "$CONFIG" ]; then
   write_config
 fi
 
-# Set environment variables
-echo "SQL_ALCHEMY_URL is provided to entrypoint as $CKAN_SQLALCHEMY_URL"
-
-
+# Get or create CKAN_SQLALCHEMY_URL
 if [ -z "$CKAN_SQLALCHEMY_URL" ]; then
   if ! CKAN_SQLALCHEMY_URL=$(link_postgres_url); then
     abort "ERROR: no CKAN_SQLALCHEMY_URL specified and linked container called 'db' was not found"
@@ -83,10 +94,9 @@ if [ -z "$CKAN_SQLALCHEMY_URL" ]; then
     export PGDATABASE=${DB_ENV_POSTGRES_DB}
     export PGUSER=${DB_ENV_POSTGRES_USER}
     export PGPASSWORD=${DB_ENV_POSTGRES_PASSWORD}
-    echo "PG password is $PGPASSWORD"
+    echo "CKAN_SQLALCHEMY_URL: $CKAN_SQLALCHEMY_URL"
 
-    # wait for postgres db to be available, immediately after creation
-    # its entrypoint creates the cluster and dbs and this can take a moment
+    # Give the db container time to initialize the db cluster (if first run)
     for tries in $(seq 60); do
       psql -c 'SELECT 1;' 2> /dev/null && break
       sleep 0.3
@@ -96,13 +106,19 @@ fi
 
 if [ -z "$CKAN_SOLR_URL" ]; then
   if ! CKAN_SOLR_URL=$(link_solr_url); then
-    abort "ERROR: no CKAN_SOLR_URL specified and linked container called 'solr' was not found"
+    abort "ERROR: no CKAN_SOLR_URL specified and no linked container called 'solr' found"
   fi
 fi
 
 if [ -z "$CKAN_REDIS_URL" ]; then
   if ! CKAN_REDIS_URL=$(link_redis_url); then
-    abort "ERROR: no CKAN_REDIS_URL specified and linked container called 'redis' was not found"
+    abort "ERROR: no CKAN_REDIS_URL specified and no linked container called 'redis' found"
+  fi
+fi
+
+if [ -z "$CKAN_DATAPUSHER_URL" ]; then
+  if ! CKAN_DATAPUSHER_URL=$(link_datapusher_url); then
+    abort "ERROR: no CKAN_DATAPUSHER_URL specified and no linked container called 'datapusher' found"
   fi
 fi
 

contrib/docker/docker-compose.yml

@@ -23,15 +23,21 @@ services:
       - "0.0.0.0:${CKAN_PORT}:5000"
     environment:
       - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
+      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
+      - CKAN_REDIS_URL=redis://redis:6379/1
       - CKAN_SITE_URL=${CKAN_SITE_URL}
-      - DB_PORT_5432_TCP_ADDR=db
-      - DB_PORT_5432_TCP_PORT=${POSTGRES_PORT}
+      - CKAN_DATAPUSHER_URL=http://datapusher:8800
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
-      - SOLR_PORT_8983_TCP_ADDR=solr
-      - SOLR_PORT_8983_TCP_PORT=8983
-      - REDIS_PORT_6379_TCP_ADDR=redis
-      - REDIS_PORT_6379_TCP_PORT=6379
+      #- DB_PORT_5432_TCP_ADDR=db
+      #- DB_PORT_5432_TCP_PORT=${POSTGRES_PORT}
+      #- SOLR_PORT_8983_TCP_ADDR=solr
+      #- SOLR_PORT_8983_TCP_PORT=8983
+      #- REDIS_PORT_6379_TCP_ADDR=redis
+      #- REDIS_PORT_6379_TCP_PORT=6379
+      #- DATAPUSHER_PORT_8800_ADDR=datapusher
+      #- DATAPUSHER_PORT_8800_PORT=8800
+
     volumes:
       - ckan_config:/etc/ckan/default
       - ckan_home:/usr/lib/ckan/default
@@ -41,7 +47,7 @@ services:
     container_name: datapusher
     image: clementmouchet/datapusher
     ports:
-        - "0.0.0.0:8800:8800"
+      - "8800:8800"
 
   db:
     container_name: db

contrib/docker/postgresql/Dockerfile

@@ -10,5 +10,6 @@ ENV POSTGRES_USER ckan
 ARG POSTGRES_PASSWORD
 ARG DS_RO_PASS
 
+# Include datastore setup scripts
 ADD /docker-entrypoint-initdb.d/00_create_datastore.sql ./00_create_datastore.sql
 ADD /docker-entrypoint-initdb.d/10_set_permissions.sql ./10_set_permissions.sql

doc/maintaining/installing/install-from-docker-compose.rst

@@ -81,11 +81,11 @@ In the  activated virtualenv, install and verify docker-compose using pip::
 In this step we will build the Docker images and create Docker data volumes with user-defined,
 sensitive settings (e.g. for database passwords).
 
-a. Sensitive settings
+a. Sensitive settings and environment variables
 
 In a production environment, copy ``contrib/docker/.env.template`` to ``contrib/docker/.env``
 and follow instructions within to set passwords and other sensitive or user-defined variables.
-The very unimaginative defaults will work fine in a development environment.
+The defaults will work fine in a development environment.
 
 b. Build the images