From 922d0eb168e1dfd7a2a513f5be5b661b15b9ba08 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 7 Jul 2017 13:29:17 +0800 Subject: [PATCH 01/51] docker-compose v3 working --- Dockerfile | 14 ++++--- contrib/docker/ckan-entrypoint.sh | 6 ++- contrib/docker/docker-compose.yml | 55 ++++++++++++++++++---------- contrib/docker/postgresql/Dockerfile | 4 +- 4 files changed, 52 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 47061a6a355..95091022177 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,6 @@ FROM debian:jessie MAINTAINER Open Knowledge -ENV CKAN_HOME /usr/lib/ckan/default -ENV CKAN_CONFIG /etc/ckan/default -ENV CKAN_STORAGE_PATH /var/lib/ckan -ENV CKAN_SITE_URL http://localhost:5000 - # Install required packages RUN apt-get -q -y update && apt-get -q -y upgrade && DEBIAN_FRONTEND=noninteractive apt-get -q -y install \ python-dev \ @@ -17,6 +12,15 @@ RUN apt-get -q -y update && apt-get -q -y upgrade && DEBIAN_FRONTEND=noninteract git-core \ && apt-get -q clean +# Define environment variables +ENV CKAN_HOME /usr/lib/ckan/default +ENV CKAN_CONFIG /etc/ckan/default +ENV CKAN_STORAGE_PATH /var/lib/ckan + +# Build-time variables specified by docker-compose.yml / .env or +# docker build . -t chan --build-arg CKAN_SITE_URL=http://localhost:5000 +ARG CKAN_SITE_URL + # SetUp Virtual Environment CKAN RUN mkdir -p $CKAN_HOME $CKAN_CONFIG $CKAN_STORAGE_PATH RUN virtualenv $CKAN_HOME diff --git a/contrib/docker/ckan-entrypoint.sh b/contrib/docker/ckan-entrypoint.sh index ab9d78e0952..d15a7e3faa9 100644 --- a/contrib/docker/ckan-entrypoint.sh +++ b/contrib/docker/ckan-entrypoint.sh @@ -65,6 +65,9 @@ if [ ! -e "$CONFIG" ]; then fi # Set environment variables +echo "SQL_ALCHEMY_URL is provided to entrypoint as $CKAN_SQLALCHEMY_URL" + + if [ -z "$CKAN_SQLALCHEMY_URL" ]; then if ! CKAN_SQLALCHEMY_URL=$(link_postgres_url); then abort "ERROR: no CKAN_SQLALCHEMY_URL specified and linked container called 'db' was not found" @@ -75,6 +78,7 @@ if [ -z "$CKAN_SQLALCHEMY_URL" ]; then export PGDATABASE=${DB_ENV_POSTGRES_DB} export PGUSER=${DB_ENV_POSTGRES_USER} export PGPASSWORD=${DB_ENV_POSTGRES_PASSWORD} + echo "PG password is $PGPASSWORD" # wait for postgres db to be available, immediately after creation # its entrypoint creates the cluster and dbs and this can take a moment @@ -90,7 +94,7 @@ if [ -z "$CKAN_SOLR_URL" ]; then abort "ERROR: no CKAN_SOLR_URL specified and linked container called 'solr' was not found" fi fi - + if [ -z "$CKAN_REDIS_URL" ]; then if ! CKAN_REDIS_URL=$(link_redis_url); then abort "ERROR: no CKAN_REDIS_URL specified and linked container called 'redis' was not found" diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index e883e33d222..76c2584a389 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -3,30 +3,45 @@ # able to connect to the DB, then most likely the DB has not # started up quickly enough. Just do "docker-compose up ckan" # again to retry -ckan: - container_name: ckan - # Note, if you are working on CKAN core, change this to - # build ../../ - image: ckan/ckan:latest - links: +version: "3" +services: + ckan: + container_name: ckan + build: + context: ../../ + args: + - CKAN_SITE_URL=${CKAN_SITE_URL} + links: - db - solr - redis - ports: + ports: - "80:5000" - environment: + environment: + - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan + - CKAN_SITE_URL=${CKAN_SITE_URL} - DB_PORT_5432_TCP_ADDR=db + - DB_PORT_5432_TCP_PORT=5432 - SOLR_PORT_8983_TCP_ADDR=solr + - SOLR_PORT_8983_TCP_PORT=8983 - REDIS_PORT_6379_TCP_ADDR=redis - -db: - container_name: db - build: postgresql/ - -solr: - container_name: solr - image: ckan/solr:latest - -redis: - container_name: redis - image: redis:latest + db: + container_name: db + build: + context: postgresql/ + args: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + ports: + - "5432:5432" + environment: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + solr: + container_name: solr + image: ckan/solr:latest + ports: + - "8983:8983" + redis: + container_name: redis + image: redis:latest + ports: + - "6379:6379" diff --git a/contrib/docker/postgresql/Dockerfile b/contrib/docker/postgresql/Dockerfile index 3b77d9a4e31..9d97cde79f2 100644 --- a/contrib/docker/postgresql/Dockerfile +++ b/contrib/docker/postgresql/Dockerfile @@ -6,5 +6,7 @@ RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.co # Customize default user/pass/db ENV POSTGRES_USER ckan -ENV POSTGRES_PASSWORD ckan +ARG POSTGRES_PASSWORD ENV POSTGRES_DB ckan + +RUN echo "Building postgres:9.6 with user $POSTGRES_USER, pw $POSTGRES_PASSWORD, db $POSTGRES_DB" From 65ce565fe4e51a31853eb060c02e191130c74912 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 7 Jul 2017 13:31:41 +0800 Subject: [PATCH 02/51] gitignore contrib/docker/.env --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b765bfee61f..4bc777bdd9d 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,6 @@ ckan_deb/DEBIAN/prerm # node.js node_modules/ + +# docker +contrib/docker/.env From 648fd6fd7b5c5bf22697aa45b7c4cfecca93ea48 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 7 Jul 2017 14:45:20 +0800 Subject: [PATCH 03/51] incorporate #3651 --- Dockerfile | 49 ++++++++++++++-------------- contrib/docker/apache.wsgi | 2 +- contrib/docker/postgresql/Dockerfile | 1 + 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95091022177..827c4021a92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,55 +1,56 @@ -# docker build . -t ckan && docker run -d -p 80:5000 --link db:db --link redis:redis --link solr:solr ckan +# docker build . -t ckan --build-arg CKAN_SITE_URL=http://localhost:5000 +# docker run -d -p 80:5000 --link db:db --link redis:redis --link solr:solr ckan FROM debian:jessie MAINTAINER Open Knowledge # Install required packages -RUN apt-get -q -y update && apt-get -q -y upgrade && DEBIAN_FRONTEND=noninteractive apt-get -q -y install \ +RUN apt-get -q -y update && apt-get -q -y upgrade && \ + DEBIAN_FRONTEND=noninteractive apt-get -q -y install \ python-dev \ python-pip \ python-virtualenv \ libpq-dev \ git-core \ + postgresql-client \ && apt-get -q clean # Define environment variables ENV CKAN_HOME /usr/lib/ckan/default +ENV CKAN_VENV $CKAN_HOME/default ENV CKAN_CONFIG /etc/ckan/default ENV CKAN_STORAGE_PATH /var/lib/ckan # Build-time variables specified by docker-compose.yml / .env or -# docker build . -t chan --build-arg CKAN_SITE_URL=http://localhost:5000 +# docker build . -t ckan --build-arg CKAN_SITE_URL=http://localhost:5000 ARG CKAN_SITE_URL -# SetUp Virtual Environment CKAN -RUN mkdir -p $CKAN_HOME $CKAN_CONFIG $CKAN_STORAGE_PATH -RUN virtualenv $CKAN_HOME -RUN ln -s $CKAN_HOME/bin/pip /usr/local/bin/ckan-pip -RUN ln -s $CKAN_HOME/bin/paster /usr/local/bin/ckan-paster - -# SetUp Requirements -ADD ./requirements.txt $CKAN_HOME/src/ckan/requirements.txt -RUN ckan-pip install --upgrade -r $CKAN_HOME/src/ckan/requirements.txt +# add ckan user which runs all the ckan related stuff +RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan -# TMP-BUGFIX https://github.com/ckan/ckan/issues/3388 -ADD ./dev-requirements.txt $CKAN_HOME/src/ckan/dev-requirements.txt -RUN ckan-pip install --upgrade -r $CKAN_HOME/src/ckan/dev-requirements.txt - -# TMP-BUGFIX https://github.com/ckan/ckan/issues/3594 -RUN ckan-pip install --upgrade urllib3 +# SetUp Virtual Environment CKAN +RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \ + virtualenv $CKAN_VENV && \ + ln -s $CKAN_VENV/bin/pip /usr/local/bin/ckan-pip &&\ + ln -s $CKAN_VENV/bin/paster /usr/local/bin/ckan-paster # SetUp CKAN -ADD . $CKAN_HOME/src/ckan/ -RUN ckan-pip install -e $CKAN_HOME/src/ckan/ -RUN ln -s $CKAN_HOME/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini +ADD . $CKAN_VENV/src/ckan/ +RUN ckan-pip install --upgrade -r $CKAN_VENV/src/ckan/requirements.txt && \ + ckan-pip install -e $CKAN_VENV/src/ckan/ && \ + ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \ + cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \ + chmod +x /ckan-entrypoint.sh && \ + chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH -# SetUp EntryPoint -COPY ./contrib/docker/ckan-entrypoint.sh / -RUN chmod +x /ckan-entrypoint.sh ENTRYPOINT ["/ckan-entrypoint.sh"] # Volumes VOLUME ["/etc/ckan/default"] VOLUME ["/var/lib/ckan"] + +USER ckan EXPOSE 5000 + CMD ["ckan-paster","serve","/etc/ckan/default/ckan.ini"] + diff --git a/contrib/docker/apache.wsgi b/contrib/docker/apache.wsgi index ed9b6bcb256..70625d29cd8 100644 --- a/contrib/docker/apache.wsgi +++ b/contrib/docker/apache.wsgi @@ -1,5 +1,5 @@ import os -ckan_home = os.environ.get('CKAN_HOME', '/usr/lib/ckan/default') +ckan_home = os.environ.get('CKAN_VENV', '/usr/lib/ckan/default') activate_this = os.path.join(ckan_home, 'bin/activate_this.py') execfile(activate_this, dict(__file__=activate_this)) diff --git a/contrib/docker/postgresql/Dockerfile b/contrib/docker/postgresql/Dockerfile index 9d97cde79f2..327dda57c0d 100644 --- a/contrib/docker/postgresql/Dockerfile +++ b/contrib/docker/postgresql/Dockerfile @@ -9,4 +9,5 @@ ENV POSTGRES_USER ckan ARG POSTGRES_PASSWORD ENV POSTGRES_DB ckan +# Output variables when building for the first time, will not run if using cached image RUN echo "Building postgres:9.6 with user $POSTGRES_USER, pw $POSTGRES_PASSWORD, db $POSTGRES_DB" From a45b5e86bdcf9d9ec04c98ecc1bf96f130a30813 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 7 Jul 2017 15:52:21 +0800 Subject: [PATCH 04/51] ckan Dockerfile: move useradd below mkdir --- Dockerfile | 12 ++++++------ contrib/docker/docker-compose.yml | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 827c4021a92..e3835280f51 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM debian:jessie MAINTAINER Open Knowledge -# Install required packages +# Install required system packages RUN apt-get -q -y update && apt-get -q -y upgrade && \ DEBIAN_FRONTEND=noninteractive apt-get -q -y install \ python-dev \ @@ -25,16 +25,16 @@ ENV CKAN_STORAGE_PATH /var/lib/ckan # docker build . -t ckan --build-arg CKAN_SITE_URL=http://localhost:5000 ARG CKAN_SITE_URL -# add ckan user which runs all the ckan related stuff -RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan - -# SetUp Virtual Environment CKAN +# Setup virtual environment for CKAN RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \ virtualenv $CKAN_VENV && \ ln -s $CKAN_VENV/bin/pip /usr/local/bin/ckan-pip &&\ ln -s $CKAN_VENV/bin/paster /usr/local/bin/ckan-paster -# SetUp CKAN +# Create ckan user +RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan + +# Setup CKAN ADD . $CKAN_VENV/src/ckan/ RUN ckan-pip install --upgrade -r $CKAN_VENV/src/ckan/requirements.txt && \ ckan-pip install -e $CKAN_VENV/src/ckan/ && \ diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 76c2584a389..670fc61d2e0 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -25,6 +25,7 @@ services: - SOLR_PORT_8983_TCP_ADDR=solr - SOLR_PORT_8983_TCP_PORT=8983 - REDIS_PORT_6379_TCP_ADDR=redis + - REDIS_PORT_6379_TCP_PORT=6379 db: container_name: db build: From d046e295aaffacafdefcfe57151b68cfdf90bf37 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 7 Jul 2017 16:06:34 +0800 Subject: [PATCH 05/51] fix ckan Dockerfile to follow #3651 --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e3835280f51..1fedb5c042a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get -q -y update && apt-get -q -y upgrade && \ && apt-get -q clean # Define environment variables -ENV CKAN_HOME /usr/lib/ckan/default +ENV CKAN_HOME /usr/lib/ckan ENV CKAN_VENV $CKAN_HOME/default ENV CKAN_CONFIG /etc/ckan/default ENV CKAN_STORAGE_PATH /var/lib/ckan @@ -25,15 +25,15 @@ ENV CKAN_STORAGE_PATH /var/lib/ckan # docker build . -t ckan --build-arg CKAN_SITE_URL=http://localhost:5000 ARG CKAN_SITE_URL +# Create ckan user +RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan + # Setup virtual environment for CKAN RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \ virtualenv $CKAN_VENV && \ ln -s $CKAN_VENV/bin/pip /usr/local/bin/ckan-pip &&\ ln -s $CKAN_VENV/bin/paster /usr/local/bin/ckan-paster -# Create ckan user -RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan - # Setup CKAN ADD . $CKAN_VENV/src/ckan/ RUN ckan-pip install --upgrade -r $CKAN_VENV/src/ckan/requirements.txt && \ From a9d1969d1b16f6a41a167ca65acf1fd99d20e95a Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 7 Jul 2017 16:14:04 +0800 Subject: [PATCH 06/51] add .env.template --- contrib/docker/.env.template | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 contrib/docker/.env.template diff --git a/contrib/docker/.env.template b/contrib/docker/.env.template new file mode 100644 index 00000000000..22c3ce1dcaf --- /dev/null +++ b/contrib/docker/.env.template @@ -0,0 +1,13 @@ +# Variables in this file will be substituted in docker-compose.yml +# Verify correct substitution with "docker-compose config" + +# Image: ckan +# When developing on AWS, your CKAN_SITE_URL is the output of: +# curl -s http://169.254.169.254/latest/meta-data/public-hostname +CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com +# When developing locally, your CKAN_SITE_URL can be +# CKAN_SITE_URL=http://localhost:80 + +# Image: db +POSTGRES_PASSWORD=ckan + From 7e75ec0ebec4f23abfa43dd1199fe82af8535b49 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 7 Jul 2017 16:16:58 +0800 Subject: [PATCH 07/51] add instructions to .env.template --- contrib/docker/.env.template | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/contrib/docker/.env.template b/contrib/docker/.env.template index 22c3ce1dcaf..43b0788c7b8 100644 --- a/contrib/docker/.env.template +++ b/contrib/docker/.env.template @@ -1,5 +1,12 @@ # Variables in this file will be substituted in docker-compose.yml # Verify correct substitution with "docker-compose config" +# After changing these variables, delete and rebuild the images: +# docker-compose down +# docker rmi -f docker_ckan docker_db +# docker rmi $(docker images -f dangling=true -q) +# docker-compose build +# docker-compose up -d +# docker-compose restart (a few times until ckan starts up nicely) # Image: ckan # When developing on AWS, your CKAN_SITE_URL is the output of: From d415b0d7a45f771e70e0e1278f81eb0ad049a984 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Mon, 10 Jul 2017 10:20:22 +0800 Subject: [PATCH 08/51] add comments to docker templates --- contrib/docker/.env.template | 9 +++++---- contrib/docker/postgresql/Dockerfile | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/contrib/docker/.env.template b/contrib/docker/.env.template index 43b0788c7b8..cd573fea0e8 100644 --- a/contrib/docker/.env.template +++ b/contrib/docker/.env.template @@ -1,15 +1,16 @@ -# Variables in this file will be substituted in docker-compose.yml +# Variables in this file will be substituted into docker-compose.yml +# Save a copy of this file as .env and insert your own values. # Verify correct substitution with "docker-compose config" -# After changing these variables, delete and rebuild the images: +# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: # docker-compose down # docker rmi -f docker_ckan docker_db # docker rmi $(docker images -f dangling=true -q) # docker-compose build # docker-compose up -d -# docker-compose restart (a few times until ckan starts up nicely) +# docker-compose restart (run a few times until ckan starts up nicely) # Image: ckan -# When developing on AWS, your CKAN_SITE_URL is the output of: +# On AWS, your CKAN_SITE_URL is the output of: # curl -s http://169.254.169.254/latest/meta-data/public-hostname CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com # When developing locally, your CKAN_SITE_URL can be diff --git a/contrib/docker/postgresql/Dockerfile b/contrib/docker/postgresql/Dockerfile index 327dda57c0d..b2b4a126a17 100644 --- a/contrib/docker/postgresql/Dockerfile +++ b/contrib/docker/postgresql/Dockerfile @@ -9,5 +9,5 @@ ENV POSTGRES_USER ckan ARG POSTGRES_PASSWORD ENV POSTGRES_DB ckan -# Output variables when building for the first time, will not run if using cached image +# Output variables when building for the first time. Will not run if using cached image. RUN echo "Building postgres:9.6 with user $POSTGRES_USER, pw $POSTGRES_PASSWORD, db $POSTGRES_DB" From c475ab7e99726287125762dc36d6239cb3a696d9 Mon Sep 17 00:00:00 2001 From: florianm Date: Mon, 10 Jul 2017 20:07:49 +0800 Subject: [PATCH 09/51] factor out POSTGRES_PORT, but docker_ckan cannot see it yet --- contrib/docker/.env.template | 9 +++++++-- contrib/docker/docker-compose.yml | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/contrib/docker/.env.template b/contrib/docker/.env.template index cd573fea0e8..c58f0e88b15 100644 --- a/contrib/docker/.env.template +++ b/contrib/docker/.env.template @@ -13,9 +13,14 @@ # On AWS, your CKAN_SITE_URL is the output of: # curl -s http://169.254.169.254/latest/meta-data/public-hostname CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com -# When developing locally, your CKAN_SITE_URL can be -# CKAN_SITE_URL=http://localhost:80 +# When developing locally, your CKAN_SITE_URL can be any free local port +# test whether local ports are taken: sudo netstat -na | grep 8001 +# CKAN_SITE_URL=http://localhost:8001 # Image: db POSTGRES_PASSWORD=ckan +# POSTGRES_PORT needs to be available on the host +# Which ports are taken: sudo netstat -na | grep 54 +# To apply change: docker-compose down && docker rmi docker_db && docker-compose build db +POSTGRES_PORT=5435 diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 670fc61d2e0..75ab97394fb 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -18,10 +18,10 @@ services: ports: - "80:5000" environment: - - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan + - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db:${POSTGRES_PORT}/ckan - CKAN_SITE_URL=${CKAN_SITE_URL} - DB_PORT_5432_TCP_ADDR=db - - DB_PORT_5432_TCP_PORT=5432 + - DB_PORT_5432_TCP_PORT=${POSTGRES_PORT} - SOLR_PORT_8983_TCP_ADDR=solr - SOLR_PORT_8983_TCP_PORT=8983 - REDIS_PORT_6379_TCP_ADDR=redis @@ -33,7 +33,7 @@ services: args: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} ports: - - "5432:5432" + - "${POSTGRES_PORT}:5432" environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} solr: From c5d0e3331cc05e7e8368c83623474fc2cae3ac1c Mon Sep 17 00:00:00 2001 From: florianm Date: Mon, 10 Jul 2017 20:11:51 +0800 Subject: [PATCH 10/51] update .env.template with instructions to update vars --- contrib/docker/.env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/.env.template b/contrib/docker/.env.template index c58f0e88b15..1766bdd2dc7 100644 --- a/contrib/docker/.env.template +++ b/contrib/docker/.env.template @@ -22,5 +22,5 @@ POSTGRES_PASSWORD=ckan # POSTGRES_PORT needs to be available on the host # Which ports are taken: sudo netstat -na | grep 54 -# To apply change: docker-compose down && docker rmi docker_db && docker-compose build db +# To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build POSTGRES_PORT=5435 From 40c6f47606c6a92bba9b6e27869b9ef7afc1d359 Mon Sep 17 00:00:00 2001 From: florianm Date: Mon, 10 Jul 2017 21:24:07 +0800 Subject: [PATCH 11/51] working setup with custom CKAN_PORT, but POSTGRES_PORT must be 5432 --- contrib/docker/.env.template | 2 +- contrib/docker/docker-compose.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/docker/.env.template b/contrib/docker/.env.template index 1766bdd2dc7..64f7da89f6f 100644 --- a/contrib/docker/.env.template +++ b/contrib/docker/.env.template @@ -23,4 +23,4 @@ POSTGRES_PASSWORD=ckan # POSTGRES_PORT needs to be available on the host # Which ports are taken: sudo netstat -na | grep 54 # To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build -POSTGRES_PORT=5435 +POSTGRES_PORT=5432 diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 75ab97394fb..b308fdf6cc0 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -10,15 +10,15 @@ services: build: context: ../../ args: - - CKAN_SITE_URL=${CKAN_SITE_URL} + - CKAN_SITE_URL=${CKAN_SITE_URL}:${CKAN_PORT} links: - db - solr - redis ports: - - "80:5000" + - "${CKAN_PORT}:5000" environment: - - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db:${POSTGRES_PORT}/ckan + - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan - CKAN_SITE_URL=${CKAN_SITE_URL} - DB_PORT_5432_TCP_ADDR=db - DB_PORT_5432_TCP_PORT=${POSTGRES_PORT} From c0ab1e97f6a59d3702477fb3044d337e90bd676f Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Tue, 11 Jul 2017 10:25:59 +0800 Subject: [PATCH 12/51] restrict db, redis, solr ports to local connections --- contrib/docker/.env.template | 13 +++++++------ contrib/docker/docker-compose.yml | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/contrib/docker/.env.template b/contrib/docker/.env.template index 64f7da89f6f..3f5e2ebade1 100644 --- a/contrib/docker/.env.template +++ b/contrib/docker/.env.template @@ -13,14 +13,15 @@ # On AWS, your CKAN_SITE_URL is the output of: # curl -s http://169.254.169.254/latest/meta-data/public-hostname CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com -# When developing locally, your CKAN_SITE_URL can be any free local port -# test whether local ports are taken: sudo netstat -na | grep 8001 -# CKAN_SITE_URL=http://localhost:8001 +# CKAN_SITE_URL=http://localhost +# +# CKAN_PORT must be available on the host: sudo netstat -na | grep 80 +# To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan +CKAN_PORT=80 # Image: db POSTGRES_PASSWORD=ckan - -# POSTGRES_PORT needs to be available on the host -# Which ports are taken: sudo netstat -na | grep 54 +# +# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 54 # To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build POSTGRES_PORT=5432 diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index b308fdf6cc0..59c3282c96c 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -16,7 +16,7 @@ services: - solr - redis ports: - - "${CKAN_PORT}:5000" + - "0.0.0.0:${CKAN_PORT}:5000" environment: - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan - CKAN_SITE_URL=${CKAN_SITE_URL} @@ -33,16 +33,16 @@ services: args: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} ports: - - "${POSTGRES_PORT}:5432" + - "127.0.0.1:${POSTGRES_PORT}:5432" environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} solr: container_name: solr image: ckan/solr:latest ports: - - "8983:8983" + - "127.0.0.1:8983:8983" redis: container_name: redis image: redis:latest ports: - - "6379:6379" + - "127.0.0.1:6379:6379" From a756c726bb7284e65175bf23ebb632ef24f12f4b Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Tue, 11 Jul 2017 12:41:06 +0800 Subject: [PATCH 13/51] add volume for ckan config --- contrib/docker/docker-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 59c3282c96c..c0cc25d4612 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -4,6 +4,8 @@ # started up quickly enough. Just do "docker-compose up ckan" # again to retry version: "3" +volumes: + ckan_config: services: ckan: container_name: ckan @@ -26,6 +28,8 @@ services: - SOLR_PORT_8983_TCP_PORT=8983 - REDIS_PORT_6379_TCP_ADDR=redis - REDIS_PORT_6379_TCP_PORT=6379 + volumes: + - ckan_config:/etc/ckan/default db: container_name: db build: @@ -40,7 +44,7 @@ services: container_name: solr image: ckan/solr:latest ports: - - "127.0.0.1:8983:8983" + - "8983:8983" redis: container_name: redis image: redis:latest From bc54bc82b97ad18051ebf34930075565be7fa225 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Tue, 11 Jul 2017 15:46:08 +0800 Subject: [PATCH 14/51] add named data volume ckan_lib to service ckan in docker-compose.yml --- contrib/docker/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index c0cc25d4612..b9fe077dc16 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,6 +6,7 @@ version: "3" volumes: ckan_config: + ckan_lib: services: ckan: container_name: ckan @@ -30,6 +31,7 @@ services: - REDIS_PORT_6379_TCP_PORT=6379 volumes: - ckan_config:/etc/ckan/default + - ckan_lib:/var/lib/ckan db: container_name: db build: From 34471a9ba72f401d102365a257e44ac1d62f7df7 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Tue, 11 Jul 2017 16:06:55 +0800 Subject: [PATCH 15/51] docker-compose.yml: add named data volumes for ckan_home, ckan_storage and pg_data --- contrib/docker/docker-compose.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index b9fe077dc16..c34bc289fa7 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -4,9 +4,13 @@ # started up quickly enough. Just do "docker-compose up ckan" # again to retry version: "3" + volumes: ckan_config: - ckan_lib: + ckan_home: + ckan_storage: + pg_data: + services: ckan: container_name: ckan @@ -31,7 +35,9 @@ services: - REDIS_PORT_6379_TCP_PORT=6379 volumes: - ckan_config:/etc/ckan/default - - ckan_lib:/var/lib/ckan + - ckan_home:/usr/lib/ckan/default + - ckan_storage:/var/lib/ckan + db: container_name: db build: @@ -42,11 +48,15 @@ services: - "127.0.0.1:${POSTGRES_PORT}:5432" environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + volumes: + - pg_data:/var/lib/postgresql/data + solr: container_name: solr image: ckan/solr:latest ports: - - "8983:8983" + - "0.0.0.0:8983:8983" + redis: container_name: redis image: redis:latest From 8ebbc5694e192ac877a579d0f9bc695273959d49 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 12 Jul 2017 13:19:46 +0800 Subject: [PATCH 16/51] docker-compose.yml and entrypoint: sanitise port settings --- contrib/docker/ckan-entrypoint.sh | 2 +- contrib/docker/docker-compose.yml | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/contrib/docker/ckan-entrypoint.sh b/contrib/docker/ckan-entrypoint.sh index d15a7e3faa9..62a3ce57dfe 100644 --- a/contrib/docker/ckan-entrypoint.sh +++ b/contrib/docker/ckan-entrypoint.sh @@ -82,7 +82,7 @@ if [ -z "$CKAN_SQLALCHEMY_URL" ]; then # wait for postgres db to be available, immediately after creation # its entrypoint creates the cluster and dbs and this can take a moment - for tries in $(seq 30); do + for tries in $(seq 60); do psql -c 'SELECT 1;' 2> /dev/null && break sleep 0.3 done diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index c34bc289fa7..8a9eedb0f35 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -1,8 +1,5 @@ -# docker-compose build && docker-compose up -# If you experience problems with the CKAN container not being -# able to connect to the DB, then most likely the DB has not -# started up quickly enough. Just do "docker-compose up ckan" -# again to retry +# docker-compose build && docker-compose up -d +# If "docker-compose logs ckan" shows DB not ready, run "docker-compose restart ckan" a few times. version: "3" volumes: @@ -44,8 +41,6 @@ services: context: postgresql/ args: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - ports: - - "127.0.0.1:${POSTGRES_PORT}:5432" environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} volumes: @@ -54,11 +49,7 @@ services: solr: container_name: solr image: ckan/solr:latest - ports: - - "0.0.0.0:8983:8983" redis: container_name: redis image: redis:latest - ports: - - "127.0.0.1:6379:6379" From ef71f6601fa64f1bf55c22989f81922934b91096 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 12 Jul 2017 16:42:40 +0800 Subject: [PATCH 17/51] first steps with datastore --- contrib/docker/.env.template | 5 +++++ contrib/docker/ckan-entrypoint.sh | 5 +++++ contrib/docker/docker-compose.yml | 3 +++ contrib/docker/postgresql/Dockerfile | 1 + .../docker-entrypoint-initdb.d/00_create_datastore.sh | 10 ++++++++++ 5 files changed, 24 insertions(+) create mode 100644 contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh diff --git a/contrib/docker/.env.template b/contrib/docker/.env.template index 3f5e2ebade1..645b2a7afdb 100644 --- a/contrib/docker/.env.template +++ b/contrib/docker/.env.template @@ -25,3 +25,8 @@ POSTGRES_PASSWORD=ckan # POSTGRES_PORT must be available on the host: sudo netstat -na | grep 54 # To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build POSTGRES_PORT=5432 +# +# The datastore database will be created in the db container +# Readwrite user/pass will be ckan:POSTGRES_PASSWORD +# Readonly user/pass will be datastore:DATASTORE_READONLY_PASSWORD +DATASTORE_READONLY_PASSWORD=datastore diff --git a/contrib/docker/ckan-entrypoint.sh b/contrib/docker/ckan-entrypoint.sh index 62a3ce57dfe..7b0fb698f06 100644 --- a/contrib/docker/ckan-entrypoint.sh +++ b/contrib/docker/ckan-entrypoint.sh @@ -22,8 +22,13 @@ set_environment () { export CKAN_REDIS_URL=${CKAN_REDIS_URL} export CKAN_STORAGE_PATH=${CKAN_STORAGE_PATH} export CKAN_SITE_URL=${CKAN_SITE_URL} + #ckan.datastore.write_url = postgresql://ckan:ckan@db/datastore + #ckan.datastore.read_url = postgresql://datastore:datastore@db/datastore + export CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore + export CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DS_RO_PASS}@db/datastore } + write_config () { # Note that this only gets called if there is no config, see below! ckan-paster make-config --no-interactive ckan "$CONFIG" diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 8a9eedb0f35..f17368f6db4 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -40,11 +40,14 @@ services: build: context: postgresql/ args: + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} environment: + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} volumes: - pg_data:/var/lib/postgresql/data + #- ./00_create_datastore.sh:/docker-entrypoint-initdb.d/00_create_datastore.sh solr: container_name: solr diff --git a/contrib/docker/postgresql/Dockerfile b/contrib/docker/postgresql/Dockerfile index b2b4a126a17..ef4b05e4de9 100644 --- a/contrib/docker/postgresql/Dockerfile +++ b/contrib/docker/postgresql/Dockerfile @@ -7,6 +7,7 @@ RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.co # Customize default user/pass/db ENV POSTGRES_USER ckan ARG POSTGRES_PASSWORD +ARG DS_RO_PASS ENV POSTGRES_DB ckan # Output variables when building for the first time. Will not run if using cached image. diff --git a/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh new file mode 100644 index 00000000000..7a6f45ea775 --- /dev/null +++ b/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e +echo "Creating datastore readonly user and database in image db..." +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE USER datastore_ro NOCREATEDB NOSUPERUSER NOCREATEROLE PASSWORD $DS_RO_PASS; + CREATE DATABASE datastore OWNER ckan ENCODING utf-8; + GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; +EOSQL +echo"done." +#sudo -u postgres createuser -S -D -R -P -l datastore_ro From 53aac5e8b76b5537b82620e6e8d4f46474540d91 Mon Sep 17 00:00:00 2001 From: florianm Date: Wed, 12 Jul 2017 19:59:49 +0800 Subject: [PATCH 18/51] updates to datastore script --- contrib/docker/postgresql/Dockerfile | 2 ++ .../docker-entrypoint-initdb.d/00_create_datastore.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/docker/postgresql/Dockerfile b/contrib/docker/postgresql/Dockerfile index ef4b05e4de9..978d035825f 100644 --- a/contrib/docker/postgresql/Dockerfile +++ b/contrib/docker/postgresql/Dockerfile @@ -12,3 +12,5 @@ ENV POSTGRES_DB ckan # Output variables when building for the first time. Will not run if using cached image. RUN echo "Building postgres:9.6 with user $POSTGRES_USER, pw $POSTGRES_PASSWORD, db $POSTGRES_DB" + +ADD /docker-entrypoint-initdb.d/00_create_datastore.sh ./00_create_datastore.sh diff --git a/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh index 7a6f45ea775..3bdfaeeed55 100644 --- a/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh +++ b/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh @@ -2,7 +2,7 @@ set -e echo "Creating datastore readonly user and database in image db..." psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL - CREATE USER datastore_ro NOCREATEDB NOSUPERUSER NOCREATEROLE PASSWORD $DS_RO_PASS; + CREATE USER datastore_ro LOGIN NOCREATEDB NOSUPERUSER NOCREATEROLE ENCRYPTED PASSWORD "$DS_RO_PASS"; CREATE DATABASE datastore OWNER ckan ENCODING utf-8; GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; EOSQL From ed8745025ae425f0b06a0982c1288a73d7a4d9cd Mon Sep 17 00:00:00 2001 From: florianm Date: Wed, 12 Jul 2017 21:03:28 +0800 Subject: [PATCH 19/51] 00_create_datastore.sh works --- .../docker-entrypoint-initdb.d/00_create_datastore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh index 3bdfaeeed55..0d067db9eba 100644 --- a/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh +++ b/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh @@ -2,7 +2,7 @@ set -e echo "Creating datastore readonly user and database in image db..." psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL - CREATE USER datastore_ro LOGIN NOCREATEDB NOSUPERUSER NOCREATEROLE ENCRYPTED PASSWORD "$DS_RO_PASS"; + CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS'; CREATE DATABASE datastore OWNER ckan ENCODING utf-8; GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; EOSQL From 325ef5c7ed892c4e4ef30c8a8b72a9bf8889a04a Mon Sep 17 00:00:00 2001 From: florianm Date: Wed, 12 Jul 2017 22:52:28 +0800 Subject: [PATCH 20/51] datastore works --- contrib/docker/docker-compose.yml | 3 +- contrib/docker/postgresql/Dockerfile | 8 +-- .../00_create_datastore.sh | 10 --- .../00_create_datastore.sql | 3 + .../10_set_permissions.sql | 72 +++++++++++++++++++ 5 files changed, 80 insertions(+), 16 deletions(-) delete mode 100644 contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh create mode 100644 contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sql create mode 100644 contrib/docker/postgresql/docker-entrypoint-initdb.d/10_set_permissions.sql diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index f17368f6db4..4c813253562 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -26,6 +26,8 @@ services: - CKAN_SITE_URL=${CKAN_SITE_URL} - DB_PORT_5432_TCP_ADDR=db - DB_PORT_5432_TCP_PORT=${POSTGRES_PORT} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - SOLR_PORT_8983_TCP_ADDR=solr - SOLR_PORT_8983_TCP_PORT=8983 - REDIS_PORT_6379_TCP_ADDR=redis @@ -47,7 +49,6 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} volumes: - pg_data:/var/lib/postgresql/data - #- ./00_create_datastore.sh:/docker-entrypoint-initdb.d/00_create_datastore.sh solr: container_name: solr diff --git a/contrib/docker/postgresql/Dockerfile b/contrib/docker/postgresql/Dockerfile index 978d035825f..5e28c855431 100644 --- a/contrib/docker/postgresql/Dockerfile +++ b/contrib/docker/postgresql/Dockerfile @@ -5,12 +5,10 @@ MAINTAINER Open Knowledge RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf # Customize default user/pass/db +ENV POSTGRES_DB ckan ENV POSTGRES_USER ckan ARG POSTGRES_PASSWORD ARG DS_RO_PASS -ENV POSTGRES_DB ckan - -# Output variables when building for the first time. Will not run if using cached image. -RUN echo "Building postgres:9.6 with user $POSTGRES_USER, pw $POSTGRES_PASSWORD, db $POSTGRES_DB" -ADD /docker-entrypoint-initdb.d/00_create_datastore.sh ./00_create_datastore.sh +ADD /docker-entrypoint-initdb.d/00_create_datastore.sql ./00_create_datastore.sql +ADD /docker-entrypoint-initdb.d/10_set_permissions.sql ./10_set_permissions.sql diff --git a/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh deleted file mode 100644 index 0d067db9eba..00000000000 --- a/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e -echo "Creating datastore readonly user and database in image db..." -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL - CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS'; - CREATE DATABASE datastore OWNER ckan ENCODING utf-8; - GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; -EOSQL -echo"done." -#sudo -u postgres createuser -S -D -R -P -l datastore_ro diff --git a/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sql b/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sql new file mode 100644 index 00000000000..06340ecb932 --- /dev/null +++ b/contrib/docker/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sql @@ -0,0 +1,3 @@ +CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS'; +CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; +GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; diff --git a/contrib/docker/postgresql/docker-entrypoint-initdb.d/10_set_permissions.sql b/contrib/docker/postgresql/docker-entrypoint-initdb.d/10_set_permissions.sql new file mode 100644 index 00000000000..137110e8bf3 --- /dev/null +++ b/contrib/docker/postgresql/docker-entrypoint-initdb.d/10_set_permissions.sql @@ -0,0 +1,72 @@ +/* +This script configures the permissions for the datastore. + +It ensures that the datastore read-only user will only be able to select from +the datastore database but has no create/write/edit permission or any +permissions on other databases. You must execute this script as a database +superuser on the PostgreSQL server that hosts your datastore database. + +For example, if PostgreSQL is running locally and the "postgres" user has the +appropriate permissions (as in the default Ubuntu PostgreSQL install), you can +run: + + paster datastore set-permissions | sudo -u postgres psql + +Or, if your PostgreSQL server is remote, you can pipe the permissions script +over SSH: + + paster datastore set-permissions | ssh dbserver sudo -u postgres psql + +*/ + +-- Most of the following commands apply to an explicit database or to the whole +-- 'public' schema, and could be executed anywhere. But ALTER DEFAULT +-- PERMISSIONS applies to the current database, and so we must be connected to +-- the datastore DB: +\connect "datastore" + +-- revoke permissions for the read-only user +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +REVOKE USAGE ON SCHEMA public FROM PUBLIC; + +GRANT CREATE ON SCHEMA public TO "ckan"; +GRANT USAGE ON SCHEMA public TO "ckan"; + +GRANT CREATE ON SCHEMA public TO "ckan"; +GRANT USAGE ON SCHEMA public TO "ckan"; + +-- take connect permissions from main db +REVOKE CONNECT ON DATABASE "ckan" FROM "datastore_ro"; + +-- grant select permissions for read-only user +GRANT CONNECT ON DATABASE "datastore" TO "datastore_ro"; +GRANT USAGE ON SCHEMA public TO "datastore_ro"; + +-- grant access to current tables and views to read-only user +GRANT SELECT ON ALL TABLES IN SCHEMA public TO "datastore_ro"; + +-- grant access to new tables and views by default +ALTER DEFAULT PRIVILEGES FOR USER "ckan" IN SCHEMA public + GRANT SELECT ON TABLES TO "datastore_ro"; + +CREATE OR REPLACE VIEW "_table_metadata" AS + SELECT DISTINCT + substr(md5(dependee.relname || COALESCE(dependent.relname, '')), 0, 17) AS "_id", + dependee.relname AS name, + dependee.oid AS oid, + dependent.relname AS alias_of + -- dependent.oid AS oid + FROM + pg_class AS dependee + LEFT OUTER JOIN pg_rewrite AS r ON r.ev_class = dependee.oid + LEFT OUTER JOIN pg_depend AS d ON d.objid = r.oid + LEFT OUTER JOIN pg_class AS dependent ON d.refobjid = dependent.oid + WHERE + (dependee.oid != dependent.oid OR dependent.oid IS NULL) AND + (dependee.relname IN (SELECT tablename FROM pg_catalog.pg_tables) + OR dependee.relname IN (SELECT viewname FROM pg_catalog.pg_views)) AND + dependee.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname='public') + ORDER BY dependee.oid DESC; +ALTER VIEW "_table_metadata" OWNER TO "ckan"; +GRANT SELECT ON "_table_metadata" TO "datastore_ro"; + From 3d106649834e744be191cac2a8ed8e996dcaabe0 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 13 Jul 2017 15:26:01 +0800 Subject: [PATCH 21/51] dockermentation first cut --- .../install-from-docker-compose.rst | 240 ++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 doc/maintaining/installing/install-from-docker-compose.rst diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst new file mode 100644 index 00000000000..cd8b8ca6062 --- /dev/null +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -0,0 +1,240 @@ +.. include:: /_substitutions.rst + +=================================== +Installing CKAN with docker-compose +=================================== + +This section describes how to install CKAN with docker-compose. The scenario shown here is one of +many possibile scenarios and environments in which CKAN can be used with Docker. + + +-------------- +1. Environment +-------------- +We will use a Ubuntu environment, so far this has been tested on: + +* Amazon AWS EC2 Ubuntu 14.04 LTS +* Ubuntu 16.04 LTS on a desktop + +Docker is installed system-wide following the official `Docker CE installation guidelines +`_. + +.. tip:: + + Using a cloud based VM, external storage volumes are cheaper than VMs and easy to backup. + In our use case, we use an EC2 with 16 GB storage, have mounted a 100 GB btrfs-formatted + external storage volume, and symlinked ``/var/lib/docker`` to the external volume. + This allows us to store the bulky and precious cargo -- Docker images, Docker data volumes + containing the CKAN databases, filestore, and config -- on a cheaper service. + On the other hand, a snapshotting filesystem like btrfs is ideal for backups, and it is easy to + maintain rolling backups. + The same cost consideration might apply to other cloud-based providers. + +To verify a successful Docker installation, run ``docker run hello-world``. +``docker version`` should output versions for client and server. + +----------------------------- +2. CKAN source and virtualenv +----------------------------- +In this step, we will create a virtualenv, clone the CKAN repository and install docker-compose. + +a. virtualenvwrapper + +While not essential, `virtualenvwrapper `_ +provides convenience methods to manage virtualenvs. + +If using virtulenvwrapper, append the following lines (using your preferred and existing locations +for virtualenvs and projects) to ``~/.bashrc``:: + + export WORKON_HOME=$HOME/.venvs + export PROJECT_HOME=/path/to/projects + source /usr/local/bin/virtualenvwrapper.sh + export PIP_VIRTUALENV_BASE=WORKON_HOME + +b. Create virtualenv for CKAN + +If using virtualenvwrapper, create a virtualenv for CKAN with ``mkproject ckan``, else follow +the virtulenv setup instructions in :doc:`install-from-source`. + +To verify a successful virtualenvwrapper setup for CKAN, check that ``workon ckan`` will jump you +into the CKAN project directory (``/path/to/projects/ckan`` in our example) +and activate the CKAN virtualenv (``~/.venvs/ckan``). + +c. Clone CKAN source + +Clone CKAN into the activated virtualenv:: + + workon ckan + git clone git@github.com:ckan/ckan.git . + +d. Install docker-compose + +In the activated virtualenv, install and verify docker-compose using pip:: + + workon ckan + pip install docker-compose + docker-compose version + +---------------------- +3. Build Docker images +---------------------- +In this step we will build the Docker images and create Docker data volumes with user-defined, +sensitive settings (e.g. for database passwords). + +a. Sensitive settings + +In a production environment, copy ``contrib/docker/.env.template`` to ``contrib/docker/.env`` +and follow instructions within to set passwords and other sensitive or user-defined variables. +The very unimaginative defaults will work fine in a development environment. + +b. Build the images + +With an activated virtualenv:: + + workon ckan + cd contrib/docker + docker-compose build + docker-compose up -d + +For the remainder of this chapter, we assume that docker-compose commands are all run inside +``contrib/docker`` with the ``ckan`` virtualenv activated. + +On first runs, the postgres container could need longer to initialise the database cluster than +the ckan container will wait for. In this case, simply restart the ckan container a few times:: + + docker-compose restart ckan + docker ps | grep ckan + docker-compose logs -f ckan + +After this step, CKAN should be running at ``CKAN_SITE_URL``. +There should be four containers running (``docker ps``) + +* ``ckan``: CKAN with standard extensions +* ``db``: CKAN's database, later also running CKAN's datastore database +* ``redis`` +* ``solr`` + +There should be four named Docker volumes (``docker volume ls | grep docker``). They will be +prefixed with the Docker project name (default: ``docker``) + +* ``docker_ckan_config``: home of ckan.ini +* ``docker_ckan_home``: home of ckan venv and source, later also additional CKAN extensions +* ``docker_ckan_storage``: home of CKAN's filestore (resource files) +* ``docker_pg_data``: home of the database files for CKAN's default and datapusher databases + +The location of these named volumes need to be backed up in a production environment. +To migrate CKAN data between different hosts, simply transfer the content of the named volumes. +A detailed use case of data transfer will be discussed later. + +------------------- +4. Enable Datastore +------------------- +To enable the datastore, the datastore database and database users have to be created before +enabling the datastore and datapusher settings in the ``ckan.ini``. + +a. Create and configure datastore database + +With running CKAN containers, execute two prepared scripts against the ``db`` container:: + + docker exec -it db psql -U ckan -f 00_create_datastore.sql + docker exec -it db psql -U ckan -f 10_set_permissions.sql + +The first script will create the datastore database and the datastore readonly user in the ``db`` +container. The second script is the output of ``paster ckan set-permissions``. +The effect of these scripts is persisted in the named volume ``docker_pg_data``. + +.. note:: We re-use the already privileged default user of the CKAN database as read/write user + for the datastore. The database user (``ckan``) is hard-coded, the password is supplied through + the``.env`` variable ``POSTGRES_PASSWORD``. + A new user ``datastore_ro`` is created (and also hard-coded) as readonly user with password + ``DATASTORE_READONLY_USER``. + Hard-coding the database table and usernames allows to prepare the set-permissions SQL script, + while not exposing sensitive information to the world outside the Docker host environment. + +After this step, the datastore database is ready to be enabled in the ``ckan.ini``. + +b. Enable datastore and datapusher in ``ckan.ini`` + +Find the path to the ``ckan.ini`` within the named volume:: + + docker volume inspect docker_ckan_config | grep Mountpoint + + # "Mountpoint": "/var/lib/docker/volumes/docker_ckan_config/_data", + +Edit the ``ckan.ini`` (note: requires sudo):: + + sudo vim /var/lib/docker/volumes/docker_ckan_config/_data/ckan.ini + + # With your own POSTGRES_PASSWORD and DATASTORE_READONLY_USER + ckan.datastore.write_url = postgresql://ckan:POSTGRES_PASSWORD@db/datastore + ckan.datastore.read_url = postgresql://datastore:DATASTORE_READONLY_PASSWORD@db/datastore + # add datastore and datapusher to plugins + # enable datapusher options + +Restart the ``ckan`` container to apply changes to the ``ckan.ini``:: + + docker-compose restart ckan + +------------------------- +5. Create CKAN admin user +------------------------- +With all four Docker images up and running, create the CKAN admin user (johndoe in this example):: + + docker exec -it ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin -c /etc/ckan/default/ckan.ini add johndoe + +Now you should be able to login to the new, empty CKAN. +The admin user's API key will be instrumental in tranferring data from other instances. + +----------------- +6. Migrating data +----------------- +This section is a stub. Pending testing, and presuming same dataset schema (ckanext-scheming) on +source and target CKANs, the process should be as simple as: + +* ``rsync`` contents of the filestore into the storage volume. +* Use python package ``ckanapi`` to dump orgs, groups and datasets from source CKAN instance, then + to load into new instance. +* Let datapusher populate datastore. +* Trigger a Solr index rebuild. + +-------------------- +7. Adding extensions +-------------------- +This section is a stub. + +There are two use cases how to add extensions: + +* Developers will want to have access to the extensions' source. +* Maintainers of production instances will want extensions to be part of the ``ckan`` image and an + easy way to enable them in the ``ckan.ini``. + +-------------------------------------------- +8. Changing and adding environment variables +-------------------------------------------- +This section is targetted at the CKAN developer seeking to factor out settings as new ``.env`` +variables. + +The flow of variable substitution is as follows: + +* ``.env.template`` holds the defaults and the usage instructions for variables. +* ``.env`` is copied and modified from ``.env.template`` with values chosen by the maintainer. +* docker-compose interpolates variables in ``docker-compose.yml`` from ``.env``. +* docker-compose can pass on these variables to the containers as build time variables + (when building the images) and / or as run time variables (when running the containers). +* ``ckan-entrypoint.sh`` has access to all run time variables of the ``ckan`` service. +* ``ckan-entrypoint.sh`` injects some variables (e.g. ``CKAN_SQLALCHEMY_URL``) into the running + ``ckan`` container, supplementing the CKAN config variables from ``ckan.ini``. + +After adding new, or changing the value of existing ``.env`` variables, locally built images and +volumes need to be dropped and rebuilt. +Otherwise, docker will re-use cached images with old or without variables:: + + docker-compose down + docker rmi $(docker images -f dangling=true -q) + docker volume rm $(docker volume ls -f dangling=true -q) + docker-compose build + docker-compose up -d + +.. warning:: Removing named volumes will destroy data. + Backup all data when doing this in a production setting. + From eb7d9e106d00e448c0dbf47991624a0389e74e3b Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 13 Jul 2017 16:03:12 +0800 Subject: [PATCH 22/51] update docs --- .../installing/install-from-docker-compose.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index cd8b8ca6062..7af099b7b20 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -165,7 +165,11 @@ Edit the ``ckan.ini`` (note: requires sudo):: sudo vim /var/lib/docker/volumes/docker_ckan_config/_data/ckan.ini - # With your own POSTGRES_PASSWORD and DATASTORE_READONLY_USER + # filestore + ckan.storage_path = /var/lib/ckan + + # datastore and datapusher + # With your own POSTGRES_PASSWORD and DATASTORE_READONLY_USER: ckan.datastore.write_url = postgresql://ckan:POSTGRES_PASSWORD@db/datastore ckan.datastore.read_url = postgresql://datastore:DATASTORE_READONLY_PASSWORD@db/datastore # add datastore and datapusher to plugins @@ -175,6 +179,13 @@ Restart the ``ckan`` container to apply changes to the ``ckan.ini``:: docker-compose restart ckan +Now the datastore API should return content when visiting:: + + CKAN_SITE_URL/api/3/action/datastore_search?resource_id=_table_metadata + +TODO datapusher setup - own service / Dockerfile? + + ------------------------- 5. Create CKAN admin user ------------------------- From f26bc981e2ad45249bd20144ecdf47dd4586d0f8 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 13 Jul 2017 16:55:27 +0800 Subject: [PATCH 23/51] add datapusher, send shoutout to clement the legend mouchet --- contrib/docker/docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 4c813253562..a36f6725ca2 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -37,6 +37,12 @@ services: - ckan_home:/usr/lib/ckan/default - ckan_storage:/var/lib/ckan + datapusher: + container_name: datapusher + image: clementmouchet/datapusher + ports: + - "0.0.0.0:8800:8800" + db: container_name: db build: From 59571f5fdca853e1947ee7c25a5b642b1503dac5 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 14 Jul 2017 11:45:51 +0800 Subject: [PATCH 24/51] factor out more config variables --- Dockerfile | 12 ++---- contrib/docker/.env.template | 20 ++++++--- contrib/docker/ckan-entrypoint.sh | 42 +++++++++++++------ contrib/docker/docker-compose.yml | 20 +++++---- contrib/docker/postgresql/Dockerfile | 1 + .../install-from-docker-compose.rst | 4 +- 6 files changed, 64 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1fedb5c042a..24b0beede44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ # docker build . -t ckan --build-arg CKAN_SITE_URL=http://localhost:5000 -# docker run -d -p 80:5000 --link db:db --link redis:redis --link solr:solr ckan +# docker run -d -p 80:5000 --link db:db --link redis:redis --link solr:solr ckan \ +# -v ckan_config:/etc.ckan/default -v ckan_storage:/var/lib/ckan FROM debian:jessie MAINTAINER Open Knowledge @@ -19,10 +20,9 @@ RUN apt-get -q -y update && apt-get -q -y upgrade && \ ENV CKAN_HOME /usr/lib/ckan ENV CKAN_VENV $CKAN_HOME/default ENV CKAN_CONFIG /etc/ckan/default -ENV CKAN_STORAGE_PATH /var/lib/ckan +ENV CKAN_STORAGE_PATH=/var/lib/ckan -# Build-time variables specified by docker-compose.yml / .env or -# docker build . -t ckan --build-arg CKAN_SITE_URL=http://localhost:5000 +# Build-time variables specified by docker-compose.yml / .env ARG CKAN_SITE_URL # Create ckan user @@ -45,10 +45,6 @@ RUN ckan-pip install --upgrade -r $CKAN_VENV/src/ckan/requirements.txt && \ ENTRYPOINT ["/ckan-entrypoint.sh"] -# Volumes -VOLUME ["/etc/ckan/default"] -VOLUME ["/var/lib/ckan"] - USER ckan EXPOSE 5000 diff --git a/contrib/docker/.env.template b/contrib/docker/.env.template index 645b2a7afdb..7329d1f5d19 100644 --- a/contrib/docker/.env.template +++ b/contrib/docker/.env.template @@ -7,26 +7,36 @@ # docker rmi $(docker images -f dangling=true -q) # docker-compose build # docker-compose up -d -# docker-compose restart (run a few times until ckan starts up nicely) +# docker-compose restart ckan # give the db service time to initialize the db cluster on first run # Image: ckan +CKAN_SITE_ID=default +# # On AWS, your CKAN_SITE_URL is the output of: # curl -s http://169.254.169.254/latest/meta-data/public-hostname CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com # CKAN_SITE_URL=http://localhost # -# CKAN_PORT must be available on the host: sudo netstat -na | grep 80 +# CKAN_PORT must be available on the host: sudo netstat -na # To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan CKAN_PORT=80 +# +# Email settings +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost # Image: db POSTGRES_PASSWORD=ckan # -# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 54 +# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 5432 # To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build POSTGRES_PORT=5432 # -# The datastore database will be created in the db container +# The datastore database will be created in the db container as docs # Readwrite user/pass will be ckan:POSTGRES_PASSWORD -# Readonly user/pass will be datastore:DATASTORE_READONLY_PASSWORD +# Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD DATASTORE_READONLY_PASSWORD=datastore + diff --git a/contrib/docker/ckan-entrypoint.sh b/contrib/docker/ckan-entrypoint.sh index 7b0fb698f06..40a997241e7 100644 --- a/contrib/docker/ckan-entrypoint.sh +++ b/contrib/docker/ckan-entrypoint.sh @@ -8,6 +8,8 @@ set -e : ${CKAN_SOLR_URL:=} # URL for redis (required unless linked to a container called 'redis') : ${CKAN_REDIS_URL:=} +# URL for datapusher (required unless linked to a container called 'datapusher') +: ${CKAN_DATAPUSHER_URL:=} CONFIG="${CKAN_CONFIG}/ckan.ini" @@ -17,15 +19,20 @@ abort () { } set_environment () { + export CKAN_SITE_ID=${CKAN_SITE_ID} + export CKAN_SITE_URL=${CKAN_SITE_URL} export CKAN_SQLALCHEMY_URL=${CKAN_SQLALCHEMY_URL} export CKAN_SOLR_URL=${CKAN_SOLR_URL} export CKAN_REDIS_URL=${CKAN_REDIS_URL} - export CKAN_STORAGE_PATH=${CKAN_STORAGE_PATH} - export CKAN_SITE_URL=${CKAN_SITE_URL} - #ckan.datastore.write_url = postgresql://ckan:ckan@db/datastore - #ckan.datastore.read_url = postgresql://datastore:datastore@db/datastore + export CKAN_STORAGE_PATH=/var/lib/ckan + export CKAN_DATAPUSHER_URL=${CKAN_DATAPUSHER_URL} export CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore export CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DS_RO_PASS}@db/datastore + export CKAN_SMTP_SERVER=${CKAN_SMTP_SERVER} + export CKAN_SMTP_STARTTLS=${CKAN_SMTP_STARTTLS} + export CKAN_SMTP_USER=${CKAN_SMTP_USER} + export CKAN_SMTP_PASSWORD=${CKAN_SMTP_PASSWORD} + export CKAN_SMTP_MAIL_FROM=${CKAN_SMTP_MAIL_FROM} } @@ -64,15 +71,19 @@ link_redis_url () { echo "redis://${host}:${port}/1" } +link_datapusher_url() { + local host=$DATAPUSHER_PORT_8800_ADDR + local port=$DATAPUSHER_PORT_8800_PORT + echo "http://${host}:${port}" + +} + # If we don't already have a config file, bootstrap if [ ! -e "$CONFIG" ]; then write_config fi -# Set environment variables -echo "SQL_ALCHEMY_URL is provided to entrypoint as $CKAN_SQLALCHEMY_URL" - - +# Get or create CKAN_SQLALCHEMY_URL if [ -z "$CKAN_SQLALCHEMY_URL" ]; then if ! CKAN_SQLALCHEMY_URL=$(link_postgres_url); then abort "ERROR: no CKAN_SQLALCHEMY_URL specified and linked container called 'db' was not found" @@ -83,10 +94,9 @@ if [ -z "$CKAN_SQLALCHEMY_URL" ]; then export PGDATABASE=${DB_ENV_POSTGRES_DB} export PGUSER=${DB_ENV_POSTGRES_USER} export PGPASSWORD=${DB_ENV_POSTGRES_PASSWORD} - echo "PG password is $PGPASSWORD" + echo "CKAN_SQLALCHEMY_URL: $CKAN_SQLALCHEMY_URL" - # wait for postgres db to be available, immediately after creation - # its entrypoint creates the cluster and dbs and this can take a moment + # Give the db container time to initialize the db cluster (if first run) for tries in $(seq 60); do psql -c 'SELECT 1;' 2> /dev/null && break sleep 0.3 @@ -96,13 +106,19 @@ fi if [ -z "$CKAN_SOLR_URL" ]; then if ! CKAN_SOLR_URL=$(link_solr_url); then - abort "ERROR: no CKAN_SOLR_URL specified and linked container called 'solr' was not found" + abort "ERROR: no CKAN_SOLR_URL specified and no linked container called 'solr' found" fi fi if [ -z "$CKAN_REDIS_URL" ]; then if ! CKAN_REDIS_URL=$(link_redis_url); then - abort "ERROR: no CKAN_REDIS_URL specified and linked container called 'redis' was not found" + abort "ERROR: no CKAN_REDIS_URL specified and no linked container called 'redis' found" + fi +fi + +if [ -z "$CKAN_DATAPUSHER_URL" ]; then + if ! CKAN_DATAPUSHER_URL=$(link_datapusher_url); then + abort "ERROR: no CKAN_DATAPUSHER_URL specified and no linked container called 'datapusher' found" fi fi diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index a36f6725ca2..797b187ed5d 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -23,15 +23,21 @@ services: - "0.0.0.0:${CKAN_PORT}:5000" environment: - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan + - CKAN_SOLR_URL=http://solr:8983/solr/ckan + - CKAN_REDIS_URL=redis://redis:6379/1 - CKAN_SITE_URL=${CKAN_SITE_URL} - - DB_PORT_5432_TCP_ADDR=db - - DB_PORT_5432_TCP_PORT=${POSTGRES_PORT} + - CKAN_DATAPUSHER_URL=http://datapusher:8800 - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - - SOLR_PORT_8983_TCP_ADDR=solr - - SOLR_PORT_8983_TCP_PORT=8983 - - REDIS_PORT_6379_TCP_ADDR=redis - - REDIS_PORT_6379_TCP_PORT=6379 + #- DB_PORT_5432_TCP_ADDR=db + #- DB_PORT_5432_TCP_PORT=${POSTGRES_PORT} + #- SOLR_PORT_8983_TCP_ADDR=solr + #- SOLR_PORT_8983_TCP_PORT=8983 + #- REDIS_PORT_6379_TCP_ADDR=redis + #- REDIS_PORT_6379_TCP_PORT=6379 + #- DATAPUSHER_PORT_8800_ADDR=datapusher + #- DATAPUSHER_PORT_8800_PORT=8800 + volumes: - ckan_config:/etc/ckan/default - ckan_home:/usr/lib/ckan/default @@ -41,7 +47,7 @@ services: container_name: datapusher image: clementmouchet/datapusher ports: - - "0.0.0.0:8800:8800" + - "8800:8800" db: container_name: db diff --git a/contrib/docker/postgresql/Dockerfile b/contrib/docker/postgresql/Dockerfile index 5e28c855431..bd2c802f230 100644 --- a/contrib/docker/postgresql/Dockerfile +++ b/contrib/docker/postgresql/Dockerfile @@ -10,5 +10,6 @@ ENV POSTGRES_USER ckan ARG POSTGRES_PASSWORD ARG DS_RO_PASS +# Include datastore setup scripts ADD /docker-entrypoint-initdb.d/00_create_datastore.sql ./00_create_datastore.sql ADD /docker-entrypoint-initdb.d/10_set_permissions.sql ./10_set_permissions.sql diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 7af099b7b20..eff03182b39 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -81,11 +81,11 @@ In the activated virtualenv, install and verify docker-compose using pip:: In this step we will build the Docker images and create Docker data volumes with user-defined, sensitive settings (e.g. for database passwords). -a. Sensitive settings +a. Sensitive settings and environment variables In a production environment, copy ``contrib/docker/.env.template`` to ``contrib/docker/.env`` and follow instructions within to set passwords and other sensitive or user-defined variables. -The very unimaginative defaults will work fine in a development environment. +The defaults will work fine in a development environment. b. Build the images From 7d01ead6d19a1a25aea525c0e70bbd84a8b605e2 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 14 Jul 2017 13:50:32 +0800 Subject: [PATCH 25/51] working vanilla ckan through docker-compose --- Dockerfile | 6 +++--- contrib/docker/docker-compose.yml | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 24b0beede44..0a996b2b276 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,8 @@ RUN apt-get -q -y update && apt-get -q -y upgrade && \ # Define environment variables ENV CKAN_HOME /usr/lib/ckan -ENV CKAN_VENV $CKAN_HOME/default -ENV CKAN_CONFIG /etc/ckan/default +ENV CKAN_VENV $CKAN_HOME/venv +ENV CKAN_CONFIG /etc/ckan ENV CKAN_STORAGE_PATH=/var/lib/ckan # Build-time variables specified by docker-compose.yml / .env @@ -48,5 +48,5 @@ ENTRYPOINT ["/ckan-entrypoint.sh"] USER ckan EXPOSE 5000 -CMD ["ckan-paster","serve","/etc/ckan/default/ckan.ini"] +CMD ["ckan-paster","serve","/etc/ckan/ckan.ini"] diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 797b187ed5d..2d1964012da 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -22,13 +22,15 @@ services: ports: - "0.0.0.0:${CKAN_PORT}:5000" environment: + # Using linked services: - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan - CKAN_SOLR_URL=http://solr:8983/solr/ckan - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_SITE_URL=${CKAN_SITE_URL} - CKAN_DATAPUSHER_URL=http://datapusher:8800 + - CKAN_SITE_URL=${CKAN_SITE_URL} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + # Ingredients to interpolate service _URL #- DB_PORT_5432_TCP_ADDR=db #- DB_PORT_5432_TCP_PORT=${POSTGRES_PORT} #- SOLR_PORT_8983_TCP_ADDR=solr @@ -39,8 +41,8 @@ services: #- DATAPUSHER_PORT_8800_PORT=8800 volumes: - - ckan_config:/etc/ckan/default - - ckan_home:/usr/lib/ckan/default + - ckan_config:/etc/ckan + - ckan_home:/usr/lib/ckan - ckan_storage:/var/lib/ckan datapusher: From 270ae97aa734a6db904361dd80a53a66bef21051 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 14 Jul 2017 15:40:11 +0800 Subject: [PATCH 26/51] first working extension, updated docs --- contrib/docker/.env.template | 9 +- contrib/docker/ckan-entrypoint.sh | 77 +---------- contrib/docker/docker-compose.yml | 15 +-- .../install-from-docker-compose.rst | 124 +++++++++++++----- 4 files changed, 109 insertions(+), 116 deletions(-) diff --git a/contrib/docker/.env.template b/contrib/docker/.env.template index 7329d1f5d19..1788f4c82fa 100644 --- a/contrib/docker/.env.template +++ b/contrib/docker/.env.template @@ -14,12 +14,13 @@ CKAN_SITE_ID=default # # On AWS, your CKAN_SITE_URL is the output of: # curl -s http://169.254.169.254/latest/meta-data/public-hostname -CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com -# CKAN_SITE_URL=http://localhost +# CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com +# When running locally, CKAN_SITE_URL must contain the port +CKAN_SITE_URL=http://localhost:5000 # # CKAN_PORT must be available on the host: sudo netstat -na # To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan -CKAN_PORT=80 +CKAN_PORT=5000 # # Email settings CKAN_SMTP_SERVER=smtp.corporateict.domain:25 @@ -27,7 +28,7 @@ CKAN_SMTP_STARTTLS=True CKAN_SMTP_USER=user CKAN_SMTP_PASSWORD=pass CKAN_SMTP_MAIL_FROM=ckan@localhost - +# # Image: db POSTGRES_PASSWORD=ckan # diff --git a/contrib/docker/ckan-entrypoint.sh b/contrib/docker/ckan-entrypoint.sh index 40a997241e7..6fe3fd2facc 100644 --- a/contrib/docker/ckan-entrypoint.sh +++ b/contrib/docker/ckan-entrypoint.sh @@ -26,8 +26,8 @@ set_environment () { export CKAN_REDIS_URL=${CKAN_REDIS_URL} export CKAN_STORAGE_PATH=/var/lib/ckan export CKAN_DATAPUSHER_URL=${CKAN_DATAPUSHER_URL} - export CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore - export CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DS_RO_PASS}@db/datastore + export CKAN_DATASTORE_WRITE_URL=${CKAN_DATASTORE_WRITE_URL} + export CKAN_DATASTORE_READ_URL=${CKAN_DATASTORE_READ_URL} export CKAN_SMTP_SERVER=${CKAN_SMTP_SERVER} export CKAN_SMTP_STARTTLS=${CKAN_SMTP_STARTTLS} export CKAN_SMTP_USER=${CKAN_SMTP_USER} @@ -35,47 +35,8 @@ set_environment () { export CKAN_SMTP_MAIL_FROM=${CKAN_SMTP_MAIL_FROM} } - write_config () { - # Note that this only gets called if there is no config, see below! ckan-paster make-config --no-interactive ckan "$CONFIG" - - # The variables above will be used by CKAN, but - # in case want to use the config from ckan.ini use this - #ckan-paster --plugin=ckan config-tool "$CONFIG" -e \ - # "sqlalchemy.url = ${CKAN_SQLALCHEMY_URL}" \ - # "solr_url = ${CKAN_SOLR_URL}" \ - # "ckan.redis.url = ${CKAN_REDIS_URL}" \ - # "ckan.storage_path = ${CKAN_STORAGE_PATH}" \ - # "ckan.site_url = ${CKAN_SITE_URL}" -} - -link_postgres_url () { - local user=$DB_ENV_POSTGRES_USER - local pass=$DB_ENV_POSTGRES_PASSWORD - local db=$DB_ENV_POSTGRES_DB - local host=$DB_PORT_5432_TCP_ADDR - local port=$DB_PORT_5432_TCP_PORT - echo "postgresql://${user}:${pass}@${host}:${port}/${db}" -} - -link_solr_url () { - local host=$SOLR_PORT_8983_TCP_ADDR - local port=$SOLR_PORT_8983_TCP_PORT - echo "http://${host}:${port}/solr/ckan" -} - -link_redis_url () { - local host=$REDIS_PORT_6379_TCP_ADDR - local port=$REDIS_PORT_6379_TCP_PORT - echo "redis://${host}:${port}/1" -} - -link_datapusher_url() { - local host=$DATAPUSHER_PORT_8800_ADDR - local port=$DATAPUSHER_PORT_8800_PORT - echo "http://${host}:${port}" - } # If we don't already have a config file, bootstrap @@ -85,46 +46,22 @@ fi # Get or create CKAN_SQLALCHEMY_URL if [ -z "$CKAN_SQLALCHEMY_URL" ]; then - if ! CKAN_SQLALCHEMY_URL=$(link_postgres_url); then - abort "ERROR: no CKAN_SQLALCHEMY_URL specified and linked container called 'db' was not found" - else - #If that worked, use the DB details to wait for the DB - export PGHOST=${DB_PORT_5432_TCP_ADDR} - export PGPORT=${DB_PORT_5432_TCP_PORT} - export PGDATABASE=${DB_ENV_POSTGRES_DB} - export PGUSER=${DB_ENV_POSTGRES_USER} - export PGPASSWORD=${DB_ENV_POSTGRES_PASSWORD} - echo "CKAN_SQLALCHEMY_URL: $CKAN_SQLALCHEMY_URL" - - # Give the db container time to initialize the db cluster (if first run) - for tries in $(seq 60); do - psql -c 'SELECT 1;' 2> /dev/null && break - sleep 0.3 - done - fi + abort "ERROR: no CKAN_SQLALCHEMY_URL specified in docker-compose.yml" fi if [ -z "$CKAN_SOLR_URL" ]; then - if ! CKAN_SOLR_URL=$(link_solr_url); then - abort "ERROR: no CKAN_SOLR_URL specified and no linked container called 'solr' found" - fi + abort "ERROR: no CKAN_SOLR_URL specified in docker-compose.yml" fi if [ -z "$CKAN_REDIS_URL" ]; then - if ! CKAN_REDIS_URL=$(link_redis_url); then - abort "ERROR: no CKAN_REDIS_URL specified and no linked container called 'redis' found" - fi + abort "ERROR: no CKAN_REDIS_URL specified in docker-compose.yml" fi if [ -z "$CKAN_DATAPUSHER_URL" ]; then - if ! CKAN_DATAPUSHER_URL=$(link_datapusher_url); then - abort "ERROR: no CKAN_DATAPUSHER_URL specified and no linked container called 'datapusher' found" - fi + abort "ERROR: no CKAN_DATAPUSHER_URL specified in docker-compose.yml" fi set_environment - -# Initializes the Database ckan-paster --plugin=ckan db init -c "${CKAN_CONFIG}/ckan.ini" - exec "$@" + diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 2d1964012da..6fef3f5ac8f 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -14,7 +14,7 @@ services: build: context: ../../ args: - - CKAN_SITE_URL=${CKAN_SITE_URL}:${CKAN_PORT} + - CKAN_SITE_URL=${CKAN_SITE_URL} links: - db - solr @@ -22,23 +22,16 @@ services: ports: - "0.0.0.0:${CKAN_PORT}:5000" environment: - # Using linked services: + # Defaults work with linked containers, change to use own Postgres, SolR, Redis or Datapusher - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan + - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore + - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore - CKAN_SOLR_URL=http://solr:8983/solr/ckan - CKAN_REDIS_URL=redis://redis:6379/1 - CKAN_DATAPUSHER_URL=http://datapusher:8800 - CKAN_SITE_URL=${CKAN_SITE_URL} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - # Ingredients to interpolate service _URL - #- DB_PORT_5432_TCP_ADDR=db - #- DB_PORT_5432_TCP_PORT=${POSTGRES_PORT} - #- SOLR_PORT_8983_TCP_ADDR=solr - #- SOLR_PORT_8983_TCP_PORT=8983 - #- REDIS_PORT_6379_TCP_ADDR=redis - #- REDIS_PORT_6379_TCP_PORT=6379 - #- DATAPUSHER_PORT_8800_ADDR=datapusher - #- DATAPUSHER_PORT_8800_PORT=8800 volumes: - ckan_config:/etc/ckan diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index eff03182b39..b36d68095a8 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -75,6 +75,8 @@ In the activated virtualenv, install and verify docker-compose using pip:: pip install docker-compose docker-compose version +Step c and d are interchangeable. + ---------------------- 3. Build Docker images ---------------------- @@ -107,12 +109,13 @@ the ckan container will wait for. In this case, simply restart the ckan containe docker-compose logs -f ckan After this step, CKAN should be running at ``CKAN_SITE_URL``. -There should be four containers running (``docker ps``) +There should be five containers running (``docker ps``): * ``ckan``: CKAN with standard extensions * ``db``: CKAN's database, later also running CKAN's datastore database -* ``redis`` -* ``solr`` +* ``redis``: A pre-built Redis image. +* ``solr``: A pre-built SolR image set up for CKAN. +* ``datapusher``: A pre-built CKAN Datapusher image. There should be four named Docker volumes (``docker volume ls | grep docker``). They will be prefixed with the Docker project name (default: ``docker``) @@ -120,7 +123,7 @@ prefixed with the Docker project name (default: ``docker``) * ``docker_ckan_config``: home of ckan.ini * ``docker_ckan_home``: home of ckan venv and source, later also additional CKAN extensions * ``docker_ckan_storage``: home of CKAN's filestore (resource files) -* ``docker_pg_data``: home of the database files for CKAN's default and datapusher databases +* ``docker_pg_data``: home of the database files for CKAN's default and datastore databases The location of these named volumes need to be backed up in a production environment. To migrate CKAN data between different hosts, simply transfer the content of the named volumes. @@ -134,7 +137,7 @@ enabling the datastore and datapusher settings in the ``ckan.ini``. a. Create and configure datastore database -With running CKAN containers, execute two prepared scripts against the ``db`` container:: +With running CKAN containers, execute the built-in setup scripts against the ``db`` container:: docker exec -it db psql -U ckan -f 00_create_datastore.sql docker exec -it db psql -U ckan -f 10_set_permissions.sql @@ -165,15 +168,18 @@ Edit the ``ckan.ini`` (note: requires sudo):: sudo vim /var/lib/docker/volumes/docker_ckan_config/_data/ckan.ini - # filestore - ckan.storage_path = /var/lib/ckan +Add ``datastore datapusher`` to ``ckan.plugins`` and enable the datapusher option +``ckan.datapusher.formats``. + +The remaining settings required for datastore and datapusher are already taken care of: + +* ``ckan.storage_path`` (``/var/lib/ckan``) is hard-coded in ``ckan-entrypoint.sh``, + ``docker-compose.yml`` and CKAN's ``Dockerfile``. This path is hard-coded as it remains internal + to the containers, and changing it would have no effect on the host system. +* ``ckan.datastore.write_url = postgresql://ckan:POSTGRES_PASSWORD@db/datastore`` and + ``ckan.datastore.read_url = postgresql://datastore:DATASTORE_READONLY_PASSWORD@db/datastore`` + are provided by ``docker-compose.yml``. - # datastore and datapusher - # With your own POSTGRES_PASSWORD and DATASTORE_READONLY_USER: - ckan.datastore.write_url = postgresql://ckan:POSTGRES_PASSWORD@db/datastore - ckan.datastore.read_url = postgresql://datastore:DATASTORE_READONLY_PASSWORD@db/datastore - # add datastore and datapusher to plugins - # enable datapusher options Restart the ``ckan`` container to apply changes to the ``ckan.ini``:: @@ -183,47 +189,94 @@ Now the datastore API should return content when visiting:: CKAN_SITE_URL/api/3/action/datastore_search?resource_id=_table_metadata -TODO datapusher setup - own service / Dockerfile? - - ------------------------- 5. Create CKAN admin user ------------------------- With all four Docker images up and running, create the CKAN admin user (johndoe in this example):: - docker exec -it ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin -c /etc/ckan/default/ckan.ini add johndoe + docker exec -it ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin -c /etc/ckan/ckan.ini add johndoe Now you should be able to login to the new, empty CKAN. +This step is interchangeable with step 4. The admin user's API key will be instrumental in tranferring data from other instances. ----------------- 6. Migrating data ----------------- -This section is a stub. Pending testing, and presuming same dataset schema (ckanext-scheming) on +.. todo:: This section is a stub. + +Pending testing, and presuming same dataset schema (ckanext-scheming) on source and target CKANs, the process should be as simple as: * ``rsync`` contents of the filestore into the storage volume. -* Use python package ``ckanapi`` to dump orgs, groups and datasets from source CKAN instance, then - to load into new instance. +* Use python package ``ckanapi`` to dump orgs, groups and datasets from source CKAN instance, + transfer files, then use ``ckanapi`` to load the exported data into the target instance. * Let datapusher populate datastore. * Trigger a Solr index rebuild. -------------------- 7. Adding extensions -------------------- -This section is a stub. - There are two use cases how to add extensions: * Developers will want to have access to the extensions' source. * Maintainers of production instances will want extensions to be part of the ``ckan`` image and an easy way to enable them in the ``ckan.ini``. +For developers, the process is: + +* Run a bash shell inside the running ``ckan`` image, download and install extension. +* Edit ``ckan.ini`` +* Restart ``ckan`` service, read logs. + +a. Download and install extension inside ``ckan`` image +The process is very similar to installing extensions in a source install. The only difference is +that the installation steps happen inside the running container, using the virtualenv created +inside the ckan image by CKAN's Dockerfile, which is different from the virtualenv we have created +on the host machine in step 2. + +The downloaded and installed files will be persisted in the named volume ``docker_ckan_home``. + +In this example we'll install `ckanext-geoview `_. + +:: + # Enter the running ckan container: + docker exec -it ckan bash + + # Inside the running container, activate the virtualenv + source $CKAN_VENV/bin/activate + + # Download source + cd $CKAN_VENV/src/ + git clone https://github.com/ckan/ckanext-geoview.git + cd ckanext-geoview + + # Install extension's requirements + pip install -r pip-requirements.txt + + # Install extension + python setup.py install + python setup.py develop + + # exit the ckan container: + exit + +b. Modify CKAN config +Follow the extension's instructions to set CKAN config variables:: + + sudo vim /var/lib/docker/volumes/docker_ckan_config/_data/ckan.ini + +c. Reload and debug +:: + + docker-compose restart ckan + docker-compose logs ckan + -------------------------------------------- 8. Changing and adding environment variables -------------------------------------------- -This section is targetted at the CKAN developer seeking to factor out settings as new ``.env`` -variables. +This section is targetted at CKAN maintainers seeking a deeper understanding of variables, +and at CKAN developers seeking to factor out settings as new ``.env`` variables. The flow of variable substitution is as follows: @@ -233,18 +286,27 @@ The flow of variable substitution is as follows: * docker-compose can pass on these variables to the containers as build time variables (when building the images) and / or as run time variables (when running the containers). * ``ckan-entrypoint.sh`` has access to all run time variables of the ``ckan`` service. -* ``ckan-entrypoint.sh`` injects some variables (e.g. ``CKAN_SQLALCHEMY_URL``) into the running - ``ckan`` container, supplementing the CKAN config variables from ``ckan.ini``. +* ``ckan-entrypoint.sh`` injects environment variables (e.g. ``CKAN_SQLALCHEMY_URL``) into the + running ``ckan`` container, overriding the CKAN config variables from ``ckan.ini``. + +See :doc:`configuration` for a list of environment variables (e.g. ``CKAN_SQLALCHEMY_URL``) which +CKAN will accept to override ``ckan.ini``. -After adding new, or changing the value of existing ``.env`` variables, locally built images and -volumes need to be dropped and rebuilt. -Otherwise, docker will re-use cached images with old or without variables:: +After adding new or changing existing ``.env`` variables, locally built images and volumes may +need to be dropped and rebuilt. Otherwise, docker will re-use cached images with old or without +variables:: docker-compose down + docker-compose --build up -d + + # if that didn't work, try: + docker rmi $(docker images -f dangling=true -q) + docker-compose --build up -d + + # if that didn't work, try: docker rmi $(docker images -f dangling=true -q) docker volume rm $(docker volume ls -f dangling=true -q) - docker-compose build - docker-compose up -d + docker-compose --build up -d .. warning:: Removing named volumes will destroy data. Backup all data when doing this in a production setting. From bdd1faca782ad6bd1f62a4560ff7afd67476cc39 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 14 Jul 2017 16:25:08 +0800 Subject: [PATCH 27/51] spellcheck docs --- doc/maintaining/installing/install-from-docker-compose.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index b36d68095a8..995eab907a0 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -101,7 +101,7 @@ With an activated virtualenv:: For the remainder of this chapter, we assume that docker-compose commands are all run inside ``contrib/docker`` with the ``ckan`` virtualenv activated. -On first runs, the postgres container could need longer to initialise the database cluster than +On first runs, the postgres container could need longer to initialize the database cluster than the ckan container will wait for. In this case, simply restart the ckan container a few times:: docker-compose restart ckan @@ -203,7 +203,7 @@ The admin user's API key will be instrumental in tranferring data from other ins ----------------- 6. Migrating data ----------------- -.. todo:: This section is a stub. +.. todo:: This section requires testing. Pending testing, and presuming same dataset schema (ckanext-scheming) on source and target CKANs, the process should be as simple as: @@ -221,7 +221,8 @@ There are two use cases how to add extensions: * Developers will want to have access to the extensions' source. * Maintainers of production instances will want extensions to be part of the ``ckan`` image and an - easy way to enable them in the ``ckan.ini``. + easy way to enable them in the ``ckan.ini``. This requires customizing CKAN's ``Dockerfile`` and + scripted post-processing of the ``ckan.ini``. For developers, the process is: From 852f935e36d8553478980e99d28230359617760b Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 14 Jul 2017 16:38:09 +0800 Subject: [PATCH 28/51] fix typo in docs --- doc/maintaining/installing/install-from-docker-compose.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 995eab907a0..1f559f0a77c 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -238,9 +238,8 @@ on the host machine in step 2. The downloaded and installed files will be persisted in the named volume ``docker_ckan_home``. -In this example we'll install `ckanext-geoview `_. +In this example we'll install `ckanext-geoview `_:: -:: # Enter the running ckan container: docker exec -it ckan bash From 8d198999ce87adebafce3b0f784dfdeeb49f7bb5 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 14 Jul 2017 16:40:06 +0800 Subject: [PATCH 29/51] fix broken crossref in docs --- doc/maintaining/installing/install-from-docker-compose.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 1f559f0a77c..13655eff42f 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -289,8 +289,8 @@ The flow of variable substitution is as follows: * ``ckan-entrypoint.sh`` injects environment variables (e.g. ``CKAN_SQLALCHEMY_URL``) into the running ``ckan`` container, overriding the CKAN config variables from ``ckan.ini``. -See :doc:`configuration` for a list of environment variables (e.g. ``CKAN_SQLALCHEMY_URL``) which -CKAN will accept to override ``ckan.ini``. +See :doc:`/maintaining/configuration` for a list of environment variables +(e.g. ``CKAN_SQLALCHEMY_URL``) which CKAN will accept to override ``ckan.ini``. After adding new or changing existing ``.env`` variables, locally built images and volumes may need to be dropped and rebuilt. Otherwise, docker will re-use cached images with old or without From 32c976adc687c1cc1046940c2b2865962a29928f Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Mon, 17 Jul 2017 12:55:05 +0800 Subject: [PATCH 30/51] update docker compose installation docs following feedback from @vanuan remove use of virtualenv discuss environment variables vs docker secrets combine docker-compose build and up into up --build discuss restarting ckan container discuss named volumes vs mapped local drives add todo on automating adding extensions --- .../install-from-docker-compose.rst | 156 +++++++++--------- 1 file changed, 79 insertions(+), 77 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 13655eff42f..a2998d45e63 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -4,9 +4,13 @@ Installing CKAN with docker-compose =================================== -This section describes how to install CKAN with docker-compose. The scenario shown here is one of +This chapter describes how to install CKAN with docker-compose. The scenario shown here is one of many possibile scenarios and environments in which CKAN can be used with Docker. +This chapter aims to provide a simple, yet fully customizable deployment - easier to configure than +a source install, more customizable than a package install. + +.. note:: Some design decisions are opinionated, and there are alternatives. -------------- 1. Environment @@ -16,69 +20,45 @@ We will use a Ubuntu environment, so far this has been tested on: * Amazon AWS EC2 Ubuntu 14.04 LTS * Ubuntu 16.04 LTS on a desktop -Docker is installed system-wide following the official `Docker CE installation guidelines -`_. - -.. tip:: - - Using a cloud based VM, external storage volumes are cheaper than VMs and easy to backup. - In our use case, we use an EC2 with 16 GB storage, have mounted a 100 GB btrfs-formatted - external storage volume, and symlinked ``/var/lib/docker`` to the external volume. - This allows us to store the bulky and precious cargo -- Docker images, Docker data volumes - containing the CKAN databases, filestore, and config -- on a cheaper service. - On the other hand, a snapshotting filesystem like btrfs is ideal for backups, and it is easy to - maintain rolling backups. - The same cost consideration might apply to other cloud-based providers. - -To verify a successful Docker installation, run ``docker run hello-world``. -``docker version`` should output versions for client and server. +a. Storage ------------------------------ -2. CKAN source and virtualenv ------------------------------ -In this step, we will create a virtualenv, clone the CKAN repository and install docker-compose. +Using a cloud based VM, external storage volumes are cheaper than VMs and easy to backup. +In our use case, we use an AWS EC2 with 16 GB storage, have mounted a 100 GB btrfs-formatted +external storage volume, and symlinked ``/var/lib/docker`` to the external volume. +This allows us to store the bulky and precious cargo -- Docker images, Docker data volumes +containing the CKAN databases, filestore, and config -- on a cheaper service. +On the other hand, a snapshotting filesystem like btrfs is ideal for rolling backups. +The same cost consideration might apply to other cloud-based providers. -a. virtualenvwrapper +.. note:: This setup stores data in named volumes, mapped to folder locations which can be + network or local storage. + An alternative would be to re-write ``docker-compose.yml`` to map local storage + directly, bypassing named volumes. Both solutions will save data to a specified location. -While not essential, `virtualenvwrapper `_ -provides convenience methods to manage virtualenvs. +b. Docker -If using virtulenvwrapper, append the following lines (using your preferred and existing locations -for virtualenvs and projects) to ``~/.bashrc``:: +Docker is installed system-wide following the official `Docker CE installation guidelines +`_. - export WORKON_HOME=$HOME/.venvs - export PROJECT_HOME=/path/to/projects - source /usr/local/bin/virtualenvwrapper.sh - export PIP_VIRTUALENV_BASE=WORKON_HOME +To verify a successful Docker installation, run ``docker run hello-world``. +``docker version`` should output versions for client and server. -b. Create virtualenv for CKAN +c. Docker Compose -If using virtualenvwrapper, create a virtualenv for CKAN with ``mkproject ckan``, else follow -the virtulenv setup instructions in :doc:`install-from-source`. +Docker Compose is installed system-wide following the official `Docker Compose installation +guidelines `_. -To verify a successful virtualenvwrapper setup for CKAN, check that ``workon ckan`` will jump you -into the CKAN project directory (``/path/to/projects/ckan`` in our example) -and activate the CKAN virtualenv (``~/.venvs/ckan``). +To verify a successful Docker Compose installation, run ``docker-compose version``. -c. Clone CKAN source +d. CKAN source -Clone CKAN into the activated virtualenv:: +Clone CKAN into a directory of your choice:: - workon ckan + cd /path/to/my/projects git clone git@github.com:ckan/ckan.git . -d. Install docker-compose - -In the activated virtualenv, install and verify docker-compose using pip:: - - workon ckan - pip install docker-compose - docker-compose version - -Step c and d are interchangeable. - ---------------------- -3. Build Docker images +2. Build Docker images ---------------------- In this step we will build the Docker images and create Docker data volumes with user-defined, sensitive settings (e.g. for database passwords). @@ -91,24 +71,31 @@ The defaults will work fine in a development environment. b. Build the images -With an activated virtualenv:: +Inside the CKAN directory:: - workon ckan cd contrib/docker - docker-compose build - docker-compose up -d + docker-compose up -d --build For the remainder of this chapter, we assume that docker-compose commands are all run inside -``contrib/docker`` with the ``ckan`` virtualenv activated. +``contrib/docker``. On first runs, the postgres container could need longer to initialize the database cluster than -the ckan container will wait for. In this case, simply restart the ckan container a few times:: +the ckan container will wait for. This time span depends heavily on available system resources. +If the CKAN logs show problems connecting to the database, restart the ckan container a few times:: docker-compose restart ckan docker ps | grep ckan docker-compose logs -f ckan +.. note:: Earlier versions of ``ckan-entrypoint.sh`` used to wait and ping the db container + using detailed db credentials (host, port, user, password). + While this delay sometimes worked, it also obfuscated other possible problems. + However, the db cluster needs to initialize only once, and starts up quickly in subsequent + runs. This setup chose the very opinionated option of doing away with the delay altogether in + favour of failing early. + After this step, CKAN should be running at ``CKAN_SITE_URL``. + There should be five containers running (``docker ps``): * ``ckan``: CKAN with standard extensions @@ -125,13 +112,13 @@ prefixed with the Docker project name (default: ``docker``) * ``docker_ckan_storage``: home of CKAN's filestore (resource files) * ``docker_pg_data``: home of the database files for CKAN's default and datastore databases -The location of these named volumes need to be backed up in a production environment. +The location of these named volumes needs to be backed up in a production environment. To migrate CKAN data between different hosts, simply transfer the content of the named volumes. -A detailed use case of data transfer will be discussed later. +A detailed use case of data transfer will be discussed in step 5. -------------------- -4. Enable Datastore -------------------- +--------------------------- +3. Datastore and datapusher +--------------------------- To enable the datastore, the datastore database and database users have to be created before enabling the datastore and datapusher settings in the ``ckan.ini``. @@ -190,19 +177,18 @@ Now the datastore API should return content when visiting:: CKAN_SITE_URL/api/3/action/datastore_search?resource_id=_table_metadata ------------------------- -5. Create CKAN admin user +4. Create CKAN admin user ------------------------- With all four Docker images up and running, create the CKAN admin user (johndoe in this example):: docker exec -it ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin -c /etc/ckan/ckan.ini add johndoe Now you should be able to login to the new, empty CKAN. -This step is interchangeable with step 4. The admin user's API key will be instrumental in tranferring data from other instances. ------------------ -6. Migrating data ------------------ +--------------- +5. Migrate data +--------------- .. todo:: This section requires testing. Pending testing, and presuming same dataset schema (ckanext-scheming) on @@ -214,9 +200,9 @@ source and target CKANs, the process should be as simple as: * Let datapusher populate datastore. * Trigger a Solr index rebuild. --------------------- -7. Adding extensions --------------------- +----------------- +6. Add extensions +----------------- There are two use cases how to add extensions: * Developers will want to have access to the extensions' source. @@ -231,6 +217,7 @@ For developers, the process is: * Restart ``ckan`` service, read logs. a. Download and install extension inside ``ckan`` image + The process is very similar to installing extensions in a source install. The only difference is that the installation steps happen inside the running container, using the virtualenv created inside the ckan image by CKAN's Dockerfile, which is different from the virtualenv we have created @@ -261,21 +248,36 @@ In this example we'll install `ckanext-geoview `_. +This section illustrates the use of evironment variables. + +This section is targeted at CKAN maintainers seeking a deeper understanding of variables, and at CKAN developers seeking to factor out settings as new ``.env`` variables. The flow of variable substitution is as follows: @@ -297,16 +299,16 @@ need to be dropped and rebuilt. Otherwise, docker will re-use cached images with variables:: docker-compose down - docker-compose --build up -d + docker-compose up -d --build # if that didn't work, try: docker rmi $(docker images -f dangling=true -q) - docker-compose --build up -d + docker-compose up -d --build # if that didn't work, try: docker rmi $(docker images -f dangling=true -q) docker volume rm $(docker volume ls -f dangling=true -q) - docker-compose --build up -d + docker-compose up -d --build .. warning:: Removing named volumes will destroy data. Backup all data when doing this in a production setting. From 6f7869fea10e9850cfa54557cc64dae73fcd1a38 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Mon, 17 Jul 2017 16:38:25 +0800 Subject: [PATCH 31/51] prepare docker-compose for ckanext-spatial modify CKAN Dockerfile to install system packages required for ckanext-spatial modify postgres Dockerfile to use postgis image add postgres Dockerfile script to set permissions for postgis tables to ckan user add working installation steps to docs --- Dockerfile | 4 ++++ contrib/docker/postgresql/Dockerfile | 5 +++- .../20_postgis_permissions.sql | 3 +++ .../install-from-docker-compose.rst | 23 +++++++++++++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 contrib/docker/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql diff --git a/Dockerfile b/Dockerfile index 0a996b2b276..153908d75d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,10 @@ RUN apt-get -q -y update && apt-get -q -y upgrade && \ libpq-dev \ git-core \ postgresql-client \ + python-dev \ + libxml2-dev \ + libxslt-dev \ + libgeos-dev \ && apt-get -q clean # Define environment variables diff --git a/contrib/docker/postgresql/Dockerfile b/contrib/docker/postgresql/Dockerfile index bd2c802f230..8c8cef90a29 100644 --- a/contrib/docker/postgresql/Dockerfile +++ b/contrib/docker/postgresql/Dockerfile @@ -1,4 +1,5 @@ -FROM postgres:9.6 +#FROM postgres:9.6 +FROM mdillon/postgis MAINTAINER Open Knowledge # Allow connections; we don't map out any ports so only linked docker containers can connect @@ -13,3 +14,5 @@ ARG DS_RO_PASS # Include datastore setup scripts ADD /docker-entrypoint-initdb.d/00_create_datastore.sql ./00_create_datastore.sql ADD /docker-entrypoint-initdb.d/10_set_permissions.sql ./10_set_permissions.sql +ADD /docker-entrypoint-initdb.d/20_postgis_permissions.sql ./20_postgis_permissions.sql + diff --git a/contrib/docker/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql b/contrib/docker/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql new file mode 100644 index 00000000000..0eb2f85b05c --- /dev/null +++ b/contrib/docker/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql @@ -0,0 +1,3 @@ +CREATE EXTENSION POSTGIS; +ALTER VIEW geometry_columns OWNER TO ckan; +ALTER TABLE spatial_ref_sys OWNER TO ckan; diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index a2998d45e63..6d99d65cf77 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -251,8 +251,31 @@ In this example we'll install `ckanext-geoview Date: Wed, 19 Jul 2017 12:28:16 +0800 Subject: [PATCH 32/51] reword compose docs following PR comments --- .../install-from-docker-compose.rst | 147 +++++++++++++----- 1 file changed, 109 insertions(+), 38 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 6d99d65cf77..55533f81f85 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -4,21 +4,30 @@ Installing CKAN with docker-compose =================================== -This chapter describes how to install CKAN with docker-compose. The scenario shown here is one of -many possibile scenarios and environments in which CKAN can be used with Docker. +This chapter describes how to install the latest CKAN master with docker-compose. +The scenario shown here is one of many possibile scenarios and environments in which CKAN can be +used with Docker. This chapter aims to provide a simple, yet fully customizable deployment - easier to configure than a source install, more customizable than a package install. -.. note:: Some design decisions are opinionated, and there are alternatives. +.. note:: Some design decisions are opinionated (see notes), which does not mean that the + alternatives are worse. + Some decisions may or may not be suitable for production scenarios, e.g. the use of CKAN master. -------------- 1. Environment -------------- -We will use a Ubuntu environment, so far this has been tested on: +In this tutorial, we will use a Ubuntu environment, which has been tested on: * Amazon AWS EC2 Ubuntu 14.04 LTS -* Ubuntu 16.04 LTS on a desktop +* Ubuntu 16.04 LTS (desktop) + +The choice of cloud provider and operating system is arbitrary and shall by no means bias the +reader. + +.. todo:: Include specific instructions for other cloud providers and operating systems. + Contributions welcome! a. Storage @@ -31,10 +40,12 @@ On the other hand, a snapshotting filesystem like btrfs is ideal for rolling bac The same cost consideration might apply to other cloud-based providers. .. note:: This setup stores data in named volumes, mapped to folder locations which can be - network or local storage. + networked or local storage. An alternative would be to re-write ``docker-compose.yml`` to map local storage directly, bypassing named volumes. Both solutions will save data to a specified location. + Further reading: `Docker Volumes `_. + b. Docker Docker is installed system-wide following the official `Docker CE installation guidelines @@ -57,6 +68,8 @@ Clone CKAN into a directory of your choice:: cd /path/to/my/projects git clone git@github.com:ckan/ckan.git . +.. note:: This example uses CKAN master, which may not be stable enough for production use. + ---------------------- 2. Build Docker images ---------------------- @@ -69,6 +82,8 @@ In a production environment, copy ``contrib/docker/.env.template`` to ``contrib/ and follow instructions within to set passwords and other sensitive or user-defined variables. The defaults will work fine in a development environment. +.. note:: Related reading: `Docker-compose .env file `_ + b. Build the images Inside the CKAN directory:: @@ -77,7 +92,7 @@ Inside the CKAN directory:: docker-compose up -d --build For the remainder of this chapter, we assume that docker-compose commands are all run inside -``contrib/docker``. +``contrib/docker``, where ``docker-compose.yml`` and ``.env`` are located. On first runs, the postgres container could need longer to initialize the database cluster than the ckan container will wait for. This time span depends heavily on available system resources. @@ -105,7 +120,8 @@ There should be five containers running (``docker ps``): * ``datapusher``: A pre-built CKAN Datapusher image. There should be four named Docker volumes (``docker volume ls | grep docker``). They will be -prefixed with the Docker project name (default: ``docker``) +prefixed with the Docker Compose project name (default: ``docker`` or value of host environment +variable ``COMPOSE_PROJECT_NAME``.) * ``docker_ckan_config``: home of ckan.ini * ``docker_ckan_home``: home of ckan venv and source, later also additional CKAN extensions @@ -147,9 +163,22 @@ b. Enable datastore and datapusher in ``ckan.ini`` Find the path to the ``ckan.ini`` within the named volume:: - docker volume inspect docker_ckan_config | grep Mountpoint + docker volume inspect docker_ckan_home | jq -c '.[] | .Mountpoint' - # "Mountpoint": "/var/lib/docker/volumes/docker_ckan_config/_data", + # "/var/lib/docker/volumes/docker_ckan_config/_data" + + # Convenience: set named volumes as env variables on host. + export VOL_CKAN_HOME=`docker volume inspect docker_ckan_home | jq -r -c '.[] | .Mountpoint'` + echo $VOL_CKAN_HOME + + export VOL_CKAN_CONFIG=`docker volume inspect docker_ckan_config | jq -r -c '.[] | .Mountpoint'` + echo $VOL_CKAN_CONFIG + +.. note:: We export the folder locations of data inside named volumes as environment variables. + We use a prefix ``VOL_`` to avoid overriding variables in ``docker-compose.yml``. + + Related reading: `Environment variables in Compose + `_. Edit the ``ckan.ini`` (note: requires sudo):: @@ -189,24 +218,64 @@ The admin user's API key will be instrumental in tranferring data from other ins --------------- 5. Migrate data --------------- -.. todo:: This section requires testing. +This section illustrates the data migration from an existing CKAN instance. + + +a. Transfer resource files + +Assuming the CKAN storage directory on ``SOURCE_CKAN`` is located at ``/path/to/files`` (containing +resource files and uploaded images in ``resources`` and ``storage``), we'll simply ``rsync`` +those into the named volume ``docker_ckan_storage``:: + + export VOL_CKAN_STORAGE=`docker volume inspect docker_ckan_storage | jq -r -c '.[] | .Mountpoint'` + sudo rsync -Pavvr USER@SOURCE_CKAN:/path/to/files/ $VOL_CKAN_STORAGE + +b. Transfer users + +Users can be exported using ``ckanapi``, but their password hashes will be excluded. +To transfer users preserving their passwords, we need to dump and restore the ``user`` table. + +On source CKAN host with access to source db ``ckan_default``, export the ``user`` table:: + + pg_dump -h CKAN_DBHOST -P CKAN_DBPORT -U CKAN_DBUSER -a -O -t user -f user.sql ckan_default + +On the target host, make ``user.sql`` accessible to the source CKAN container. +Transfer user.sql into the named volume ``$CKAN_HOME`` and chown it to the docker user:: + + rsync -Pavvr user@ckan-source-host:/path/to/user.sql $VOL_CKAN_HOME/venv/src -Pending testing, and presuming same dataset schema (ckanext-scheming) on -source and target CKANs, the process should be as simple as: + # Find owner of $VOL_CKAN_HOME + sudo ls -l $VOL_CKAN_HOME + # drwxr-xr-x 1 900 900 62 Jul 17 16:13 venv -* ``rsync`` contents of the filestore into the storage volume. -* Use python package ``ckanapi`` to dump orgs, groups and datasets from source CKAN instance, - transfer files, then use ``ckanapi`` to load the exported data into the target instance. -* Let datapusher populate datastore. -* Trigger a Solr index rebuild. + # Chown user.sql to the owner of $CKAN_HOME (here: 900) + sudo chown 900:900 $VOL_CKAN_HOME/venv/src/user.sql + +Now the file ``user.sql`` is accessible from within the ``ckan`` container:: + + docker exec -it ckan bash + + ckan@eca111c06788:/$ psql -U ckan -h db -f $CKAN_VENV/src/user.sql + +c. Export and upload groups, orgs, datasets + +Using the python package ``ckanapi`` we will dump orgs, groups and datasets from the source CKAN +instance, then use ``ckanapi`` to load the exported data into the target instance. +The datapusher will automatically ingest CSV resources into the datastore. + +d. Rebuild search index + +Trigger a Solr index rebuild:: + + docker exec -it ckan /usr/local/bin/ckan-paster --plugin=ckan search-index rebuild -c /etc/ckan/ckan.ini ----------------- 6. Add extensions ----------------- -There are two use cases how to add extensions: +There are two scenarios to add extensions: -* Developers will want to have access to the extensions' source. -* Maintainers of production instances will want extensions to be part of the ``ckan`` image and an +* Developers need to read, modify and use version control on the extensions' source. +* Maintainers of production instances need extensions to be part of the ``ckan`` image and an easy way to enable them in the ``ckan.ini``. This requires customizing CKAN's ``Dockerfile`` and scripted post-processing of the ``ckan.ini``. @@ -216,6 +285,13 @@ For developers, the process is: * Edit ``ckan.ini`` * Restart ``ckan`` service, read logs. +.. todo:: How to use git from inside the container, ssh keys, write access. + +For maintainers, the process is not fully documented yet. + +.. todo:: How to automate adding extensions. + Script installation steps, possibly use ckanext-envvars to programmatically configure .ini. + a. Download and install extension inside ``ckan`` image The process is very similar to installing extensions in a source install. The only difference is @@ -231,20 +307,18 @@ In this example we'll install `ckanext-geoview `_. -This section illustrates the use of evironment variables. +This section illustrates the use of environment variables provided by the docker-compose ``.env`` +file. This section is targeted at CKAN maintainers seeking a deeper understanding of variables, and at CKAN developers seeking to factor out settings as new ``.env`` variables. @@ -306,9 +377,9 @@ and at CKAN developers seeking to factor out settings as new ``.env`` variables. The flow of variable substitution is as follows: * ``.env.template`` holds the defaults and the usage instructions for variables. -* ``.env`` is copied and modified from ``.env.template`` with values chosen by the maintainer. -* docker-compose interpolates variables in ``docker-compose.yml`` from ``.env``. -* docker-compose can pass on these variables to the containers as build time variables +* The maintainer copies ``.env`` from ``.env.template`` and modifies it following the instructions. +* Docker Compose interpolates variables in ``docker-compose.yml`` from ``.env``. +* Docker Compose can pass on these variables to the containers as build time variables (when building the images) and / or as run time variables (when running the containers). * ``ckan-entrypoint.sh`` has access to all run time variables of the ``ckan`` service. * ``ckan-entrypoint.sh`` injects environment variables (e.g. ``CKAN_SQLALCHEMY_URL``) into the @@ -318,7 +389,7 @@ See :doc:`/maintaining/configuration` for a list of environment variables (e.g. ``CKAN_SQLALCHEMY_URL``) which CKAN will accept to override ``ckan.ini``. After adding new or changing existing ``.env`` variables, locally built images and volumes may -need to be dropped and rebuilt. Otherwise, docker will re-use cached images with old or without +need to be dropped and rebuilt. Otherwise, docker will re-use cached images with old or missing variables:: docker-compose down From 1192f8eae3b61729b71dcc95d2c5d5f39b96b72c Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 19 Jul 2017 12:31:48 +0800 Subject: [PATCH 33/51] compose docs: add option to install extensions from pypi --- doc/maintaining/installing/install-from-docker-compose.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 55533f81f85..41a5c6db600 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -316,9 +316,12 @@ In this example we'll install `ckanext-geoview Date: Thu, 20 Jul 2017 09:37:43 +0800 Subject: [PATCH 34/51] docker compose docs: add steps towards production --- .../install-from-docker-compose.rst | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 41a5c6db600..b0d767b2055 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -4,16 +4,21 @@ Installing CKAN with docker-compose =================================== -This chapter describes how to install the latest CKAN master with docker-compose. -The scenario shown here is one of many possibile scenarios and environments in which CKAN can be +This chapter describes how to install the latest CKAN master with Docker Compose. +The scenario shown here is one of many possible scenarios and environments in which CKAN can be used with Docker. This chapter aims to provide a simple, yet fully customizable deployment - easier to configure than a source install, more customizable than a package install. +The discussed setup can be useful as a development / staging environment; additional care has to be +taken to use the results in production. + .. note:: Some design decisions are opinionated (see notes), which does not mean that the - alternatives are worse. + alternatives are any worse. Some decisions may or may not be suitable for production scenarios, e.g. the use of CKAN master. + Notably, this tutorial does not use Docker Swarm; additional steps may need to be taken to adapt + the setup to use Docker Swarm. -------------- 1. Environment @@ -377,7 +382,7 @@ file. This section is targeted at CKAN maintainers seeking a deeper understanding of variables, and at CKAN developers seeking to factor out settings as new ``.env`` variables. -The flow of variable substitution is as follows: +Variable substitution propagates as follows: * ``.env.template`` holds the defaults and the usage instructions for variables. * The maintainer copies ``.env`` from ``.env.template`` and modifies it following the instructions. @@ -410,3 +415,20 @@ variables:: .. warning:: Removing named volumes will destroy data. Backup all data when doing this in a production setting. +--------------------------- +8. Steps towards production +--------------------------- + +As mentioned above, some design decisions may not be suitable for a production setup. + +A possible path towards a production-ready environment is: + +* Use the above setup to build docker images. +* Add and configure extensions. +* Make sure that no sensitive settings are hard-coded inside the images. +* Push the images to a docker repository. +* Build a separate "production" ``docker-compose.yml`` which uses the custom built images. +* Run the "production" ``docker-compose.yml`` on the production server with appropriate settings. +* Transfer production data into the new server as described above. +* Bonus: contribute a write-up of working production setups to the CKAN documentation. + From 5f4e049d121e3e79a4f843612541cb731ad305df Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 21 Jul 2017 16:01:18 +0800 Subject: [PATCH 35/51] update installation docs with link to compose docs; expand on development setup for extensions --- doc/maintaining/installing/index.rst | 30 +++--- .../install-from-docker-compose.rst | 95 ++++++++++++------- 2 files changed, 80 insertions(+), 45 deletions(-) diff --git a/doc/maintaining/installing/index.rst b/doc/maintaining/installing/index.rst index 3e78999593a..8388855b10c 100644 --- a/doc/maintaining/installing/index.rst +++ b/doc/maintaining/installing/index.rst @@ -2,11 +2,12 @@ Installing CKAN --------------- -Before you can use CKAN on your own computer, you need to install it. There are -three ways to install CKAN: +Before you can use CKAN on your own computer, you need to install it. +There are three ways to install CKAN: #. Install from an operating system package #. Install from source +#. Install from Docker Compose From package is the quickest and easiest way to install CKAN, but it requires Ubuntu 14.04 64-bit or Ubuntu 12.04 64-bit. **You should install CKAN from package if**: @@ -20,23 +21,27 @@ See :doc:`install-from-package`. * You want to install CKAN on a 32-bit computer, *or* * You want to install CKAN on a different version of Ubuntu, not 14.04 or 12.04, *or* -* You want to install CKAN on another operating system - (eg. RHEL, CentOS, OS X), *or* +* You want to install CKAN on another operating system (eg. RHEL, CentOS, OS X), *or* * You want to run multiple CKAN websites on the same server, *or* * You want to install CKAN for development See :doc:`install-from-source`. -If you've already setup a CKAN website and want to upgrade it to a newer -version of CKAN, see :doc:`/maintaining/upgrading/index`. +The provided Docker Compose configuration provides a clean and quick way to deploy a vanilla CKAN +without extensions, while still allowing the addition (and customization) of extensions. +This option comes with the caveat that some further steps need to be taken to deploy a +production-ready CKAN. **You should install CKAN from Docker Compose if**: + +* You want to install CKAN with less effort than a source install and more flexibility than a + package install, **or** +* You want to run or even develop extensions with the minimum setup effort, **or** +* You want to see whether and how CKAN, Docker and your respective infrastructure will fit + together. +See :doc:`install-from-docker-compose`. -.. note:: - There **used** to be an 'official' Docker install of CKAN. There are legacy - docker images at https://hub.docker.com/u/ckan/ which are not maintained and - use out-of-date unpatched versions of CKAN. Information about them is - archived here: - +If you've already setup a CKAN website and want to upgrade it to a newer +version of CKAN, see :doc:`/maintaining/upgrading/index`. ------------ @@ -45,4 +50,5 @@ version of CKAN, see :doc:`/maintaining/upgrading/index`. install-from-package install-from-source + install-from-docker-compose deployment diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index b0d767b2055..909e8d7825c 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -72,8 +72,12 @@ Clone CKAN into a directory of your choice:: cd /path/to/my/projects git clone git@github.com:ckan/ckan.git . + # This will use the latest CKAN master. -.. note:: This example uses CKAN master, which may not be stable enough for production use. + # To use a stable version, checkout the respective tag, e.g.: + git checkout tags/ckan-2.6.2 + +.. note:: Using master may not be stable enough for production use. ---------------------- 2. Build Docker images @@ -87,7 +91,14 @@ In a production environment, copy ``contrib/docker/.env.template`` to ``contrib/ and follow instructions within to set passwords and other sensitive or user-defined variables. The defaults will work fine in a development environment. -.. note:: Related reading: `Docker-compose .env file `_ +.. note:: Related reading: + + `Docker-compose .env file `_ + + `Environment variables in Compose `_ + + Newcomers to Docker should read the excellent write-up on + `Docker variables `_ by Vladislav Supalov (GitHub @th4t). b. Build the images @@ -180,10 +191,7 @@ Find the path to the ``ckan.ini`` within the named volume:: echo $VOL_CKAN_CONFIG .. note:: We export the folder locations of data inside named volumes as environment variables. - We use a prefix ``VOL_`` to avoid overriding variables in ``docker-compose.yml``. - - Related reading: `Environment variables in Compose - `_. + We use a prefix ``VOL_`` to avoid overriding variables in ``docker-compose.yml``. Edit the ``ckan.ini`` (note: requires sudo):: @@ -223,8 +231,8 @@ The admin user's API key will be instrumental in tranferring data from other ins --------------- 5. Migrate data --------------- -This section illustrates the data migration from an existing CKAN instance. - +This section illustrates the data migration from an existing CKAN instance assuming direct access +to the target host. a. Transfer resource files @@ -237,8 +245,9 @@ those into the named volume ``docker_ckan_storage``:: b. Transfer users -Users can be exported using ``ckanapi``, but their password hashes will be excluded. -To transfer users preserving their passwords, we need to dump and restore the ``user`` table. +Users could be exported using the python package ``ckanapi``, but their password hashes will be +excluded. To transfer users preserving their passwords, we need to dump and restore the ``user`` +table. On source CKAN host with access to source db ``ckan_default``, export the ``user`` table:: @@ -249,11 +258,11 @@ Transfer user.sql into the named volume ``$CKAN_HOME`` and chown it to the docke rsync -Pavvr user@ckan-source-host:/path/to/user.sql $VOL_CKAN_HOME/venv/src - # Find owner of $VOL_CKAN_HOME + # $VOL_CKAN_HOME is owned by the user "ckan" (UID 900) created in the CKAN Dockerfile sudo ls -l $VOL_CKAN_HOME # drwxr-xr-x 1 900 900 62 Jul 17 16:13 venv - # Chown user.sql to the owner of $CKAN_HOME (here: 900) + # Chown user.sql to the owner of $CKAN_HOME (ckan, UID 900) sudo chown 900:900 $VOL_CKAN_HOME/venv/src/user.sql Now the file ``user.sql`` is accessible from within the ``ckan`` container:: @@ -279,25 +288,21 @@ Trigger a Solr index rebuild:: ----------------- There are two scenarios to add extensions: -* Developers need to read, modify and use version control on the extensions' source. * Maintainers of production instances need extensions to be part of the ``ckan`` image and an - easy way to enable them in the ``ckan.ini``. This requires customizing CKAN's ``Dockerfile`` and - scripted post-processing of the ``ckan.ini``. + easy way to enable them in the ``ckan.ini``. + Automating the installation of existing extensions (without needing to change their source) + requires customizing CKAN's ``Dockerfile`` and scripted post-processing of the ``ckan.ini``. +* Developers need to read, modify and use version control on the extensions' source. -For developers, the process is: +For maintainers, the process is: -* Run a bash shell inside the running ``ckan`` image, download and install extension. -* Edit ``ckan.ini`` +* Run a bash shell inside the running ``ckan`` container, download and install extension. + Alternative: Insert the ``pip install`` step into a custom CKAN Dockerfile. +* Edit ``ckan.ini``. Alternative: use ``ckanext-envvars`` to configure the ``ckan.ini`` using + environment variables, which can be inserted into ``docker-compose`` via ``.env``. * Restart ``ckan`` service, read logs. -.. todo:: How to use git from inside the container, ssh keys, write access. - -For maintainers, the process is not fully documented yet. - -.. todo:: How to automate adding extensions. - Script installation steps, possibly use ckanext-envvars to programmatically configure .ini. - -a. Download and install extension inside ``ckan`` image +a. Download and install extension from inside ``ckan`` container into ``docker_ckan_home`` volume The process is very similar to installing extensions in a source install. The only difference is that the installation steps happen inside the running container, using the virtualenv created @@ -306,7 +311,10 @@ on the host machine in step 2. The downloaded and installed files will be persisted in the named volume ``docker_ckan_home``. -In this example we'll install `ckanext-geoview `_:: +In this example we'll enter the running ``ckan`` container to install +`ckanext-geoview `_ from source, +`ckanext-showcase `_ from GitHub, +and `ckanext-envvars `_ from PyPi:: # Enter the running ckan container: docker exec -it ckan bash @@ -331,7 +339,7 @@ In this example we'll install `ckanext-geoview `_:: # Enter the running ckan container: @@ -345,25 +353,26 @@ E.g., ``ckanext-spatial``:: python setup.py install && python setup.py develop exit + # On the host docker exec -it db psql -U ckan -f 20_postgis_permissions.sql docker exec -it ckan /usr/local/bin/ckan-paster --plugin=ckanext-spatial spatial initdb -c /etc/ckan/ckan.ini - sudo vim /var/lib/docker/volumes/docker_ckan_config/_data/ckan.ini + sudo vim $VOL_CKAN_CONFIG/ckan.ini - # in plugins, add: + # Inside ckan.ini, add to [plugins]: spatial_metadata spatial_query ckanext.spatial.search_backend = solr -.. note:: In order to work on your own forks of CKAN extensions, the container's - ssh publickey must be added to the respective repository's authorized keys. - b. Modify CKAN config Follow the respective extension's instructions to set CKAN config variables:: sudo vim $VOL_CKAN_CONFIG/ckan.ini +.. todo:: Demonstrate how to set ``ckan.ini`` settings from environment variables using + ``ckanext-envvars``. + c. Reload and debug :: @@ -371,6 +380,26 @@ c. Reload and debug docker-compose restart ckan docker-compose logs ckan +d. Develop extensions: modify source, install, use version control + +While maintainers will prefer to use stable versions of existing extensions, developers of +extensions will need access to the extensions' source, and be able to use version control. + +Using SSH from inside running Docker images requires the presence of sensitive data (SSH keys) +inside the running container. + +The two simplest approaches here are to either use HTTPS (and type GitHub username and password +with every ``git`` command) from inside the running container, or work on the files in the named +volumes. + +The CKAN Docker image is run by an internal user as defined in the Dockerfile (username ``ckan``, +UID 900). Named volumes are owned by this user (900:900). We have to carefully update the +permissions and ownership of the volumes to be able to access and modify files from the host. + +.. todo:: Add worked example to give a host user access to $VOL_CKAN_HOME, work on extensions, + use version control + + ------------------------ 7. Environment variables ------------------------ From 9329a81d3f74183c3f626cbcad38e574add91c50 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 21 Jul 2017 17:14:02 +0800 Subject: [PATCH 36/51] working extensions development setup documented --- .../install-from-docker-compose.rst | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 909e8d7825c..94dea3d4df0 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -385,20 +385,37 @@ d. Develop extensions: modify source, install, use version control While maintainers will prefer to use stable versions of existing extensions, developers of extensions will need access to the extensions' source, and be able to use version control. -Using SSH from inside running Docker images requires the presence of sensitive data (SSH keys) -inside the running container. +The use of Docker and the inherent encapsulation of files and processes makes the development of +extensions harder than a CKAN source install. -The two simplest approaches here are to either use HTTPS (and type GitHub username and password -with every ``git`` command) from inside the running container, or work on the files in the named -volumes. +Since we have chosen to use named volumes instead of mounted host folders, we have to make the +write-protected volumes accessible to a system user. The Ubuntu package ``bindfs`` helps here:: -The CKAN Docker image is run by an internal user as defined in the Dockerfile (username ``ckan``, -UID 900). Named volumes are owned by this user (900:900). We have to carefully update the -permissions and ownership of the volumes to be able to access and modify files from the host. + sudo apt-get install bindfs + mkdir ~/VOL_CKAN_HOME + sudo chown -R `whoami`:docker $VOL_CKAN_HOME + sudo bindfs --map=900/`whoami` $VOL_CKAN_HOME ~/VOL_CKAN_HOME -.. todo:: Add worked example to give a host user access to $VOL_CKAN_HOME, work on extensions, - use version control + cd ~/VOL_CKAN_HOME/venv/src + # Do this with your own extension fork + # Assumption: the host user running git clone (you) has write access to the repository + git clone git@github.com:parksandwildlife/ckanext-datawagovautheme.git + + # ... change files, use version control... + +Changes in templates and CSS will be visible right away. +For changes in code, we'll need to unmount the directory, change ownership back to the ``ckan`` +user, and follow the previous steps to ``python setup.py install`` and +``pip install -r requirements.txt`` from within the running container, modify the ``ckan.ini`` +and restart the container:: + + sudo umount ~/VOL_CKAN_HOME + sudo chown -R 900:900 $VOL_CKAN_HOME + # Follow steps a-c + +.. note:: Mounting host folders as volumes instead of using named volumes may result in a simpler + development workflow. ------------------------ 7. Environment variables From 95ef011f56f52d49ec3ce80a650bc34dc7b2bae0 Mon Sep 17 00:00:00 2001 From: florianm Date: Mon, 24 Jul 2017 11:50:38 +0800 Subject: [PATCH 37/51] Dockerfile: remove duplicate apt-get install python-dev --- Dockerfile | 6 +- .../install-from-docker-compose.rst | 80 +++++++++---------- 2 files changed, 38 insertions(+), 48 deletions(-) diff --git a/Dockerfile b/Dockerfile index 153908d75d8..e86053b338d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,4 @@ -# docker build . -t ckan --build-arg CKAN_SITE_URL=http://localhost:5000 -# docker run -d -p 80:5000 --link db:db --link redis:redis --link solr:solr ckan \ -# -v ckan_config:/etc.ckan/default -v ckan_storage:/var/lib/ckan - +# See CKAN docs on installation from Docker Compose on usage FROM debian:jessie MAINTAINER Open Knowledge @@ -14,7 +11,6 @@ RUN apt-get -q -y update && apt-get -q -y upgrade && \ libpq-dev \ git-core \ postgresql-client \ - python-dev \ libxml2-dev \ libxslt-dev \ libgeos-dev \ diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 94dea3d4df0..94ea98ff5b6 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -3,16 +3,15 @@ =================================== Installing CKAN with docker-compose =================================== - -This chapter describes how to install the latest CKAN master with Docker Compose. -The scenario shown here is one of many possible scenarios and environments in which CKAN can be -used with Docker. +This chapter is a tutorial on how to install the latest CKAN (master or any stable version) +with Docker Compose. The scenario shown here is one of many possible scenarios and environments +in which CKAN can be used with Docker. This chapter aims to provide a simple, yet fully customizable deployment - easier to configure than a source install, more customizable than a package install. The discussed setup can be useful as a development / staging environment; additional care has to be -taken to use the results in production. +taken to use this setup in production. .. note:: Some design decisions are opinionated (see notes), which does not mean that the alternatives are any worse. @@ -23,26 +22,19 @@ taken to use the results in production. -------------- 1. Environment -------------- -In this tutorial, we will use a Ubuntu environment, which has been tested on: - -* Amazon AWS EC2 Ubuntu 14.04 LTS -* Ubuntu 16.04 LTS (desktop) - -The choice of cloud provider and operating system is arbitrary and shall by no means bias the -reader. - -.. todo:: Include specific instructions for other cloud providers and operating systems. - Contributions welcome! +This tutorial was tested on Ubuntu 14.04 LTS and Ubuntu 16.04 LTS, respectively. +The hosts can be local environments or cloud VMs. It is assumed that the user has direct access +(via terminal / ssh) to the systems and root permissions. a. Storage Using a cloud based VM, external storage volumes are cheaper than VMs and easy to backup. -In our use case, we use an AWS EC2 with 16 GB storage, have mounted a 100 GB btrfs-formatted +In our use case, we use a cloud based VM with 16 GB storage, have mounted a 100 GB btrfs-formatted external storage volume, and symlinked ``/var/lib/docker`` to the external volume. -This allows us to store the bulky and precious cargo -- Docker images, Docker data volumes +This allows us to store the bulky and/or precious cargo -- Docker images, Docker data volumes containing the CKAN databases, filestore, and config -- on a cheaper service. On the other hand, a snapshotting filesystem like btrfs is ideal for rolling backups. -The same cost consideration might apply to other cloud-based providers. +The same cost consideration might apply to any cloud-based providers. .. note:: This setup stores data in named volumes, mapped to folder locations which can be networked or local storage. @@ -63,6 +55,9 @@ c. Docker Compose Docker Compose is installed system-wide following the official `Docker Compose installation guidelines `_. +Alternatively, Docker Compose can be installed inside a virtualenv, +which would be entirely separate from the virtualenv used inside the CKAN container, and +would need to be activated before runningn Docker Compose commands. To verify a successful Docker Compose installation, run ``docker-compose version``. @@ -74,30 +69,28 @@ Clone CKAN into a directory of your choice:: git clone git@github.com:ckan/ckan.git . # This will use the latest CKAN master. - # To use a stable version, checkout the respective tag, e.g.: - git checkout tags/ckan-2.6.2 +Using master may not be stable enough for production use. +To use a stable version, checkout the respective tag, e.g.:: -.. note:: Using master may not be stable enough for production use. + git checkout tags/ckan-2.6.2 ---------------------- 2. Build Docker images ---------------------- In this step we will build the Docker images and create Docker data volumes with user-defined, -sensitive settings (e.g. for database passwords). +sensitive settings (e.g. database passwords). a. Sensitive settings and environment variables -In a production environment, copy ``contrib/docker/.env.template`` to ``contrib/docker/.env`` -and follow instructions within to set passwords and other sensitive or user-defined variables. +Copy ``contrib/docker/.env.template`` to ``contrib/docker/.env`` and follow instructions +within to set passwords and other sensitive or user-defined variables. The defaults will work fine in a development environment. .. note:: Related reading: - `Docker-compose .env file `_ - - `Environment variables in Compose `_ - - Newcomers to Docker should read the excellent write-up on + * `Docker-compose .env file `_ + * `Environment variables in Compose `_ + * Newcomers to Docker should read the excellent write-up on `Docker variables `_ by Vladislav Supalov (GitHub @th4t). b. Build the images @@ -107,7 +100,7 @@ Inside the CKAN directory:: cd contrib/docker docker-compose up -d --build -For the remainder of this chapter, we assume that docker-compose commands are all run inside +For the remainder of this chapter, we assume that ``docker-compose`` commands are all run inside ``contrib/docker``, where ``docker-compose.yml`` and ``.env`` are located. On first runs, the postgres container could need longer to initialize the database cluster than @@ -221,7 +214,7 @@ Now the datastore API should return content when visiting:: ------------------------- 4. Create CKAN admin user ------------------------- -With all four Docker images up and running, create the CKAN admin user (johndoe in this example):: +With all images up and running, create the CKAN admin user (johndoe in this example):: docker exec -it ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin -c /etc/ckan/ckan.ini add johndoe @@ -292,22 +285,20 @@ There are two scenarios to add extensions: easy way to enable them in the ``ckan.ini``. Automating the installation of existing extensions (without needing to change their source) requires customizing CKAN's ``Dockerfile`` and scripted post-processing of the ``ckan.ini``. -* Developers need to read, modify and use version control on the extensions' source. +* Developers need to read, modify and use version control on the extensions' source. This adds + additional steps to the maintainers' workflow. For maintainers, the process is: * Run a bash shell inside the running ``ckan`` container, download and install extension. Alternative: Insert the ``pip install`` step into a custom CKAN Dockerfile. -* Edit ``ckan.ini``. Alternative: use ``ckanext-envvars`` to configure the ``ckan.ini`` using - environment variables, which can be inserted into ``docker-compose`` via ``.env``. * Restart ``ckan`` service, read logs. a. Download and install extension from inside ``ckan`` container into ``docker_ckan_home`` volume The process is very similar to installing extensions in a source install. The only difference is that the installation steps happen inside the running container, using the virtualenv created -inside the ckan image by CKAN's Dockerfile, which is different from the virtualenv we have created -on the host machine in step 2. +inside the ckan image by CKAN's Dockerfile. The downloaded and installed files will be persisted in the named volume ``docker_ckan_home``. @@ -328,6 +319,7 @@ and `ckanext-envvars `_ from PyPi:: pip install -r pip-requirements.txt python setup.py install python setup.py develop + cd .. # Option 2: Pip install from GitHub pip install -e "git+https://github.com/ckan/ckanext-showcase.git#egg=ckanext-showcase" @@ -415,7 +407,7 @@ and restart the container:: # Follow steps a-c .. note:: Mounting host folders as volumes instead of using named volumes may result in a simpler - development workflow. + development workflow. However, named volumes are Docker's canonical way to persist data. ------------------------ 7. Environment variables @@ -450,16 +442,17 @@ variables:: docker-compose up -d --build # if that didn't work, try: - docker rmi $(docker images -f dangling=true -q) + docker rmi $(docker images -q -f dangling=true) docker-compose up -d --build # if that didn't work, try: - docker rmi $(docker images -f dangling=true -q) - docker volume rm $(docker volume ls -f dangling=true -q) + docker rmi $(docker images -q -f dangling=true) + docker volume prune docker-compose up -d --build .. warning:: Removing named volumes will destroy data. - Backup all data when doing this in a production setting. + ``docker volume prune`` will delete any volumes not attached to a running(!) container. + Backup all data before doing this in a production setting. --------------------------- 8. Steps towards production @@ -473,8 +466,9 @@ A possible path towards a production-ready environment is: * Add and configure extensions. * Make sure that no sensitive settings are hard-coded inside the images. * Push the images to a docker repository. -* Build a separate "production" ``docker-compose.yml`` which uses the custom built images. +* Create a separate "production" ``docker-compose.yml`` which uses the custom built images. * Run the "production" ``docker-compose.yml`` on the production server with appropriate settings. -* Transfer production data into the new server as described above. +* Transfer production data into the new server as described above using volume orchestration + tools or transferring files directly. * Bonus: contribute a write-up of working production setups to the CKAN documentation. From 97d914fe805ff324325d83f57050f0cdb9de4acd Mon Sep 17 00:00:00 2001 From: florianm Date: Mon, 24 Jul 2017 12:17:25 +0800 Subject: [PATCH 38/51] compose docs: new section on env vars for named volume path --- .../install-from-docker-compose.rst | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 94ea98ff5b6..6da6bd48971 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -141,6 +141,29 @@ The location of these named volumes needs to be backed up in a production enviro To migrate CKAN data between different hosts, simply transfer the content of the named volumes. A detailed use case of data transfer will be discussed in step 5. +c. Convenience: paths to named volumes + +The files inside named volumes reside on a long-ish path on the host. +Purely for convenience, we'll define environment variables for these paths. +We'll use a prefix ``VOL_`` to avoid overriding variables in ``docker-compose.yml``.:: + + # Find the path to a named volume + docker volume inspect docker_ckan_home | jq -c '.[] | .Mountpoint' + # "/var/lib/docker/volumes/docker_ckan_config/_data" + + export VOL_CKAN_HOME=`docker volume inspect docker_ckan_home | jq -r -c '.[] | .Mountpoint'` + echo $VOL_CKAN_HOME + + export VOL_CKAN_CONFIG=`docker volume inspect docker_ckan_config | jq -r -c '.[] | .Mountpoint'` + echo $VOL_CKAN_CONFIG + + export VOL_CKAN_STORAGE=`docker volume inspect docker_ckan_storage | jq -r -c '.[] | .Mountpoint'` + echo $VOL_CKAN_STORAGE + +We won't need to access files inside ``docker_pg_data`` directly, therefore we'll skip the shortcut. +As shown further below, we can use ``psql`` from inside the ``ckan`` container to run commands +against the database and import / export files from ``$VOL_CKAN_HOME``. + --------------------------- 3. Datastore and datapusher --------------------------- @@ -170,25 +193,9 @@ After this step, the datastore database is ready to be enabled in the ``ckan.ini b. Enable datastore and datapusher in ``ckan.ini`` -Find the path to the ``ckan.ini`` within the named volume:: - - docker volume inspect docker_ckan_home | jq -c '.[] | .Mountpoint' - - # "/var/lib/docker/volumes/docker_ckan_config/_data" - - # Convenience: set named volumes as env variables on host. - export VOL_CKAN_HOME=`docker volume inspect docker_ckan_home | jq -r -c '.[] | .Mountpoint'` - echo $VOL_CKAN_HOME - - export VOL_CKAN_CONFIG=`docker volume inspect docker_ckan_config | jq -r -c '.[] | .Mountpoint'` - echo $VOL_CKAN_CONFIG - -.. note:: We export the folder locations of data inside named volumes as environment variables. - We use a prefix ``VOL_`` to avoid overriding variables in ``docker-compose.yml``. - Edit the ``ckan.ini`` (note: requires sudo):: - sudo vim /var/lib/docker/volumes/docker_ckan_config/_data/ckan.ini + sudo vim $VOL_CKAN_CONFIG/ckan.ini Add ``datastore datapusher`` to ``ckan.plugins`` and enable the datapusher option ``ckan.datapusher.formats``. @@ -202,7 +209,6 @@ The remaining settings required for datastore and datapusher are already taken c ``ckan.datastore.read_url = postgresql://datastore:DATASTORE_READONLY_PASSWORD@db/datastore`` are provided by ``docker-compose.yml``. - Restart the ``ckan`` container to apply changes to the ``ckan.ini``:: docker-compose restart ckan @@ -233,7 +239,6 @@ Assuming the CKAN storage directory on ``SOURCE_CKAN`` is located at ``/path/to/ resource files and uploaded images in ``resources`` and ``storage``), we'll simply ``rsync`` those into the named volume ``docker_ckan_storage``:: - export VOL_CKAN_STORAGE=`docker volume inspect docker_ckan_storage | jq -r -c '.[] | .Mountpoint'` sudo rsync -Pavvr USER@SOURCE_CKAN:/path/to/files/ $VOL_CKAN_STORAGE b. Transfer users @@ -247,7 +252,7 @@ On source CKAN host with access to source db ``ckan_default``, export the ``user pg_dump -h CKAN_DBHOST -P CKAN_DBPORT -U CKAN_DBUSER -a -O -t user -f user.sql ckan_default On the target host, make ``user.sql`` accessible to the source CKAN container. -Transfer user.sql into the named volume ``$CKAN_HOME`` and chown it to the docker user:: +Transfer user.sql into the named volume ``docker_ckan_home`` and chown it to the docker user:: rsync -Pavvr user@ckan-source-host:/path/to/user.sql $VOL_CKAN_HOME/venv/src From 097c4f2ce47028018fab23b1939b9c3b79be4c53 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 26 Jul 2017 16:19:52 +0800 Subject: [PATCH 39/51] CKAN Dockerfile: install vim --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e86053b338d..99c175e15b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ RUN apt-get -q -y update && apt-get -q -y upgrade && \ libxml2-dev \ libxslt-dev \ libgeos-dev \ + vim \ && apt-get -q clean # Define environment variables From a2baa9f4ca2c2e65e58b5ecfcf3ad01cec23b4de Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 27 Jul 2017 10:25:55 +0800 Subject: [PATCH 40/51] docker install docs: fix indent --- .../installing/install-from-docker-compose.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 6da6bd48971..2350a7dd7ad 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -3,8 +3,8 @@ =================================== Installing CKAN with docker-compose =================================== -This chapter is a tutorial on how to install the latest CKAN (master or any stable version) -with Docker Compose. The scenario shown here is one of many possible scenarios and environments +This chapter is a tutorial on how to install the latest CKAN (master or any stable version) +with Docker Compose. The scenario shown here is one of many possible scenarios and environments in which CKAN can be used with Docker. This chapter aims to provide a simple, yet fully customizable deployment - easier to configure than @@ -22,7 +22,7 @@ taken to use this setup in production. -------------- 1. Environment -------------- -This tutorial was tested on Ubuntu 14.04 LTS and Ubuntu 16.04 LTS, respectively. +This tutorial was tested on Ubuntu 14.04 LTS and Ubuntu 16.04 LTS, respectively. The hosts can be local environments or cloud VMs. It is assumed that the user has direct access (via terminal / ssh) to the systems and root permissions. @@ -55,7 +55,7 @@ c. Docker Compose Docker Compose is installed system-wide following the official `Docker Compose installation guidelines `_. -Alternatively, Docker Compose can be installed inside a virtualenv, +Alternatively, Docker Compose can be installed inside a virtualenv, which would be entirely separate from the virtualenv used inside the CKAN container, and would need to be activated before runningn Docker Compose commands. @@ -82,7 +82,7 @@ sensitive settings (e.g. database passwords). a. Sensitive settings and environment variables -Copy ``contrib/docker/.env.template`` to ``contrib/docker/.env`` and follow instructions +Copy ``contrib/docker/.env.template`` to ``contrib/docker/.env`` and follow instructions within to set passwords and other sensitive or user-defined variables. The defaults will work fine in a development environment. @@ -91,7 +91,7 @@ The defaults will work fine in a development environment. * `Docker-compose .env file `_ * `Environment variables in Compose `_ * Newcomers to Docker should read the excellent write-up on - `Docker variables `_ by Vladislav Supalov (GitHub @th4t). + `Docker variables `_ by Vladislav Supalov (GitHub @th4t) b. Build the images @@ -143,7 +143,7 @@ A detailed use case of data transfer will be discussed in step 5. c. Convenience: paths to named volumes -The files inside named volumes reside on a long-ish path on the host. +The files inside named volumes reside on a long-ish path on the host. Purely for convenience, we'll define environment variables for these paths. We'll use a prefix ``VOL_`` to avoid overriding variables in ``docker-compose.yml``.:: @@ -473,7 +473,7 @@ A possible path towards a production-ready environment is: * Push the images to a docker repository. * Create a separate "production" ``docker-compose.yml`` which uses the custom built images. * Run the "production" ``docker-compose.yml`` on the production server with appropriate settings. -* Transfer production data into the new server as described above using volume orchestration +* Transfer production data into the new server as described above using volume orchestration tools or transferring files directly. * Bonus: contribute a write-up of working production setups to the CKAN documentation. From 56eb18e2c5bc3f088ffd53bf4cc32ea958a230a2 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 27 Jul 2017 14:06:18 +0800 Subject: [PATCH 41/51] docker install docs: add working example of editing extension templates --- doc/maintaining/installing/install-from-docker-compose.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 2350a7dd7ad..3f9c7b77425 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -414,6 +414,13 @@ and restart the container:: .. note:: Mounting host folders as volumes instead of using named volumes may result in a simpler development workflow. However, named volumes are Docker's canonical way to persist data. +Alternatively, the files in ``VOL_CKAN_HOME`` are editable by the host's superuser:: + + sudo vim $VOL_CKAN_HOME/venv/src/ckanext-datawagovautheme/ckanext/datawagovautheme/templates/package/search.html + +A change to HTML templates in docker volumes by the host superuser applies to the CKAN instance +right away. + ------------------------ 7. Environment variables ------------------------ From 6e4aaba3e546b4f981e180442df520549db8bf97 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Mon, 31 Jul 2017 11:59:57 +0800 Subject: [PATCH 42/51] compose docs: clarify development workflow and version control --- .../install-from-docker-compose.rst | 54 ++++++++++++++----- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 3f9c7b77425..674d4aefc08 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -1,7 +1,7 @@ .. include:: /_substitutions.rst =================================== -Installing CKAN with docker-compose +Installing CKAN with Docker Compose =================================== This chapter is a tutorial on how to install the latest CKAN (master or any stable version) with Docker Compose. The scenario shown here is one of many possible scenarios and environments @@ -382,11 +382,43 @@ d. Develop extensions: modify source, install, use version control While maintainers will prefer to use stable versions of existing extensions, developers of extensions will need access to the extensions' source, and be able to use version control. -The use of Docker and the inherent encapsulation of files and processes makes the development of +The use of Docker and the inherent encapsulation of files and permissions makes the development of extensions harder than a CKAN source install. -Since we have chosen to use named volumes instead of mounted host folders, we have to make the -write-protected volumes accessible to a system user. The Ubuntu package ``bindfs`` helps here:: +Firstly, the absence of private SSH keys inside Docker containers will make interacting with +GitHub a lot harder. On the other hand, two-factor authentication on GitHub breaks BasicAuth +(HTTPS, username and password) and requires a "personal access token" in place of the password. + +To use version control from inside the Docker container: + +* Clone the HTTPS version of the GitHub repo. +* On GitHub, create a personal access token with "full control of private repositories". +* Copy the token code and use as password when running ``git push``. + +Secondly, the persisted extension source at ``VOL_CKAN_HOME`` is owned by the CKAN container's +``docker`` user (UID 900) and therefore not writeable to the developer's host user account by +default. There are various workarounds. The extension source can be accessed from both outside and +inside the container. + +Option 1: Accessing the source from inside the container:: + + docker exec -it ckan bash + source $CKAN_VENV/bin/activate && cd $CKAN_VENV/src/ + # ... work on extensions, use version control ... + # in extension folder: + python setup.py install + exit + # ... edit extension settings in ckan.ini and restart ckan container + sudo vim $VOL_CKAN_CONFIG/ckan.ini + docker-compose restart ckan + +Option 2: Accessing the source from outside the container using ``sudo``:: + + sudo vim $VOL_CKAN_CONFIG/ckan.ini + sudo vim $VOL_CKAN_HOME/venv/src/ckanext-datawagovautheme/ckanext/datawagovautheme/templates/package/search.html + +Option 3:: The Ubuntu package ``bindfs`` makes the write-protected volumes accessible to a system +user:: sudo apt-get install bindfs mkdir ~/VOL_CKAN_HOME @@ -397,11 +429,11 @@ write-protected volumes accessible to a system user. The Ubuntu package ``bindfs # Do this with your own extension fork # Assumption: the host user running git clone (you) has write access to the repository - git clone git@github.com:parksandwildlife/ckanext-datawagovautheme.git + git clone https://github.com/parksandwildlife/ckanext-datawagovautheme.git # ... change files, use version control... -Changes in templates and CSS will be visible right away. +Changes in HTML templates and CSS will be visible right away. For changes in code, we'll need to unmount the directory, change ownership back to the ``ckan`` user, and follow the previous steps to ``python setup.py install`` and ``pip install -r requirements.txt`` from within the running container, modify the ``ckan.ini`` @@ -413,20 +445,14 @@ and restart the container:: .. note:: Mounting host folders as volumes instead of using named volumes may result in a simpler development workflow. However, named volumes are Docker's canonical way to persist data. - -Alternatively, the files in ``VOL_CKAN_HOME`` are editable by the host's superuser:: - - sudo vim $VOL_CKAN_HOME/venv/src/ckanext-datawagovautheme/ckanext/datawagovautheme/templates/package/search.html - -A change to HTML templates in docker volumes by the host superuser applies to the CKAN instance -right away. + The steps shown above are only some of several possible approaches. ------------------------ 7. Environment variables ------------------------ Sensitive settings can be managed in (at least) two ways, either as environment variables, or as `Docker secrets `_. -This section illustrates the use of environment variables provided by the docker-compose ``.env`` +This section illustrates the use of environment variables provided by the Docker Compose ``.env`` file. This section is targeted at CKAN maintainers seeking a deeper understanding of variables, From 2d713e12a52c611276ed815a6de9ad3105840e90 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Mon, 31 Jul 2017 13:39:05 +0800 Subject: [PATCH 43/51] add wget to CKAN Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 99c175e15b1..f67aee0ea07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get -q -y update && apt-get -q -y upgrade && \ libxslt-dev \ libgeos-dev \ vim \ + wget \ && apt-get -q clean # Define environment variables From 0ca47a6645f7acd59a1fe9e73efba6e5419ed1a0 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Mon, 31 Jul 2017 13:46:47 +0800 Subject: [PATCH 44/51] CKAN Dockerfile: factor out CKAN_CONFIG --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f67aee0ea07..af3b9a3ba78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,5 +50,5 @@ ENTRYPOINT ["/ckan-entrypoint.sh"] USER ckan EXPOSE 5000 -CMD ["ckan-paster","serve","/etc/ckan/ckan.ini"] +CMD ["ckan-paster","serve","${CKAN_CONFIG}/ckan.ini"] From 22227e2570b7e98f36c85d748f158a14019ea5af Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Tue, 1 Aug 2017 17:00:30 +0800 Subject: [PATCH 45/51] CKAN Dockerfile: hardcode /etc/ckan to fix entrypoint --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index af3b9a3ba78..f67aee0ea07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,5 +50,5 @@ ENTRYPOINT ["/ckan-entrypoint.sh"] USER ckan EXPOSE 5000 -CMD ["ckan-paster","serve","${CKAN_CONFIG}/ckan.ini"] +CMD ["ckan-paster","serve","/etc/ckan/ckan.ini"] From dd1c554130a32a7d59c475fe0900929c21b4a1da Mon Sep 17 00:00:00 2001 From: florianm Date: Fri, 4 Aug 2017 12:13:44 +0800 Subject: [PATCH 46/51] update install docs with fix for ckan bash $TERM --- .../installing/install-from-docker-compose.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 674d4aefc08..87de3132510 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -265,7 +265,7 @@ Transfer user.sql into the named volume ``docker_ckan_home`` and chown it to the Now the file ``user.sql`` is accessible from within the ``ckan`` container:: - docker exec -it ckan bash + docker exec -it ckan /bin/bash -c "export TERM=xterm; exec bash" ckan@eca111c06788:/$ psql -U ckan -h db -f $CKAN_VENV/src/user.sql @@ -313,7 +313,7 @@ In this example we'll enter the running ``ckan`` container to install and `ckanext-envvars `_ from PyPi:: # Enter the running ckan container: - docker exec -it ckan bash + docker exec -it ckan /bin/bash -c "export TERM=xterm; exec bash" # Inside the running container, activate the virtualenv source $CKAN_VENV/bin/activate && cd $CKAN_VENV/src/ @@ -340,7 +340,7 @@ E.g., `ckanext-spatial `_:: # Enter the running ckan container: - docker exec -it ckan bash + docker exec -it ckan /bin/bash -c "export TERM=xterm; exec bash" # Inside the running ckan container source $CKAN_VENV/bin/activate && cd $CKAN_VENV/src/ @@ -402,7 +402,7 @@ inside the container. Option 1: Accessing the source from inside the container:: - docker exec -it ckan bash + docker exec -it ckan /bin/bash -c "export TERM=xterm; exec bash" source $CKAN_VENV/bin/activate && cd $CKAN_VENV/src/ # ... work on extensions, use version control ... # in extension folder: From bd85e546d055640b5af7f178ded7e393f86e507e Mon Sep 17 00:00:00 2001 From: florianm Date: Wed, 20 Sep 2017 12:04:28 +0800 Subject: [PATCH 47/51] set datastore permissions live --- .../install-from-docker-compose.rst | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 87de3132510..783f0d8765c 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -34,7 +34,7 @@ external storage volume, and symlinked ``/var/lib/docker`` to the external volum This allows us to store the bulky and/or precious cargo -- Docker images, Docker data volumes containing the CKAN databases, filestore, and config -- on a cheaper service. On the other hand, a snapshotting filesystem like btrfs is ideal for rolling backups. -The same cost consideration might apply to any cloud-based providers. +The same cost consideration might apply to other cloud-based providers. .. note:: This setup stores data in named volumes, mapped to folder locations which can be networked or local storage. @@ -57,7 +57,7 @@ Docker Compose is installed system-wide following the official `Docker Compose i guidelines `_. Alternatively, Docker Compose can be installed inside a virtualenv, which would be entirely separate from the virtualenv used inside the CKAN container, and -would need to be activated before runningn Docker Compose commands. +would need to be activated before running Docker Compose commands. To verify a successful Docker Compose installation, run ``docker-compose version``. @@ -67,9 +67,8 @@ Clone CKAN into a directory of your choice:: cd /path/to/my/projects git clone git@github.com:ckan/ckan.git . - # This will use the latest CKAN master. -Using master may not be stable enough for production use. +This will use the latest CKAN master, which may not be stable enough for production use. To use a stable version, checkout the respective tag, e.g.:: git checkout tags/ckan-2.6.2 @@ -160,7 +159,7 @@ We'll use a prefix ``VOL_`` to avoid overriding variables in ``docker-compose.ym export VOL_CKAN_STORAGE=`docker volume inspect docker_ckan_storage | jq -r -c '.[] | .Mountpoint'` echo $VOL_CKAN_STORAGE -We won't need to access files inside ``docker_pg_data`` directly, therefore we'll skip the shortcut. +We won't need to access files inside ``docker_pg_data`` directly, so we'll skip creating the shortcut. As shown further below, we can use ``psql`` from inside the ``ckan`` container to run commands against the database and import / export files from ``$VOL_CKAN_HOME``. @@ -175,10 +174,14 @@ a. Create and configure datastore database With running CKAN containers, execute the built-in setup scripts against the ``db`` container:: docker exec -it db psql -U ckan -f 00_create_datastore.sql - docker exec -it db psql -U ckan -f 10_set_permissions.sql + # canned, possibly outdated: + # docker exec -it db psql -U ckan -f 10_set_permissions.sql + # fresh: + docker exec ckan /usr/local/bin/ckan-paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | docker exec -i db psql -U ckan The first script will create the datastore database and the datastore readonly user in the ``db`` -container. The second script is the output of ``paster ckan set-permissions``. +container. The second script is the output of ``paster ckan set-permissions`` - however, +as this output can change in future versions of CKAN, we set the permissions directly. The effect of these scripts is persisted in the named volume ``docker_pg_data``. .. note:: We re-use the already privileged default user of the CKAN database as read/write user @@ -230,14 +233,14 @@ The admin user's API key will be instrumental in tranferring data from other ins --------------- 5. Migrate data --------------- -This section illustrates the data migration from an existing CKAN instance assuming direct access -to the target host. +This section illustrates the data migration from an existing CKAN instance ``SOURCE_CKAN`` +into our new Docker Compose CKAN instance assuming direct (ssh) access to ``SOURCE_CKAN``. a. Transfer resource files Assuming the CKAN storage directory on ``SOURCE_CKAN`` is located at ``/path/to/files`` (containing resource files and uploaded images in ``resources`` and ``storage``), we'll simply ``rsync`` -those into the named volume ``docker_ckan_storage``:: +``SOURCE_CKAN``'s storage directory into the named volume ``docker_ckan_storage``:: sudo rsync -Pavvr USER@SOURCE_CKAN:/path/to/files/ $VOL_CKAN_STORAGE @@ -252,11 +255,11 @@ On source CKAN host with access to source db ``ckan_default``, export the ``user pg_dump -h CKAN_DBHOST -P CKAN_DBPORT -U CKAN_DBUSER -a -O -t user -f user.sql ckan_default On the target host, make ``user.sql`` accessible to the source CKAN container. -Transfer user.sql into the named volume ``docker_ckan_home`` and chown it to the docker user:: +Transfer user.sql into the named volume ``docker_ckan_home`` and ``chown`` it to the docker user:: rsync -Pavvr user@ckan-source-host:/path/to/user.sql $VOL_CKAN_HOME/venv/src - # $VOL_CKAN_HOME is owned by the user "ckan" (UID 900) created in the CKAN Dockerfile + # $VOL_CKAN_HOME is owned by the user "ckan" (UID 900) as created in the CKAN Dockerfile sudo ls -l $VOL_CKAN_HOME # drwxr-xr-x 1 900 900 62 Jul 17 16:13 venv @@ -293,17 +296,17 @@ There are two scenarios to add extensions: * Developers need to read, modify and use version control on the extensions' source. This adds additional steps to the maintainers' workflow. -For maintainers, the process is: +For maintainers, the process is in summary: * Run a bash shell inside the running ``ckan`` container, download and install extension. - Alternative: Insert the ``pip install`` step into a custom CKAN Dockerfile. + Alternatively, add a ``pip install`` step for the extension into a custom CKAN Dockerfile. * Restart ``ckan`` service, read logs. a. Download and install extension from inside ``ckan`` container into ``docker_ckan_home`` volume The process is very similar to installing extensions in a source install. The only difference is -that the installation steps happen inside the running container, using the virtualenv created -inside the ckan image by CKAN's Dockerfile. +that the installation steps will happen inside the running container, and they will use the +virtualenv created inside the ckan image by CKAN's Dockerfile. The downloaded and installed files will be persisted in the named volume ``docker_ckan_home``. @@ -495,7 +498,6 @@ variables:: --------------------------- 8. Steps towards production --------------------------- - As mentioned above, some design decisions may not be suitable for a production setup. A possible path towards a production-ready environment is: From 9b3ed5a4dfc0ef5bc29a1d5fd048836c51de2699 Mon Sep 17 00:00:00 2001 From: florianm Date: Wed, 27 Sep 2017 13:26:32 +0800 Subject: [PATCH 48/51] remove duplicate set_permissions.sql in favour of setting perms on the fly --- contrib/docker/postgresql/Dockerfile | 1 - .../10_set_permissions.sql | 72 ------------------- .../install-from-docker-compose.rst | 3 - 3 files changed, 76 deletions(-) delete mode 100644 contrib/docker/postgresql/docker-entrypoint-initdb.d/10_set_permissions.sql diff --git a/contrib/docker/postgresql/Dockerfile b/contrib/docker/postgresql/Dockerfile index 8c8cef90a29..e14cbd210fd 100644 --- a/contrib/docker/postgresql/Dockerfile +++ b/contrib/docker/postgresql/Dockerfile @@ -13,6 +13,5 @@ ARG DS_RO_PASS # Include datastore setup scripts ADD /docker-entrypoint-initdb.d/00_create_datastore.sql ./00_create_datastore.sql -ADD /docker-entrypoint-initdb.d/10_set_permissions.sql ./10_set_permissions.sql ADD /docker-entrypoint-initdb.d/20_postgis_permissions.sql ./20_postgis_permissions.sql diff --git a/contrib/docker/postgresql/docker-entrypoint-initdb.d/10_set_permissions.sql b/contrib/docker/postgresql/docker-entrypoint-initdb.d/10_set_permissions.sql deleted file mode 100644 index 137110e8bf3..00000000000 --- a/contrib/docker/postgresql/docker-entrypoint-initdb.d/10_set_permissions.sql +++ /dev/null @@ -1,72 +0,0 @@ -/* -This script configures the permissions for the datastore. - -It ensures that the datastore read-only user will only be able to select from -the datastore database but has no create/write/edit permission or any -permissions on other databases. You must execute this script as a database -superuser on the PostgreSQL server that hosts your datastore database. - -For example, if PostgreSQL is running locally and the "postgres" user has the -appropriate permissions (as in the default Ubuntu PostgreSQL install), you can -run: - - paster datastore set-permissions | sudo -u postgres psql - -Or, if your PostgreSQL server is remote, you can pipe the permissions script -over SSH: - - paster datastore set-permissions | ssh dbserver sudo -u postgres psql - -*/ - --- Most of the following commands apply to an explicit database or to the whole --- 'public' schema, and could be executed anywhere. But ALTER DEFAULT --- PERMISSIONS applies to the current database, and so we must be connected to --- the datastore DB: -\connect "datastore" - --- revoke permissions for the read-only user -REVOKE CREATE ON SCHEMA public FROM PUBLIC; -REVOKE USAGE ON SCHEMA public FROM PUBLIC; - -GRANT CREATE ON SCHEMA public TO "ckan"; -GRANT USAGE ON SCHEMA public TO "ckan"; - -GRANT CREATE ON SCHEMA public TO "ckan"; -GRANT USAGE ON SCHEMA public TO "ckan"; - --- take connect permissions from main db -REVOKE CONNECT ON DATABASE "ckan" FROM "datastore_ro"; - --- grant select permissions for read-only user -GRANT CONNECT ON DATABASE "datastore" TO "datastore_ro"; -GRANT USAGE ON SCHEMA public TO "datastore_ro"; - --- grant access to current tables and views to read-only user -GRANT SELECT ON ALL TABLES IN SCHEMA public TO "datastore_ro"; - --- grant access to new tables and views by default -ALTER DEFAULT PRIVILEGES FOR USER "ckan" IN SCHEMA public - GRANT SELECT ON TABLES TO "datastore_ro"; - -CREATE OR REPLACE VIEW "_table_metadata" AS - SELECT DISTINCT - substr(md5(dependee.relname || COALESCE(dependent.relname, '')), 0, 17) AS "_id", - dependee.relname AS name, - dependee.oid AS oid, - dependent.relname AS alias_of - -- dependent.oid AS oid - FROM - pg_class AS dependee - LEFT OUTER JOIN pg_rewrite AS r ON r.ev_class = dependee.oid - LEFT OUTER JOIN pg_depend AS d ON d.objid = r.oid - LEFT OUTER JOIN pg_class AS dependent ON d.refobjid = dependent.oid - WHERE - (dependee.oid != dependent.oid OR dependent.oid IS NULL) AND - (dependee.relname IN (SELECT tablename FROM pg_catalog.pg_tables) - OR dependee.relname IN (SELECT viewname FROM pg_catalog.pg_views)) AND - dependee.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname='public') - ORDER BY dependee.oid DESC; -ALTER VIEW "_table_metadata" OWNER TO "ckan"; -GRANT SELECT ON "_table_metadata" TO "datastore_ro"; - diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 783f0d8765c..8aac6d475f8 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -174,9 +174,6 @@ a. Create and configure datastore database With running CKAN containers, execute the built-in setup scripts against the ``db`` container:: docker exec -it db psql -U ckan -f 00_create_datastore.sql - # canned, possibly outdated: - # docker exec -it db psql -U ckan -f 10_set_permissions.sql - # fresh: docker exec ckan /usr/local/bin/ckan-paster --plugin=ckan datastore set-permissions -c /etc/ckan/ckan.ini | docker exec -i db psql -U ckan The first script will create the datastore database and the datastore readonly user in the ``db`` From bbe6eefbff1749bc5cab3fea687022b2d50d0cb0 Mon Sep 17 00:00:00 2001 From: florianm Date: Wed, 27 Sep 2017 13:43:45 +0800 Subject: [PATCH 49/51] fix circleci test failure: indent test_api.py:284 --- ckan/tests/controllers/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py index 162b8289af5..40429ae7ccd 100644 --- a/ckan/tests/controllers/test_api.py +++ b/ckan/tests/controllers/test_api.py @@ -281,7 +281,7 @@ def test_api_info(self): 'http://test.ckan.net/api/3/action/datastore_search_sql?sql=SELECT * from "588dfa82-760c-45a2-b78a-e3bc314a4a9b" WHERE title LIKE 'jones'', "url: 'http://test.ckan.net/api/3/action/datastore_search'", "http://test.ckan.net/api/3/action/datastore_search?resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b&limit=5&q=title:jones", - ) + ) for url in expected_urls: assert url in page, url From 34fe6d1ef0c028f0123d29a007d98ca1da585d05 Mon Sep 17 00:00:00 2001 From: florianm Date: Wed, 27 Sep 2017 13:55:08 +0800 Subject: [PATCH 50/51] add missing template ajax_snippets/api_info.html --- ckan/templates/ajax_snippets/api_info.html | 1 + 1 file changed, 1 insertion(+) create mode 100644 ckan/templates/ajax_snippets/api_info.html diff --git a/ckan/templates/ajax_snippets/api_info.html b/ckan/templates/ajax_snippets/api_info.html new file mode 100644 index 00000000000..ba24d6cd22d --- /dev/null +++ b/ckan/templates/ajax_snippets/api_info.html @@ -0,0 +1 @@ +{# Snippet stub api_info.html #} \ No newline at end of file From 997e85a57f642a86f0da1c6a1746503a3d91e2cf Mon Sep 17 00:00:00 2001 From: florianm Date: Wed, 27 Sep 2017 14:16:50 +0800 Subject: [PATCH 51/51] drop templates/axaj_snippets/api_info.html - failing test should load datastore/.../api_info.html --- ckan/templates/ajax_snippets/api_info.html | 1 - 1 file changed, 1 deletion(-) delete mode 100644 ckan/templates/ajax_snippets/api_info.html diff --git a/ckan/templates/ajax_snippets/api_info.html b/ckan/templates/ajax_snippets/api_info.html deleted file mode 100644 index ba24d6cd22d..00000000000 --- a/ckan/templates/ajax_snippets/api_info.html +++ /dev/null @@ -1 +0,0 @@ -{# Snippet stub api_info.html #} \ No newline at end of file