Underpinnings for Python, CKAN projects in Lagoon

Makefile

@@ -79,6 +79,12 @@ BRANCH_NAME :=
 # Docker Build Context
 docker_build = docker build $(DOCKER_BUILD_PARAMS) --build-arg LAGOON_VERSION=$(LAGOON_VERSION) --build-arg IMAGE_REPO=$(CI_BUILD_TAG) -t $(CI_BUILD_TAG)/$(1) -f $(2) $(3)
 
+# Build a Python docker image. Expects as arguments:
+# 1. Python version
+# 2. Location of Dockerfile
+# 3. Path of Docker Build context
+docker_build_python = docker build $(DOCKER_BUILD_PARAMS) --build-arg LAGOON_VERSION=$(LAGOON_VERSION) --build-arg IMAGE_REPO=$(CI_BUILD_TAG) --build-arg PYTHON_VERSION=$(1) -t $(CI_BUILD_TAG)/python:$(2) -f $(3) $(4)
+
 # Build a PHP docker image. Expects as arguments:
 # 1. PHP version
 # 2. PHP version and type of image (ie 7.0-fpm, 7.0-cli etc)
@@ -108,6 +114,7 @@ images :=     oc \
 							mariadb-galera \
 							mariadb-galera-drupal \
 							postgres \
+							postgres-ckan \
 							postgres-drupal \
 							oc-build-deploy-dind \
 							commons \
@@ -156,6 +163,7 @@ build/mariadb-drupal: build/mariadb images/mariadb-drupal/Dockerfile
 build/mariadb-galera: build/commons images/mariadb-galera/Dockerfile
 build/mariadb-galera-drupal: build/mariadb-galera images/mariadb-galera-drupal/Dockerfile
 build/postgres: build/commons images/postgres/Dockerfile
+build/postgres-ckan: build/postgres images/postgres-ckan/Dockerfile
 build/postgres-drupal: build/postgres images/postgres-drupal/Dockerfile
 build/commons: images/commons/Dockerfile
 build/nginx: build/commons images/nginx/Dockerfile
@@ -178,6 +186,39 @@ build/curator: build/commons images/curator/Dockerfile
 build/oc-build-deploy-dind: build/oc images/oc-build-deploy-dind
 build/athenapdf-service: images/athenapdf-service/Dockerfile
 
+
+#######
+####### Python Images
+#######
+####### Python Images are alpine linux based Python images.
+
+pythonimages :=  python__2.7 \
+								 python__3.7 \
+								 python__2.7-ckan \
+								 python__2.7-ckandatapusher
+
+build-pythonimages = $(foreach image,$(pythonimages),build/$(image))
+
+# Define the make recepie for all base images
+$(build-pythonimages): build/commons
+	$(eval clean = $(subst build/python__,,$@))
+	$(eval version = $(word 1,$(subst -, ,$(clean))))
+	$(eval type = $(word 2,$(subst -, ,$(clean))))
+# this fills variables only if $type is existing, if not they are just empty
+	$(eval type_dash = $(if $(type),-$(type)))
+# Call the docker build
+	$(call docker_build_python,$(version),$(version)$(type_dash),images/python$(type_dash)/Dockerfile,images/python$(type_dash))
+# Touch an empty file which make itself is using to understand when the image has been last build
+	touch $@
+
+base-images-with-versions += $(pythonimages)
+s3-images += python
+
+build/python__2.7 build/python__3.7: images/commons
+build/python__2.7-ckan: build/python__2.7
+build/python__2.7-ckandatapusher: build/python__2.7
+
+
 #######
 ####### PHP Images
 #######
@@ -244,7 +285,9 @@ solrimages := 	solr__5.5 \
 								solr__7.5 \
 								solr__5.5-drupal \
 								solr__6.6-drupal \
-								solr__7.5-drupal
+								solr__7.5-drupal \
+								solr__5.5-ckan \
+								solr__6.6-ckan
 
 
 build-solrimages = $(foreach image,$(solrimages),build/$(image))
@@ -268,6 +311,8 @@ build/solr__5.5  build/solr__6.6 build/solr__7.5: images/commons
 build/solr__5.5-drupal: build/solr__5.5
 build/solr__6.6-drupal: build/solr__6.6
 build/solr__7.5-drupal: build/solr__7.5
+build/solr__5.5-ckan: build/solr__5.5
+build/solr__6.6-ckan: build/solr__6.6
 
 #######
 ####### Node Images
@@ -476,7 +521,7 @@ $(run-rest-tests): minishift build/node__6-builder build/node__8-builder build/o
 		IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) up -d $(deployment-test-services-rest)
 		IMAGE_REPO=$(CI_BUILD_TAG) docker exec -i $$(docker-compose -p $(CI_BUILD_TAG) ps -q tests) ansible-playbook /ansible/tests/$(testname).yaml $(testparameter)
 
-tests/drupal tests/drupal-postgres tests/drupal-galera: minishift build/varnish-drupal build/solr__5.5-drupal build/nginx-drupal build/redis build/php__5.6-cli-drupal build/php__7.0-cli-drupal build/php__7.1-cli-drupal build/php__7.2-cli-drupal build/php__7.3-cli-drupal build/api-db build/postgres-drupal build/mariadb-drupal build/oc-build-deploy-dind $(foreach image,$(deployment-test-services-rest),build/$(image)) build/drush-alias push-minishift
+tests/drupal tests/drupal-postgres tests/drupal-galera: minishift build/varnish-drupal build/solr__5.5-drupal build/nginx-drupal build/redis build/php__5.6-cli-drupal build/php__7.0-cli-drupal build/php__7.1-cli-drupal build/php__7.2-cli-drupal build/php__7.3-cli-drupal build/api-db build/postgres-drupal build/mariadb-drupal build/postgres-ckan build/oc-build-deploy-dind $(foreach image,$(deployment-test-services-rest),build/$(image)) build/drush-alias push-minishift
 		$(eval testname = $(subst tests/,,$@))
 		IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) up -d $(deployment-test-services-rest) drush-alias
 		IMAGE_REPO=$(CI_BUILD_TAG) docker exec -i $$(docker-compose -p $(CI_BUILD_TAG) ps -q tests) ansible-playbook /ansible/tests/$(testname).yaml $(testparameter)

images/postgres-ckan/90-datastore-user.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE USER ckan_datastore with encrypted password 'ckan';
+    GRANT ALL PRIVILEGES ON DATABASE ckan TO ckan_datastore;
+EOSQL

images/postgres-ckan/Dockerfile

@@ -0,0 +1,12 @@
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/postgres
+
+# change log_min_error_statement and log_min_messages from `error` to `log` as drupal is prone to cause some errors which are all logged (yes `log` is a less verbose mode than `error`) 
+RUN sed -i "s/#log_min_error_statement = error/log_min_error_statement = log/" /usr/local/share/postgresql/postgresql.conf.sample \
+    && sed -i "s/#log_min_messages = warning/log_min_messages = log/" /usr/local/share/postgresql/postgresql.conf.sample
+
+ENV POSTGRES_PASSWORD=ckan \
+    POSTGRES_USER=ckan \
+    POSTGRES_DB=ckan
+
+COPY 90-datastore-user.sh /docker-entrypoint-initdb.d/

images/python-ckan/Dockerfile

@@ -0,0 +1,22 @@
+ARG PYTHON_VERSION
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/python:${PYTHON_VERSION}
+
+RUN apk update \
+    && apk upgrade \
+    && apk add --no-cache git \
+    libpq \
+    postgresql-dev \
+    gcc \
+    musl-dev \
+    libmagic \
+    libxslt-dev \
+    libxml2-dev \
+    libffi-dev
+
+RUN mkdir -p /app/ckan/default \
+    && fix-permissions /app/ckan/default
+
+RUN virtualenv --no-site-packages /app/ckan/default \
+    && . /app/ckan/default/bin/activate \
+    && pip install setuptools==20.4

images/python-ckandatapusher/Dockerfile

@@ -0,0 +1,34 @@
+ARG PYTHON_VERSION
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/python:${PYTHON_VERSION}
+
+RUN apk update \
+    && apk upgrade \
+    && apk add --no-cache git \
+    libpq \
+    postgresql-dev \
+    gcc \
+    musl-dev \
+    libmagic \
+    libxslt-dev \
+    libxml2-dev \
+    libffi-dev \
+    pcre-dev
+
+RUN virtualenv /app/ckan/datapusher
+
+RUN mkdir -p /app/ckan/datapusher/src \
+    && mkdir -p /etc/ckan \
+    && fix-permissions /app/ckan \
+    && ln -s /app/ckan /usr/lib/ckan \
+    && . /app/ckan/datapusher/bin/activate \
+    && pip install uwsgi \
+    && cd /app/ckan/datapusher/src \
+    && git clone -b 0.0.14 https://github.com/ckan/datapusher.git \
+    && cd datapusher \
+    && /app/ckan/datapusher/bin/pip install -r requirements.txt \
+    && /app/ckan/datapusher/bin/python setup.py develop \
+    && cp deployment/datapusher.wsgi /etc/ckan/ \
+    && cp deployment/datapusher_settings.py /etc/ckan/
+
+CMD ["sh", "-c", ". /app/ckan/datapusher/bin/activate && uwsgi --http :8800 --wsgi-file /etc/ckan/datapusher.wsgi"]

images/python/80-shell-timeout.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# If we are running within kubernetes, set a shell timeout of 10mins.
+# We do that so old shells are closed and we can idle the cli container
+if [ $KUBERNETES_PORT ]; then
+  TMOUT=600
+fi

images/python/Dockerfile

@@ -0,0 +1,37 @@
+ARG PYTHON_VERSION
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/commons as commons
+FROM python:${PYTHON_VERSION}-alpine
+
+LABEL maintainer="amazee.io"
+ENV LAGOON=python
+
+# Copy commons files
+COPY --from=commons /lagoon /lagoon
+COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/
+COPY --from=commons /sbin/tini /sbin/
+COPY --from=commons /home /home
+
+RUN chmod g+w /etc/passwd \
+    && mkdir -p /home
+
+ENV TMPDIR=/tmp \
+    TMP=/tmp \
+    HOME=/home \
+    # When Bash is invoked via `sh` it behaves like the old Bourne Shell and sources a file that is given in `ENV`
+    ENV=/home/.bashrc \
+    # When Bash is invoked as non-interactive (like `bash -c command`) it sources a file that is given in `BASH_ENV`
+    BASH_ENV=/home/.bashrc
+
+RUN apk update \
+    && apk upgrade \
+    && apk add --no-cache python-dev \
+    py2-pip \
+    py2-virtualenv
+
+# Make sure shells are not running forever
+COPY 80-shell-timeout.sh /lagoon/entrypoints/
+RUN echo "source /lagoon/entrypoints/80-shell-timeout.sh" >> /home/.bashrc
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
+CMD ["python"]

images/solr-ckan/Dockerfile

@@ -0,0 +1,10 @@
+ARG SOLR_MAJ_MIN_VERSION
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/solr:${SOLR_MAJ_MIN_VERSION}
+ARG SOLR_MAJ_MIN_VERSION
+
+COPY solr${SOLR_MAJ_MIN_VERSION} /solr-conf
+
+RUN precreate-core ckan /solr-conf
+
+CMD ["solr-foreground"]

images/solr-ckan/solr5.5/conf/elevate.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<!--
+ This file allows you to boost certain search items to the top of search
+ results. You can find out an item's ID by searching directly on the Solr
+ server. Search API generally constructs item IDs (esp. for entities) as:
+     $document->id = "$site_hash-$index_id-$datasource:$entity_id:$language_id";
+
+ If you want this file to be automatically re-loaded when a Solr commit takes
+ place (e.g., if you have an automatic script active which updates elevate.xml
+ according to newly-indexed data), place it into Solr's data/ directory.
+ Otherwise, place it with the other configuration files into the conf/
+ directory.
+
+ See http://wiki.apache.org/solr/QueryElevationComponent for more information.
+-->
+
+<elevate>
+<!-- Example for ranking the node #789 first in searches for "example query": -->
+<!--
+  <query text="example query">
+    <doc id="ab12cd34-site_index-entity:789:en" />
+ </query>
+-->
+<!-- Multiple <query> elements can be specified, contained in one <elevate>. -->
+<!-- <query text="...">...</query> -->
+</elevate>

images/solr-ckan/solr5.5/conf/mapping-ISOLatin1Accent.txt

@@ -0,0 +1,14 @@
+# This file contains character mappings for the default fulltext field type.
+# The source characters (on the left) will be replaced by the respective target
+# characters before any other processing takes place.
+# Lines starting with a pound character # are ignored.
+#
+# For sensible defaults, use the mapping-ISOLatin1Accent.txt file distributed
+# with the example application of your Solr version.
+#
+# Examples:
+#   "À" => "A"
+#   "\u00c4" => "A"
+#   "\u00c4" => "\u0041"
+#   "æ" => "ae"
+#   "\n" => " "

images/solr-ckan/solr5.5/conf/protwords.txt

@@ -0,0 +1,7 @@
+#-----------------------------------------------------------------------
+# This file blocks words from being operated on by the stemmer and word delimiter.
+&
+<
+>
+'
+"