WIP: Implement s2i support

10.1/Dockerfile

@@ -1,4 +1,4 @@
-FROM centos:centos7
+FROM centos/s2i-base-centos7
 
 # MariaDB image for OpenShift.
 #
@@ -11,6 +11,7 @@ FROM centos:centos7
 #  * $MYSQL_ROOT_PASSWORD (Optional) - Password for the 'root' MySQL account
 
 ENV MYSQL_VERSION=10.1 \
+    APP_DATA=/opt/app-root/src \
     HOME=/var/lib/mysql
 
 ENV SUMMARY="MariaDB 10.1 SQL database server" \
@@ -32,7 +33,7 @@ EXPOSE 3306
 # safe in the future. This should *never* change, the last test is there
 # to make sure of that.
 RUN yum install -y centos-release-scl-rh && \
-    INSTALL_PKGS="rsync tar gettext hostname bind-utils rh-mariadb101" && \
+    INSTALL_PKGS="rsync tar gettext hostname bind-utils groff-base rh-mariadb101" && \
     yum install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
     rpm -V $INSTALL_PKGS && \
     yum clean all && \
@@ -51,6 +52,8 @@ ENV BASH_ENV=${CONTAINER_SCRIPTS_PATH}/scl_enable \
     ENV=${CONTAINER_SCRIPTS_PATH}/scl_enable \
     PROMPT_COMMAND=". ${CONTAINER_SCRIPTS_PATH}/scl_enable"
 
+ADD s2i/bin/ $STI_SCRIPTS_PATH
+
 ADD root /
 
 # this is needed due to issues with squash

10.1/Dockerfile.rhel7

@@ -1,4 +1,4 @@
-FROM rhel7
+FROM rhscl/s2i-base-rhel7
 
 # MariaDB image for OpenShift.
 #
@@ -11,6 +11,7 @@ FROM rhel7
 #  * $MYSQL_ROOT_PASSWORD (Optional) - Password for the 'root' MySQL account
 
 ENV MYSQL_VERSION=10.1 \
+    APP_DATA=/opt/app-root/src \
     HOME=/var/lib/mysql
 
 ENV SUMMARY="MariaDB 10.1 SQL database server" \
@@ -41,7 +42,7 @@ EXPOSE 3306
 RUN yum install -y yum-utils gettext hostname && \
     yum-config-manager --enable rhel-server-rhscl-7-rpms && \
     yum-config-manager --enable rhel-7-server-optional-rpms && \
-    INSTALL_PKGS="rsync tar gettext hostname bind-utils rh-mariadb101" && \
+    INSTALL_PKGS="rsync tar gettext hostname bind-utils groff-base rh-mariadb101" && \
     yum install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
     rpm -V $INSTALL_PKGS && \
     yum clean all && \
@@ -60,6 +61,8 @@ ENV BASH_ENV=${CONTAINER_SCRIPTS_PATH}/scl_enable \
     ENV=${CONTAINER_SCRIPTS_PATH}/scl_enable \
     PROMPT_COMMAND=". ${CONTAINER_SCRIPTS_PATH}/scl_enable"
 
+ADD s2i/bin/ $STI_SCRIPTS_PATH
+
 ADD root /
 
 # this is needed due to issues with squash

10.1/root/usr/bin/run-mysqld

@@ -4,28 +4,17 @@ export_vars=$(cgroup-limits); export $export_vars
 source ${CONTAINER_SCRIPTS_PATH}/common.sh
 set -eu
 
-[ -f ${CONTAINER_SCRIPTS_PATH}/validate-variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate-variables.sh
-
-# Process the MySQL configuration files
-log_info 'Processing MySQL configuration files ...'
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf
+# pre-init files
+process_extending_files ${APP_DATA}/mysql-pre-init/ ${CONTAINER_SCRIPTS_PATH}/pre-init/
 
 if [ ! -d "$MYSQL_DATADIR/mysql" ]; then
   initialize_database "$@"
 else
   start_local_mysql "$@"
 fi
 
-if [ -f ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh ]; then
-  log_info 'Setting passwords ...'
-  source ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh
-fi
-if [ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ]; then
-  log_info 'Sourcing post-init.sh ...'
-  source ${CONTAINER_SCRIPTS_PATH}/post-init.sh
-fi
+# init files
+process_extending_files ${APP_DATA}/mysql-init/ ${CONTAINER_SCRIPTS_PATH}/init/
 
 # Restart the MySQL server with public IP bindings
 shutdown_local_mysql

10.1/root/usr/bin/run-mysqld-master

@@ -8,20 +8,12 @@ set -eu
 
 export MYSQL_RUNNING_AS_MASTER=1
 
-[ -f ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh
-[ -f ${CONTAINER_SCRIPTS_PATH}/validate_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_variables.sh
-
 # The 'server-id' for master needs to be constant
 export MYSQL_SERVER_ID=1
 log_info "The 'master' server-id is ${MYSQL_SERVER_ID}"
 
-# Process the MySQL configuration files
-log_info 'Processing MySQL configuration files ...'
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-master.cnf.template > /etc/my.cnf.d/master.cnf
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-repl-gtid.cnf.template > /etc/my.cnf.d/repl-gtid.cnf
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf
+# pre-init files
+process_extending_files ${APP_DATA}/mysql-pre-init/ ${CONTAINER_SCRIPTS_PATH}/pre-init/
 
 if [ ! -d "$MYSQL_DATADIR/mysql" ]; then
   initialize_database "$@"
@@ -39,8 +31,8 @@ mysql $mysql_flags <<EOSQL
   FLUSH PRIVILEGES;
 EOSQL
 
-log_info 'Sourcing post-init.sh ...'
-[ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ] && source ${CONTAINER_SCRIPTS_PATH}/post-init.sh
+# init files
+process_extending_files ${APP_DATA}/mysql-init/ ${CONTAINER_SCRIPTS_PATH}/init/
 
 # Restart the MySQL server with public IP bindings
 shutdown_local_mysql

10.1/root/usr/bin/run-mysqld-slave

@@ -13,18 +13,12 @@ fi
 
 export MYSQL_RUNNING_AS_SLAVE=1
 
-[ -f ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh
-
 # Generate the unique 'server-id' for this master
 export MYSQL_SERVER_ID=$(server_id)
 log_info "The 'slave' server-id is ${MYSQL_SERVER_ID}"
 
-# Process the MySQL configuration files
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-slave.cnf.template > /etc/my.cnf.d/slave.cnf
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-repl-gtid.cnf.template > /etc/my.cnf.d/repl-gtid.cnf
-envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf
+# pre-init files
+process_extending_files ${APP_DATA}/mysql-pre-init/ ${CONTAINER_SCRIPTS_PATH}/pre-init/
 
 # Initialize MySQL database and wait for the MySQL master to accept
 # connections.
@@ -48,8 +42,8 @@ mysql $mysql_flags <<EOSQL
   START SLAVE;
 EOSQL
 
-log_info 'Sourcing post-init.sh ...'
-[ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ] && source ${CONTAINER_SCRIPTS_PATH}/post-init.sh
+# init files
+process_extending_files ${APP_DATA}/mysql-init/ ${CONTAINER_SCRIPTS_PATH}/init/
 
 # Restart the MySQL server with public IP bindings
 shutdown_local_mysql

10.1/root/usr/libexec/container-setup

@@ -50,6 +50,11 @@ mkdir -p /var/lib/mysql/data
 chown -R mysql:0 /var/lib/mysql
 restorecon -R /var/lib/mysql
 
+# setup directory for s2i app
+chown -R mysql:0 ${APP_DATA}/..
+restorecon -R ${APP_DATA}/..
+chmod g+w -R ${APP_DATA}/..
+
 # Loosen permission bits for group to avoid problems running container with
 # arbitrary UID
 # When only specifying user, group is 0, that's why /var/lib/mysql must have

10.1/root/usr/share/container-scripts/mysql/README.md

@@ -115,6 +115,7 @@ values stored in the variables and the actual passwords. Whenever a database
 container starts it will reset the passwords to the values stored in the
 environment variables.
 
+
 Default my.cnf file
 -------------------
 With environment variables we are able to customize a lot of different parameters
@@ -124,6 +125,56 @@ variable with the full path of the file you wish to use. For example, the defaul
 location is `/etc/my.cnf` but you can change it to `/etc/mysql/my.cnf` by setting
  `MYSQL_DEFAULTS_FILE=/etc/mysql/my.cnf`
 
+
+Extending image
+---------------------------------
+This image can be extended using [source-to-image](https://github.com/openshift/source-to-image).
+
+For example to build customized MariaDB database image `my-mariadb-centos7` with configuration in `~/image-configuration/` run:
+
+```
+$ s2i build ~/image-configuration/ centos/mariadb-100-centos7 my-mariadb-centos7
+```
+
+The directory passed to `s2i build` can contain these directories:
+- `mysql-cfg/`
+  - when starting the container, files from this directory will be used as a configuration for the `mysqld` daemon
+    - `envsubst` command is run on this file to still allow customization of the image using environmental variables
+
+- `mysql-pre-init/`
+  - shell scripts (`*.sh`) available in this directory are sourced before `mysqld` daemon is started
+
+- `mysql-init/`
+  - shell scripts (`*.sh`) available in this directory are sourced when `mysqld` daemon is started locally
+    - in this phase, use `${mysql_flags}` to connect to the locally running daemon, for example `mysql $mysql_flags < dump.sql`
+
+Variables that can be used in the scripts provided to s2i:
+
+- `$mysql_flags` -- arguments for the `mysql` tool that will connect to the locally running `mysqld` during initialization
+- `$MYSQL_RUNNING_AS_MASTER` -- variable defined when the container is run with `run-mysqld-master` command
+- `$MYSQL_RUNNING_AS_SLAVE` -- variable defined when the container is run with `run-mysqld-slave` command
+- `$MYSQL_DATADIR_FIRST_INIT` -- variable defined when the container was initialized from the empty data dir
+
+During `s2i build` all provided files are copied into `/opt/app-root/src` directory into the resulting image. If some configuration files are present in the destination directory, files with the same name are overwritten. Also only one file with the same name can be used for customization and user provided files are preferred over default files in `/usr/share/container-scripts/mysql/`- so it is possible to overwrite them.
+
+Same configuration directory structure can be used to customize the image every time the image is started using `docker run`. The directory has to be mounted into `/opt/app-root/src/` in the image (`-v ./image-configuration/:/opt/app-root/src/`). This overwrites customization built into the image.
+
+It is also possible to use a `Dockerfile` to add the additional files into the new image. This is in particular helpful when we need to change the user for some commands (like installing additional RPMs). A `Dockerfile` that installs an additional RPM and adds a directory `./image-configuration` as s2i source, may look like this:
+
+```
+FROM rhscl/mariadb-101-rhel7
+USER 0
+RUN INSTALL_PKGS="openssh-server" && \
+    yum install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
+    rpm -V $INSTALL_PKGS && \
+    yum clean all
+USER 27
+COPY image-configuration /opt/app-root/src
+```
+
+To build such a Dockerfile, use either appropriate strategy in OpenShift or `docker build` command directly.
+
+
 Changing the replication binlog_format
 --------------------------------------
 Some applications may wish to use `row` binlog_formats (for example, those built

10.1/root/usr/share/container-scripts/mysql/common.sh

@@ -38,6 +38,9 @@ export MYSQL_INNODB_BUFFER_POOL_SIZE=${MYSQL_INNODB_BUFFER_POOL_SIZE:-$innodb_bu
 export MYSQL_INNODB_LOG_FILE_SIZE=${MYSQL_INNODB_LOG_FILE_SIZE:-$innodb_log_file_size}
 export MYSQL_INNODB_LOG_BUFFER_SIZE=${MYSQL_INNODB_LOG_BUFFER_SIZE:-$innodb_log_buffer_size}
 
+# this stores whether the database was initialized from empty datadir
+export MYSQL_DATADIR_FIRST_INIT=false
+
 # Be paranoid and stricter than we should be.
 # https://dev.mysql.com/doc/refman/en/identifiers.html
 mysql_identifier_regex='^[a-zA-Z0-9_]+$'
@@ -142,6 +145,9 @@ mysql $mysql_flags <<EOSQL
 EOSQL
   fi
   log_info 'Initialization finished'
+
+  # remember that the database was just initialized, it may be needed on other places
+  export MYSQL_DATADIR_FIRST_INIT=true
 }
 
 # The 'server_id' number for slave needs to be within 1-4294967295 range.
@@ -162,3 +168,50 @@ function wait_for_mysql_master() {
     sleep 1
   done
 }
+
+function get_matched_files() {
+  local custom_dir default_dir
+  custom_dir="$1"
+  default_dir="$2"
+  files_matched="$3"
+  find "$default_dir" -maxdepth 1 -type f -name "$files_matched" -printf "%f\n"
+  [ -d "$custom_dir" ] && find "$custom_dir" -maxdepth 1 -type f -name "$files_matched" -printf "%f\n"
+}
+
+# process extending files in $1 and $2 directories
+# - source all *.sh files
+#   (if there are files with same name source only file from $1)
+function process_extending_files() {
+  local custom_dir default_dir
+  custom_dir=$1
+  default_dir=$2
+
+  while read filename ; do
+    # Custom file is prefered
+    if [ -f $custom_dir/$filename ]; then
+      source $custom_dir/$filename
+    else
+      source $default_dir/$filename
+    fi
+  done <<<"$(get_matched_files "$custom_dir" "$default_dir" '*.sh' | sort -u)"
+}
+
+# process extending config files in $1 and $2 directories
+# - expand variables in *.cnf and copy the files into /etc/my.cnf.d directory
+#   (if there are files with same name source only file from $1)
+function process_extending_config_files() {
+  local custom_dir default_dir
+  custom_dir=$1
+  default_dir=$2
+
+  get_matched_files "$custom_dir" "$default_dir" '*.cnf' | sort -u | while read filename ; do
+    # Custom file is prefered
+    if [ -f $custom_dir/$filename ]; then
+       envsubst < $custom_dir/$filename > /etc/my.cnf.d/$filename
+    else
+       envsubst < $default_dir/$filename > /etc/my.cnf.d/$filename
+    fi
+  done
+}
+
+

10.1/root/usr/share/container-scripts/mysql/init/50-passwd-change.sh

@@ -0,0 +1,32 @@
+password_change() {
+  log_info 'Setting passwords ...'
+
+  # Set the password for MySQL user and root everytime this container is started.
+  # This allows to change the password by editing the deployment configuration.
+  if [[ -v MYSQL_USER && -v MYSQL_PASSWORD ]]; then
+    mysql $mysql_flags <<EOSQL
+      SET PASSWORD FOR '${MYSQL_USER}'@'%' = PASSWORD('${MYSQL_PASSWORD}');
+EOSQL
+  fi
+
+  # The MYSQL_ROOT_PASSWORD is optional, therefore we need to either enable remote
+  # access with a password if the variable is set or disable remote access otherwise.
+  if [ -v MYSQL_ROOT_PASSWORD ]; then
+    # create a user if it doesn't exist and set its password
+    mysql $mysql_flags <<EOSQL
+      CREATE USER IF NOT EXISTS 'root'@'%';
+      GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' WITH GRANT OPTION;
+EOSQL
+  else
+    mysql $mysql_flags <<EOSQL
+      DROP USER IF EXISTS 'root'@'%';
+      FLUSH PRIVILEGES;
+EOSQL
+  fi
+}
+
+if ! [ -v MYSQL_RUNNING_AS_SLAVE ] ; then
+  password_change
+fi
+
+unset -f password_change

10.1/root/usr/share/container-scripts/mysql/validate-variables.sh → 10.1/root/usr/share/container-scripts/mysql/pre-init/20-validate-variables.sh

@@ -75,4 +75,6 @@ function validate_variables() {
   fi
 }
 
-validate_variables
+if ! [ -v MYSQL_RUNNING_AS_SLAVE ] ; then
+  validate_variables
+fi

10.1/root/usr/share/container-scripts/mysql/validate-replication-variables.sh → 10.1/root/usr/share/container-scripts/mysql/pre-init/25-validate-replication-variables.sh

@@ -8,11 +8,14 @@ function validate_replication_variables() {
     echo "  MYSQL_MASTER_USER"
     echo "  MYSQL_MASTER_PASSWORD"
     echo
+    return 1
   fi
   [[ "$MYSQL_DATABASE" =~ $mysql_identifier_regex ]] || usage "Invalid database name"
   [[ "$MYSQL_MASTER_USER"     =~ $mysql_identifier_regex ]] || usage "Invalid MySQL master username"
   [ ${#MYSQL_MASTER_USER} -le 16 ] || usage "MySQL master username too long (maximum 16 characters)"
   [[ "$MYSQL_MASTER_PASSWORD" =~ $mysql_password_regex   ]] || usage "Invalid MySQL master password"
 }
 
-validate_replication_variables
+if [ -v MYSQL_RUNNING_AS_MASTER ] || [ -v MYSQL_RUNNING_AS_SLAVE ] ; then
+  validate_replication_variables
+fi

10.1/root/usr/share/container-scripts/mysql/pre-init/30-base-config.sh

@@ -0,0 +1,3 @@
+log_info 'Processing basic MySQL configuration files ...'
+envsubst < ${CONTAINER_SCRIPTS_PATH}/pre-init/my-base.cnf.template > /etc/my.cnf.d/base.cnf
+