Provide mariadb 10.1 and 10.2, fixes #1221 (#1318)

cmd/ddev/cmd/config.go

@@ -98,6 +98,9 @@ var (
 
 	// omitContainersArg allows user to determine value of omit_containers
 	omitContainersArg string
+
+	// mariadbVersionArg is mariadb version 10.1 or 10.2
+	mariaDBVersionArg string
 )
 
 var providerName = ddevapp.ProviderDefault
@@ -201,6 +204,7 @@ func init() {
 	ConfigCommand.Flags().BoolVar(&dbWorkingDirDefaultArg, "db-working-dir-default", false, "Unsets a db service working directory override")
 	ConfigCommand.Flags().BoolVar(&dbaWorkingDirDefaultArg, "dba-working-dir-default", false, "Unsets a dba service working directory override")
 	ConfigCommand.Flags().BoolVar(&workingDirDefaultsArg, "working-dir-defaults", false, "Unsets all service working directory overrides")
+	ConfigCommand.Flags().StringVar(&mariaDBVersionArg, "mariadb-version", "10.2", "mariadb version to use")
 
 	// projectname flag exists for backwards compatability.
 	ConfigCommand.Flags().StringVar(&projectNameArg, "projectname", "", projectNameUsage)
@@ -352,6 +356,9 @@ func handleMainConfigArgs(cmd *cobra.Command, args []string, app *ddevapp.DdevAp
 		app.RouterHTTPSPort = httpsPortArg
 	}
 
+	if mariaDBVersionArg != "" {
+		app.MariaDBVersion = mariaDBVersionArg
+	}
 	// This bool flag is false by default, so only use the value if the flag was explicity set.
 	if cmd.Flag("xdebug-enabled").Changed {
 		app.XdebugEnabled = xdebugEnabledArg

cmd/ddev/cmd/describe.go

@@ -88,6 +88,7 @@ func renderAppDescribe(desc map[string]interface{}) (string, error) {
 			dbTable.AddRow("Database name:", dbinfo["dbname"])
 			dbTable.AddRow("Host:", dbinfo["host"])
 			dbTable.AddRow("Port:", dbinfo["port"])
+			dbTable.AddRow("MariaDB version", dbinfo["mariadb_version"])
 			output = output + fmt.Sprint(dbTable)
 			output = output + fmt.Sprintf("\nTo connect to mysql from your host machine, use port %d on %s.\nFor example: mysql --host=%s --port=%d --user=db --password=db --database=db", dbinfo["published_port"], dockerIP, dockerIP, dbinfo["published_port"])
 		}

cmd/ddev/cmd/version_test.go

@@ -18,7 +18,7 @@ func TestVersion(t *testing.T) {
 	assert.Contains(string(out), version.WebImg)
 	assert.Contains(string(out), version.WebTag)
 	assert.Contains(string(out), version.DBImg)
-	assert.Contains(string(out), version.DBTag)
+	assert.Contains(string(out), version.GetDBImage())
 	assert.Contains(string(out), version.DBAImg)
 	assert.Contains(string(out), version.DBATag)
 	assert.Contains(string(out), version.DDevTLD)

containers/ddev-dbserver/10.1/Dockerfile

@@ -0,0 +1,31 @@
+FROM alpine:3.7
+
+ENV MYSQL_DATABASE db
+ENV MYSQL_USER db
+ENV MYSQL_PASSWORD db
+ENV MYSQL_ROOT_PASSWORD root
+ENV MARIADB_VERSION 10.1
+
+# Install mariadb and other packages
+RUN apk add --no-cache mariadb mariadb-client bash tzdata shadow sudo
+# Remove the installed version as we need to set up our own from scratch
+
+RUN rm -rf /var/lib/mysql/* /etc/mysql
+RUN mkdir -p /var/lib/mysql && chmod 777 /var/lib/mysql
+
+ADD files /
+
+RUN chmod ugo+x /healthcheck.sh
+
+# Security-sensitive changes: Make sure our start script can do what is needed
+# But make sure these are right
+RUN chmod ugo+wx /mnt /var/tmp
+RUN chmod -R ugo+wx /var/log /var/tmp/mysqlbase /etc/mysql/conf.d
+RUN ln -s /dev/stderr /var/log/mysqld.err
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+EXPOSE 3306
+# The following line overrides any cmd entry
+CMD []
+HEALTHCHECK --interval=2s --retries=30 CMD ["/healthcheck.sh"]

containers/ddev-dbserver/10.1/Makefile

@@ -0,0 +1,53 @@
+# Makefile for a standard repo with associated container
+
+##### These variables need to be adjusted in most repositories #####
+
+# This repo's root import path (under GOPATH).
+# PKG := github.com/drud/repo_name
+
+# Docker repo for a push
+DOCKER_REPO ?= drud/ddev-dbserver
+
+MARIADB_VERSION=10.1
+
+# Upstream repo used in the Dockerfile
+# UPSTREAM_REPO ?= oraclelinux:7.3
+
+# Top-level directories to build
+#SRC_DIRS := files drudapi secrets utils
+
+# Optional to docker build
+# DOCKER_ARGS = --build-arg MYSQL_PACKAGE_VERSION=5.7.17-1
+
+
+# VERSION can be set by
+  # Default: git tag
+  # make command line: make VERSION=0.9.0
+# It can also be explicitly set in the Makefile as commented out below.
+
+# This version-strategy uses git tags to set the version string
+# VERSION can be overridden on make commandline: make VERSION=0.9.1 push
+ifndef VERSION
+	VERSION := $(shell git describe --tags --always --dirty)
+endif
+override VERSION := $(VERSION)-$(MARIADB_VERSION)
+
+#
+# This version-strategy uses a manual value to set the version string
+#VERSION := 1.2.3
+
+# Each section of the Makefile is included from standard components below.
+# If you need to override one, import its contents below and comment out the
+# include. That way the base components can easily be updated as our general needs
+# change.
+#include build-tools/makefile_components/base_build_go.mak
+include ../../../build-tools/makefile_components/base_build_python-docker.mak
+include ../../../build-tools/makefile_components/base_container.mak
+include ../../../build-tools/makefile_components/base_push.mak
+#include build-tools/makefile_components/base_test_go.mak
+#include ../../build-tools/makefile_components/base_test_python.mak
+
+build: container
+
+test: container
+	cd .. &&  ./test/testserver.sh $(DOCKER_REPO):$(VERSION) $(MARIADB_VERSION)

containers/ddev-dbserver/10.1/files/docker-entrypoint.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+set -x
+set -eu
+set -o pipefail
+
+SOCKET=/var/tmp/mysql.sock
+
+# Wait for mysql server to be ready.
+function serverwait {
+	for i in {60..0};
+	do
+        if mysqladmin ping -uroot --socket=$SOCKET >/dev/null 2>&1; then
+            return 0
+        fi
+        # Test to make sure we got it started in the first place. kill -s 0 just tests to see if process exists.
+        if ! kill -s 0 $pid 2>/dev/null; then
+            echo "MariaDB initialization startup failed"
+            return 2
+        fi
+        echo "MariaDB initialization startup process in progress... Try# $i"
+        sleep 1
+	done
+	return 1
+}
+
+# If we have a restore_snapshot arg, get the snapshot directory
+# otherwise, fail and abort startup
+if [ $# = "2" -a "${1:-}" = "restore_snapshot" ] ; then
+  snapshot_dir="/mnt/ddev_config/db_snapshots/${2:-nothingthere}"
+  if [ -d "$snapshot_dir" ] ; then
+    echo "Restoring from snapshot directory $snapshot_dir"
+    sudo rm -rf /var/lib/mysql/*
+  else
+    echo "$snapshot_dir does not exist, not attempting restore of snapshot"
+    unset snapshot_dir
+    exit 101
+  fi
+fi
+
+
+if [ -d /var/lib/mysql/mysql ]; then
+    database_mariadb_version=10.1
+    if [ -f /var/lib/mysql/db_mariadb_version.txt ]; then
+       database_mariadb_version=$(cat /var/lib/mysql/db_mariadb_version.txt)
+    fi
+
+    if [  "$MARIADB_VERSION" == "10.1" ] && [ "$database_mariadb_version" != "10.1" ]; then
+       echo "Can't start MariaDB 10.1 server with a database from $database_mariadb_version."
+       echo "Please export your database, remove it completely, and retry if you need to start it with MariaDB $MARIADB_VERSION."
+       exit 102
+    fi
+fi
+
+sudo chown -R "$UID:$(id -g)" /var/lib/mysql
+
+# If we have extra mariadb cnf files,, copy them to where they go.
+if [ -d /mnt/ddev_config/mysql -a "$(echo /mnt/ddev_config/mysql/*.cnf)" != "/mnt/ddev_config/mysql/*.cnf" ] ; then
+  cp /mnt/ddev_config/mysql/*.cnf /etc/mysql/conf.d
+  chmod ugo-w /etc/mysql/conf.d/*
+fi
+
+# If mariadb has not been initialized, copy in the base image from either the default starter image (/var/tmp/mysqlbase)
+# or from a provided $snapshot_dir.
+if [ ! -d "/var/lib/mysql/mysql" ]; then
+    target=${snapshot_dir:-/var/tmp/mysqlbase/}
+    name=$(basename $target)
+    sudo rm -rf /var/lib/mysql/* /var/lib/mysql/.[a-z]* && sudo chmod -R ugo+w /var/lib/mysql
+    sudo chmod -R ugo+r $target
+    mariabackup --prepare --target-dir "$target" --user root --password root --socket=$SOCKET 2>&1 | tee "/var/log/mariabackup_prepare_$name.log"
+    mariabackup --copy-back --force-non-empty-directories --target-dir "$target" --user root --password root --socket=$SOCKET 2>&1 | tee "/var/log/mariabackup_copy_back_$name.log"
+    echo 'Database initialized from $target'
+fi
+
+if [ -f /var/lib/mysql/db_mariadb_version.txt ]; then
+  db_mariadb_version=$(cat /var/lib/mysql/db_mariadb_version.txt)
+else
+  echo "No existing db_mariadb_version.txt was found, so assuming it was mariadb 10.1"
+  db_mariadb_version="10.1"  # Assume it was 10.1; we didn't maintain this before 10.2
+fi
+
+my_mariadb_version=$(mysql -V | awk '{sub( /\.[0-9]+-MariaDB,/, ""); print $5 }')
+if [ "$my_mariadb_version" != "$db_mariadb_version" ]; then
+    mysqld --skip-networking --skip-grant-tables --socket=$SOCKET >/tmp/mysqld_temp_startup.log 2>&1 &
+    pid=$!
+    if ! serverwait ; then
+        echo "Failed to get mysqld running to run mysql_upgrade"
+        exit 103
+    fi
+    echo "Running mysql_upgrade because my_mariadb_version=$my_mariadb_version is not the same as db_mariadb_version=$db_mariadb_version"
+    mysql_upgrade --socket=$SOCKET
+    kill $pid
+fi
+
+# And use the mariadb version we have here.
+echo $my_mariadb_version >/var/lib/mysql/db_mariadb_version.txt
+
+echo
+echo 'MySQL init process done. Ready for start up.'
+echo
+
+echo "Starting mysqld."
+tail -f /var/log/mysqld.log &
+exec mysqld

containers/ddev-dbserver/10.1/files/var/tmp/mysqlbase/backup-my.cnf

@@ -0,0 +1,16 @@
+# This MySQL options file was generated by innobackupex.
+
+# The MySQL server
+[mysqld]
+innodb_checksum_algorithm=INNODB
+innodb_log_checksum_algorithm=INNODB
+innodb_data_file_path=ibdata1:12M:autoextend
+innodb_log_files_in_group=2
+innodb_log_file_size=67108864
+innodb_page_size=16384
+innodb_log_block_size=512
+innodb_undo_directory=.
+innodb_undo_tablespaces=0
+
+
+

containers/ddev-dbserver/10.1/files/var/tmp/mysqlbase/db_mariadb_version.txt

@@ -0,0 +1 @@
+10.1

containers/ddev-dbserver/10.1/files/var/tmp/mysqlbase/mysql/proxies_priv.MYD

@@ -0,0 +1 @@
+�localhost                                                                                                                                                                           root                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                                                                                                                                                                                       \W��8a4e3d46e4b2                                                                                                                                                                        root                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                                                                                                                                                                                       \W�

containers/ddev-dbserver/10.1/files/var/tmp/mysqlbase/xtrabackup_binlog_info

@@ -0,0 +1 @@
+on.000004	4206934	0-1-4734

containers/ddev-dbserver/10.1/files/var/tmp/mysqlbase/xtrabackup_checkpoints

@@ -0,0 +1,5 @@
+backup_type = full-backuped
+from_lsn = 0
+to_lsn = 1616727
+last_lsn = 1616727
+recover_binlog_info = 0

containers/ddev-dbserver/10.1/files/var/tmp/mysqlbase/xtrabackup_info

@@ -0,0 +1,17 @@
+uuid = 568216a0-f717-11e8-b03c-0242ac110002
+name = 
+tool_name = mariabackup
+tool_command = --backup --target-dir=/mysqlbase --user root --password=... root --socket=/var/tmp/mysql.sock
+tool_version = 10.1.32-MariaDB
+ibbackup_version = 10.1.32-MariaDB
+server_version = 10.1.32-MariaDB
+start_time = 2018-12-03 16:20:21
+end_time = 2018-12-03 16:20:24
+lock_time = 0
+binlog_pos = filename 'on.000004', position '4206934', GTID of the last change '0-1-4734'
+innodb_from_lsn = 0
+innodb_to_lsn = 1616727
+partial = N
+incremental = N
+format = file
+compressed = N

containers/ddev-dbserver/Dockerfile → containers/ddev-dbserver/10.2/Dockerfile

@@ -4,6 +4,7 @@ ENV MYSQL_DATABASE db
 ENV MYSQL_USER db
 ENV MYSQL_PASSWORD db
 ENV MYSQL_ROOT_PASSWORD root
+ENV MARIADB_VERSION 10.2
 
 # Install mariadb and other packages
 RUN apk add --no-cache mariadb mariadb-client mariadb-backup mariadb-server-utils bash tzdata shadow sudo

containers/ddev-dbserver/10.2/Makefile

@@ -0,0 +1,53 @@
+# Makefile for a standard repo with associated container
+
+##### These variables need to be adjusted in most repositories #####
+
+# This repo's root import path (under GOPATH).
+# PKG := github.com/drud/repo_name
+
+# Docker repo for a push
+DOCKER_REPO ?= drud/ddev-dbserver
+
+MARIADB_VERSION=10.2
+
+# Upstream repo used in the Dockerfile
+# UPSTREAM_REPO ?= oraclelinux:7.3
+
+# Top-level directories to build
+#SRC_DIRS := files drudapi secrets utils
+
+# Optional to docker build
+# DOCKER_ARGS = --build-arg MYSQL_PACKAGE_VERSION=5.7.17-1
+
+
+# VERSION can be set by
+  # Default: git tag
+  # make command line: make VERSION=0.9.0
+# It can also be explicitly set in the Makefile as commented out below.
+
+# This version-strategy uses git tags to set the version string
+# VERSION can be overridden on make commandline: make VERSION=0.9.1 push
+ifndef VERSION
+	VERSION := $(shell git describe --tags --always --dirty)
+endif
+override VERSION := $(VERSION)-$(MARIADB_VERSION)
+
+#
+# This version-strategy uses a manual value to set the version string
+#VERSION := 1.2.3
+
+# Each section of the Makefile is included from standard components below.
+# If you need to override one, import its contents below and comment out the
+# include. That way the base components can easily be updated as our general needs
+# change.
+#include build-tools/makefile_components/base_build_go.mak
+include ../../../build-tools/makefile_components/base_build_python-docker.mak
+include ../../../build-tools/makefile_components/base_container.mak
+include ../../../build-tools/makefile_components/base_push.mak
+#include build-tools/makefile_components/base_test_go.mak
+#include ../../build-tools/makefile_components/base_test_python.mak
+
+build: container
+
+test: container
+	cd .. &&  ./test/testserver.sh $(DOCKER_REPO):$(VERSION) $(MARIADB_VERSION)