Switch to use mariabackup instead of just copying files for starter db,

fixes #748 (#959)

containers/mariadb-local/Dockerfile.in

@@ -5,8 +5,6 @@ ENV MYSQL_USER db
 ENV MYSQL_PASSWORD db
 ENV MYSQL_ROOT_PASSWORD root
 
-RUN mkdir /docker-entrypoint-initdb.data
-
 # Install mariadb and other packages
 RUN apk add --no-cache mariadb mariadb-client bash tzdata shadow
 # Remove the installed version as we need to set up our own from scratch
@@ -21,7 +19,7 @@ RUN chmod ugo+x /healthcheck.sh
 # Security-sensitive changes: Make sure our start script can do what is needed
 # But make sure these are right
 RUN chmod ugo+wx /mnt /var/tmp
-RUN chmod -R ugo+wx /var/log
+RUN chmod -R ugo+wx /var/log /var/tmp/mysqlbase
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 

containers/mariadb-local/README.md

@@ -12,9 +12,7 @@ The create_base_db.sh script is there for that. You can run it from the
 root of this repository like this and it will update the db starter file:
 
 
-```
-docker run -it -v "$PWD/files/var/tmp:/mysqlbase" --rm --entrypoint=/create_base_db.sh drud/mariadb-local:<your_version>
-```
+`docker run -it -v "$PWD/files/var/tmp/mysqlbase:/mysqlbase" --rm --entrypoint=/create_base_db.sh drud/mariadb-local:<your_version>`
 
 Of course the assumption is that you might have to change the name of the output
 file or make other changes in the process.

containers/mariadb-local/files/create_base_db.sh

@@ -4,13 +4,12 @@ set -x
 set -eu
 set -o pipefail
 
-# This script can be used to create a bare database tarball for use by
+# This script can be used to create a bare database directory for use by
 # ddev startup. It can be run from the host with:
-# docker run -it -v "$PWD/files/var/tmp:/mysqlbase" --rm --entrypoint=/create_base_db.sh drud/mariadb-local:<your_version>
+# docker run -it -v "$PWD/files/var/tmp/mysqlbase:/mysqlbase" --rm --entrypoint=/create_base_db.sh drud/mariadb-local:<your_version>
 
 SOCKET=/var/tmp/mysql.sock
 OUTDIR=/mysqlbase
-OUTFILE=$OUTDIR/mariadb_10.1_base_db.tgz
 
 if [ ! -d $OUTDIR ] ; then
   echo "The required output directory $OUTDIR does not seem to exist."
@@ -71,13 +70,14 @@ mysql -uroot <<EOF
 	GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION;
 	GRANT ALL ON *.* to 'root'@'localhost' IDENTIFIED BY '$MYSQL_ROOT_PASSWORD';
 	FLUSH PRIVILEGES;
+	FLUSH TABLES;
 EOF
 
+mariabackup --backup --target-dir=$OUTDIR --user root --password root --socket=$SOCKET
+
 if ! kill -s TERM "$pid" || ! wait "$pid"; then
 	echo >&2 'Mariadb initialization process failed.'
 	exit 5
 fi
 
-cd /var/lib/mysql
-tar -czf $OUTFILE db mysql
-echo "The required tarball is now in $OUTFILE"
+echo "The startup database files (in mariabackup format) are now in $OUTDIR"

containers/mariadb-local/files/docker-entrypoint.sh

@@ -13,8 +13,8 @@ fi
 # If mariadb has not been initialized, copy in the base image.
 if [ ! -d "/var/lib/mysql/mysql" ]; then
 	mkdir -p /var/lib/mysql
-	# The tarball should include only the contents of the db and mysql directories.
-	tar --no-same-owner -C /var/lib/mysql -zxf /var/tmp/mariadb_10.1_base_db.tgz
+	mariabackup --prepare --target-dir /var/tmp/mysqlbase/ --user root --password root --socket=/var/tmp/mysql.sock
+	mariabackup --copy-back --target-dir /var/tmp/mysqlbase/ --user root --password root --socket=/var/tmp/mysql.sock
 	echo 'Database initialized'
 fi
 

containers/mariadb-local/files/var/tmp/mysqlbase/backup-my.cnf

@@ -0,0 +1,16 @@
+# This MySQL options file was generated by innobackupex.
+
+# The MySQL server
+[mysqld]
+innodb_checksum_algorithm=INNODB
+innodb_log_checksum_algorithm=INNODB
+innodb_data_file_path=ibdata1:12M:autoextend
+innodb_log_files_in_group=2
+innodb_log_file_size=67108864
+innodb_page_size=16384
+innodb_log_block_size=512
+innodb_undo_directory=.
+innodb_undo_tablespaces=0
+
+
+

containers/mariadb-local/files/var/tmp/mysqlbase/db/db.opt

@@ -0,0 +1,2 @@
+default-character-set=utf8mb4
+default-collation=utf8mb4_bin

containers/mariadb-local/files/var/tmp/mysqlbase/mysql/db.MYD

@@ -0,0 +1 @@
+�%                                                                                                                                                                                   test                                                                                                                                                                                                                                                                                                                                                                                                                                            


�%                                                                                                                                                                                   test\_%                                                                                                                                                                                                                                                                                                                                                                                                                                         


�%                                                                                                                                                                                   db                                                                                                                                                                                              db                                                                                                                                                                                                                                              
�localhost                                                                                                                                                                           db                                                                                                                                                                                              db                                                                                                                                                                                                                                              


containers/mariadb-local/files/var/tmp/mysqlbase/mysql/proxies_priv.MYD

@@ -0,0 +1 @@
+�localhost                                                                                                                                                                           root                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                                                                                                                                                                                       [8��4c129a25133f                                                                                                                                                                        root                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                                                                                                                                                                                       [8�

containers/mariadb-local/files/var/tmp/mysqlbase/performance_schema/db.opt

@@ -0,0 +1,2 @@
+default-character-set=utf8
+default-collation=utf8_general_ci

containers/mariadb-local/files/var/tmp/mysqlbase/xtrabackup_binlog_info

@@ -0,0 +1 @@
+on.000004	4206934	0-1-4734

containers/mariadb-local/files/var/tmp/mysqlbase/xtrabackup_checkpoints

@@ -0,0 +1,5 @@
+backup_type = full-backuped
+from_lsn = 0
+to_lsn = 1616727
+last_lsn = 1616727
+recover_binlog_info = 0

containers/mariadb-local/files/var/tmp/mysqlbase/xtrabackup_info

@@ -0,0 +1,17 @@
+uuid = 9a5f8a4a-7cc2-11e8-bec4-0242ac110002
+name = 
+tool_name = mariabackup
+tool_command = --backup --target-dir=/mysqlbase --user root --password=... root --socket=/var/tmp/mysql.sock
+tool_version = 10.1.28-MariaDB
+ibbackup_version = 10.1.28-MariaDB
+server_version = 10.1.28-MariaDB
+start_time = 2018-07-01 00:06:27
+end_time = 2018-07-01 00:06:29
+lock_time = 0
+binlog_pos = filename 'on.000004', position '4206934', GTID of the last change '0-1-4734'
+innodb_from_lsn = 0
+innodb_to_lsn = 1616727
+partial = N
+incremental = N
+format = file
+compressed = N

containers/mariadb-local/test/testserver.sh

@@ -109,8 +109,8 @@ cleanup
 outdir="${HOME}/tmp/mariadb_testserver/output_${RANDOM}_$$"
 mkdir -p $outdir
 docker run -t -v "/$outdir://mysqlbase" --rm --entrypoint=//create_base_db.sh $IMAGE
-if [ ! -f "$outdir/mariadb_10.1_base_db.tgz" ] ; then
-  echo "Failed to build test starter tarball for mariadb."
+if [ ! -f "$outdir/ibdata1" ] ; then
+  echo "Failed to build test starter database for mariadb."
   exit 4
 fi
 rm -rf $outdir $MYTMPDIR

pkg/ddevapp/ddevapp_test.go

@@ -648,7 +648,7 @@ func TestDdevLogs(t *testing.T) {
 		assert.Contains(out, "Server started")
 
 		stdout = testcommon.CaptureUserOut()
-		err = app.Logs("db", false, false, "")
+		err = app.Logs("db", false, false, "1000")
 		assert.NoError(err)
 		out = stdout()
 		assert.Contains(out, "Database initialized")

pkg/version/version.go

@@ -26,7 +26,7 @@ var WebTag = "v0.20.0" // Note that this can be overridden by make
 var DBImg = "drud/mariadb-local"
 
 // DBTag defines the default db image tag for drud dev
-var DBTag = "20180627_mariadb_no_apk_cache" // Note that this may be overridden by make
+var DBTag = "20180630_mariabackup" // Note that this may be overridden by make
 
 // DBAImg defines the default phpmyadmin image tag used for applications.
 var DBAImg = "drud/phpmyadmin"