Upgrade containers (and mariadb to 10.2), fixes #947, fixes #1068 (#1123

)

* Upgrade dbserver to alpine 3.8, upgrades to mariadb 10.2, fixes #947

* Upgrade container source for ddev-router

* Upgrade ddev-webserver to debian stretch, fixes #1068

* Add mariabackup explicitly in db container

* Update nc command for netcat on debian stretch

* Update base_db for mariadb 10.2

* Simplification of create_base_db.sh, no more need for root

* Remove user=root from my.cnf as not really allowed in mariadb 10.2

* Update testserver.sh for mariadb 10.2, use volume not bind mount

* Update bind mount to not use it directly (mariadb 10.2)

* Report the docker logs if WaitContainer() fails

containers/ddev-dbserver/Dockerfile

@@ -1,12 +1,12 @@
-FROM alpine:3.7
+FROM alpine:3.8
 
 ENV MYSQL_DATABASE db
 ENV MYSQL_USER db
 ENV MYSQL_PASSWORD db
 ENV MYSQL_ROOT_PASSWORD root
 
 # Install mariadb and other packages
-RUN apk add --no-cache mariadb mariadb-client bash tzdata shadow sudo
+RUN apk add --no-cache mariadb mariadb-client mariadb-backup bash tzdata shadow sudo
 # Remove the installed version as we need to set up our own from scratch
 
 RUN rm -rf /var/lib/mysql/* /etc/mysql

containers/ddev-dbserver/Makefile

@@ -41,7 +41,7 @@ include ../../build-tools/makefile_components/base_push.mak
 #include build-tools/makefile_components/base_test_go.mak
 include ../../build-tools/makefile_components/base_test_python.mak
 
-MYSQL_VERSION=10.1
+MYSQL_VERSION=10.2
 
 test: container
 	./test/testserver.sh $(DOCKER_REPO):$(VERSION) $(MYSQL_VERSION) 

containers/ddev-dbserver/files/create_base_db.sh

@@ -6,7 +6,7 @@ set -o pipefail
 
 # This script can be used to create a bare database directory for use by
 # ddev startup. It can be run from the host with:
-# docker run -it -v "$PWD/files/var/tmp/mysqlbase:/mysqlbase" --rm --entrypoint=/create_base_db.sh drud/ddev-dbserver:<your_version>
+# docker run -t -u $(id -u) -v "$PWD/files/var/tmp/mysqlbase:/mysqlbase" --rm --entrypoint=/create_base_db.sh drud/ddev-dbserver:<your_version>
 
 SOCKET=/var/tmp/mysql.sock
 OUTDIR=/mysqlbase
@@ -18,17 +18,10 @@ fi
 
 # For this script we don't want the defaults in .my.cnf
 # However, this script is never run on a normal startup, so we can just throw it away.
-rm -f /home/.my.cnf
+sudo rm -f /home/.my.cnf
 
-chgrp mysql /var/tmp
-chmod ug+rw /var/tmp
-
-if [ -d "/var/lib/mysql/mysql" ]; then
-	echo "A mysql installation already exists, aborting"
-	exit 2
-fi
-mkdir -p /var/lib/mysql /mnt/ddev_config/mysql
-chown -R mysql:mysql /var/lib/mysql /mnt/ddev_config/mysql /var/log/mysql*
+sudo chmod ugo+w /var/tmp
+sudo mkdir -p /var/lib/mysql /mnt/ddev_config/mysql && sudo rm -f /var/lib/mysql/* && sudo chmod -R ugo+w /var/lib/mysql
 
 echo 'Initializing mysql'
 mysql_install_db
@@ -73,6 +66,7 @@ mysql -uroot <<EOF
 	FLUSH TABLES;
 EOF
 
+sudo rm -rf $OUTDIR/*
 mariabackup --backup --target-dir=$OUTDIR --user root --password root --socket=$SOCKET
 
 if ! kill -s TERM "$pid" || ! wait "$pid"; then

containers/ddev-dbserver/files/etc/my.cnf

@@ -40,7 +40,6 @@ symbolic-links=0
 
 
 # GENERAL #
-user                           = mysql
 default-storage-engine         = InnoDB
 socket                         = /var/tmp/mysql.sock
 pid-file                       = /var/tmp/mysql.pid

containers/ddev-dbserver/files/migrate_file_to_volume.sh

@@ -10,23 +10,26 @@ set -o pipefail
 # months.
 #
 # Run this command in the project directory:
-# docker run -t -u "$(id -u):$(id -g)" -e SNAPSHOT_NAME=<migration_snapshot_name -v "$PWD/.ddev:/mnt/ddev_config" -v "$HOME/.ddev/<projectname>/mysql:/var/lib/mysql" --rm --entrypoint=/migrate_file_to_volume.sh drud/ddev-dbserver:<your_version>
+# docker run -t -u "$(id -u):$(id -g)" -e SNAPSHOT_NAME=<migration_snapshot_name -v "$PWD/.ddev:/mnt/ddev_config" -v "$HOME/.ddev/<projectname>/mysql:/mysqlmount" --rm --entrypoint=/migrate_file_to_volume.sh drud/ddev-dbserver:<your_version>
 
 if [ -z "${SNAPSHOT_NAME:-}" ] ; then
     echo "SNAPSHOT_NAME environment variable must be set"
-    exit 1
+    exit 101
 fi
 
 OUTDIR="/mnt/ddev_config/db_snapshots/${SNAPSHOT_NAME}"
 SOCKET=/var/tmp/mysql.sock
 
 mkdir -p $OUTDIR
 
-if [ ! -d "/var/lib/mysql/mysql" ]; then
+if [ ! -d "/mysqlmount/mysql" ]; then
 	echo "No mysql bind-mount directory was found, aborting"
-	exit 2
+	exit 102
 fi
 
+sudo mkdir -p /var/lib/mysql && sudo chmod -R ugo+w /var/lib/mysql
+cp -r /mysqlmount/* /var/lib/mysql
+
 
 # Wait for mysql server to be ready.
 function serverwait {
@@ -46,19 +49,20 @@ function serverwait {
 	return 1
 }
 
-sudo chmod -R ugo+rw /var/lib/mysql /var/log/mysql*
-
 # Using --skip-grant-tables here becasue some old projects may not have working
 # --user root --password root
 mysqld --skip-networking --skip-grant-tables --socket=$SOCKET 2>&1 &
 pid=$!
 if ! serverwait ; then
     echo "Failed to get mysqld running"
-    exit 2
+    exit 103
 fi
 
 mariabackup --backup --target-dir=$OUTDIR --user root --socket=$SOCKET 2>&1
-if [ "$?" != 0 ] ; then echo "Failed mariabackup command."; exit $?; fi
+if [ "$?" != 0 ] ; then
+    echo "Failed mariabackup command.";
+    exit $((200+$?));
+fi
 
 # Wait for mysqld to exit
 kill -s TERM "$pid" && wait "$pid"

containers/ddev-dbserver/files/var/tmp/mysqlbase/backup-my.cnf

@@ -2,15 +2,12 @@
 
 # The MySQL server
 [mysqld]
-innodb_checksum_algorithm=INNODB
-innodb_log_checksum_algorithm=INNODB
+innodb_checksum_algorithm=innodb
 innodb_data_file_path=ibdata1:12M:autoextend
 innodb_log_files_in_group=2
 innodb_log_file_size=67108864
 innodb_page_size=16384
-innodb_log_block_size=512
-innodb_undo_directory=.
+innodb_undo_directory=./
 innodb_undo_tablespaces=0
 
 
-

containers/ddev-dbserver/files/var/tmp/mysqlbase/ib_buffer_pool

@@ -0,0 +1,167 @@
+0,4
+0,12
+0,11
+0,10
+0,9
+0,6
+0,7
+0,8
+0,63
+0,62
+0,61
+0,60
+0,59
+0,58
+0,57
+0,56
+0,55
+0,54
+0,53
+0,52
+0,51
+0,50
+0,49
+0,48
+0,47
+0,46
+0,1
+0,45
+0,2
+0,0
+0,5
+1,3
+1,2
+1,1
+1,0
+2,3
+2,2
+2,1
+2,0
+3,3
+3,2
+3,1
+3,0
+0,255
+0,254
+0,253
+0,252
+0,251
+0,250
+0,249
+0,248
+0,247
+0,246
+0,245
+0,244
+0,243
+0,242
+0,241
+0,240
+0,239
+0,238
+0,237
+0,236
+0,235
+0,234
+0,233
+0,232
+0,231
+0,230
+0,229
+0,228
+0,227
+0,226
+0,225
+0,224
+0,223
+0,222
+0,221
+0,220
+0,219
+0,218
+0,217
+0,216
+0,215
+0,214
+0,213
+0,212
+0,211
+0,210
+0,209
+0,208
+0,207
+0,206
+0,205
+0,204
+0,203
+0,202
+0,201
+0,200
+0,199
+0,198
+0,197
+0,196
+0,195
+0,194
+0,193
+0,192
+0,315
+0,314
+0,313
+0,312
+0,311
+0,310
+0,309
+0,308
+0,307
+0,306
+0,305
+0,304
+0,303
+0,302
+0,301
+0,300
+0,299
+0,298
+0,297
+0,296
+0,295
+0,294
+0,293
+0,292
+0,291
+0,290
+0,289
+0,288
+0,287
+0,286
+0,285
+0,284
+0,283
+0,282
+0,281
+0,280
+0,279
+0,278
+0,277
+0,276
+0,275
+0,274
+0,273
+0,272
+0,271
+0,270
+0,269
+0,268
+0,267
+0,266
+0,265
+0,264
+0,263
+0,262
+0,261
+0,260
+0,259
+0,258
+0,257
+0,256

containers/ddev-dbserver/files/var/tmp/mysqlbase/mysql/proxies_priv.MYD

@@ -1 +1 @@
-�localhost                                                                                                                                                                           root                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                                                                                                                                                                                       [j"t�8f2e6f66f85d                                                                                                                                                                        root                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                                                                                                                                                                                       [j"t
+�localhost                                                                                                                                                                           root                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                                                                                                                                                                                       [��\�76fbb4f64e67                                                                                                                                                                        root                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                                                                                                                                                                                       [��\

containers/ddev-dbserver/files/var/tmp/mysqlbase/xtrabackup_binlog_info

@@ -1 +1 @@
-on.000004	4206934	0-1-4734
+on.000002	4282389	0-1-4723

containers/ddev-dbserver/files/var/tmp/mysqlbase/xtrabackup_checkpoints

@@ -1,5 +1,5 @@
 backup_type = full-backuped
 from_lsn = 0
-to_lsn = 1616727
-last_lsn = 1616727
+to_lsn = 1619987
+last_lsn = 1620015
 recover_binlog_info = 0

containers/ddev-dbserver/files/var/tmp/mysqlbase/xtrabackup_info

@@ -1,16 +1,16 @@
-uuid = 74fd824c-9a94-11e8-8949-0242ac110002
+uuid = 8d9c7988-bf58-11e8-aac8-0242ac110004
 name = 
 tool_name = mariabackup
 tool_command = --backup --target-dir=/mysqlbase --user root --password=... root --socket=/var/tmp/mysql.sock
-tool_version = 10.1.32-MariaDB
-ibbackup_version = 10.1.32-MariaDB
-server_version = 10.1.32-MariaDB
-start_time = 2018-08-07 22:51:42
-end_time = 2018-08-07 22:51:44
+tool_version = 10.2.15-MariaDB
+ibbackup_version = 10.2.15-MariaDB
+server_version = 10.2.15-MariaDB-log
+start_time = 2018-09-23 17:46:07
+end_time = 2018-09-23 17:46:08
 lock_time = 0
-binlog_pos = filename 'on.000004', position '4206934', GTID of the last change '0-1-4734'
+binlog_pos = filename 'on.000002', position '4282389', GTID of the last change '0-1-4723'
 innodb_from_lsn = 0
-innodb_to_lsn = 1616727
+innodb_to_lsn = 1619987
 partial = N
 incremental = N
 format = file