Remove sudo from ddev-dbserver and ddev-ssh-agent, limit use in ddev-…

…webserver, fixes #2607 (#2806)

.circleci/linux_docker_buildx_setup.sh

@@ -1,6 +1,6 @@
 sudo apt-get install docker-ce-cli binfmt-support qemu-user-static
 
-BUILDX_BINARY_URL="https://github.com/docker/buildx/releases/download/v0.4.2/buildx-v0.4.2.linux-amd64"
+BUILDX_BINARY_URL="https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64"
 
 curl --output docker-buildx \
     --silent --show-error --location --fail --retry 3 \

.travis.yml

@@ -21,19 +21,20 @@ before_install:
 # Group tests to break up travis-CI build so it doesn't hit time limit.
 jobs:
   include:
-    - name: testcmd tests
-      script: make testcmd
-    - name: testpkg tests except ddevapp
-      script: make testnotddevapp
-    - name: ddevapp tests
-      script: make testddevapp
     - name: container tests
       script: |
         set -eu -o pipefail
         for dir in containers/*/
-          do pushd $dir
-          echo "--- Test container $dir"
-          time make test
-          popd
+          do
+            pushd $dir >/dev/null
+            echo "--- Test container $dir"
+            time make test
+            popd >/dev/null
         done
         set +eu
+    - name: testcmd tests
+      script: make testcmd
+    - name: testpkg tests except ddevapp
+      script: make testnotddevapp
+    - name: ddevapp tests
+      script: make testddevapp

.travis/linux_travis_arm64_setup.sh

@@ -13,7 +13,7 @@ if [ ! -z "${DOCKERHUB_PULL_USERNAME:-}" ]; then
 fi
 
 sudo apt-get update -qq
-sudo apt-get install -qq mysql-client zip jq expect nfs-kernel-server build-essential curl git libnss3-tools libcurl4-gnutls-dev docker-compose
+sudo rm -f /usr/local/bin/jq && sudo apt-get install -y mysql-client zip jq expect nfs-kernel-server build-essential curl git libnss3-tools libcurl4-gnutls-dev docker-compose
 
 # Copy docker-compose to /usr/local/bin because Travis' pre-installed version leads to exec format error
 sudo cp /usr/bin/docker-compose /usr/local/bin/docker-compose
@@ -25,7 +25,7 @@ echo "capath=/etc/ssl/certs/" >>~/.curlrc
 
 . ~/.bashrc
 
-npm install -g bats
+git clone --branch v1.2.1 https://github.com/bats-core/bats-core.git /tmp/bats-core && pushd /tmp/bats-core >/dev/null && sudo ./install.sh /usr/local
 
 # Install mkcert
 sudo curl -sSL https://github.com/drud/mkcert/releases/download/v1.4.6/mkcert-v1.4.6-linux-arm64 -o /usr/local/bin/mkcert && sudo chmod +x /usr/local/bin/mkcert

containers/ddev-dbserver/Dockerfile

@@ -13,11 +13,11 @@ ENV MYSQL_ROOT_PASSWORD root
 
 SHELL ["/bin/bash", "-c"]
 
-RUN apt-get -qq update && apt-get -qq install -y tzdata sudo gnupg2 pv less vim wget curl lsb-release >/dev/null
+RUN apt-get -qq update && apt-get -qq install -y tzdata gnupg2 pv less vim wget curl lsb-release >/dev/null
 
 RUN set -x; if ( ! command -v xtrabackup && ! command -v mariabackup ); then \
     curl -sSL https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb -o percona-release_latest.$(lsb_release -sc)_all.deb; \
-    sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb >/dev/null; \
+    dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb >/dev/null; \
     rm percona-release*.deb ; \
     xtrabackup_version=percona-xtrabackup-24 ; \
     if [ "$DB_VERSION" = "8.0" ]; then xtrabackup_version=percona-xtrabackup-80; fi ; \
@@ -26,7 +26,7 @@ fi
 RUN apt-get -qq autoclean
 
 RUN rm -rf /var/lib/mysql/* /etc/mysql
-RUN mkdir -p /var/lib/mysql && chmod 777 /var/lib/mysql
+RUN mkdir -p /var/lib/mysql
 
 ADD files /
 
@@ -41,6 +41,9 @@ RUN chmod ugo+wx /mnt /var/tmp
 RUN mkdir -p /var/log /var/tmp/mysqlbase /etc/mysql/conf.d && chmod -R ugo+wx /var/log /var/tmp/mysqlbase /etc/mysql/conf.d
 RUN ln -s -f /dev/stderr /var/log/mysqld.err
 
+RUN rm -rf /var/lib/mysql/*
+RUN chmod -R ugo+rw /var/lib/mysql /etc/mysql/conf.d /mysqlbase && find /mysqlbase -type d | xargs chmod ugo+rwx
+
 RUN /sbin/mkhomedir_helper www-data
 
 ENTRYPOINT ["/docker-entrypoint.sh"]

containers/ddev-dbserver/Makefile

@@ -17,6 +17,7 @@ TEST_TARGETS=$(shell if [ "$(CURRENT_ARCH)" = "amd64" ] ; then \
 	  echo "mariadb_10.2_test mariadb_10.3_test mariadb_10.4_test mariadb_10.5_test"; \
   	fi )
 
+container: build
 build: $(BUILD_TARGETS)
 
 mariadb_5.5: mariadb_5.5_amd64
@@ -57,12 +58,13 @@ test: build $(TEST_TARGETS)
 push:
 	@echo "To push all images, use make PUSH=true VERSION=<tag>, to push a specific image, use something like make mariadb_10.3 PUSH=true VERSION=<tag>" && exit 1
 
-# make mariadb_10.2_test mysql_8.0_test
-# make test # for all
+# make mariadb_10.2_test mysql_8.0_test VERSION=20210213_db_image_no_sudo
+# make test VERSION=20210213_db_image_no_sudo # for all
 $(TEST_TARGETS):
 	@export DB_TYPE=$(word 1, $(subst _, ,$@)) && \
 	export DB_MAJOR_VERSION=$(word 2, $(subst _, ,$@)) && \
 	export ARCH=$(CURRENT_ARCH) && \
+	printf "\n\n\n==== Testing $${DB_TYPE} $${DB_MAJOR_VERSION} $(VERSION)" && \
 	./test/test_dbserver.sh $${DB_TYPE} $${DB_MAJOR_VERSION} $(VERSION)
 
 clean:

containers/ddev-dbserver/files/create_base_db.sh

@@ -7,10 +7,10 @@ SOCKET=/var/tmp/mysql.sock
 OUTDIR=/mysqlbase
 
 mkdir -p ${OUTDIR}
-sudo chown -R "$(id -u):$(id -g)" $OUTDIR
+chown -R "$(id -u):$(id -g)" $OUTDIR
 
-sudo chmod ugo+w /var/tmp
-sudo mkdir -p /var/lib/mysql /mnt/ddev_config/mysql && sudo rm -f /var/lib/mysql/* && sudo chmod -R ugo+w /var/lib/mysql
+chmod ugo+w /var/tmp
+mkdir -p /var/lib/mysql /mnt/ddev_config/mysql && rm -f /var/lib/mysql/* && chmod -R ugo+w /var/lib/mysql
 
 echo 'Initializing mysql'
 mysqld --version
@@ -76,7 +76,7 @@ EOF
 fi
 
 
-sudo rm -rf $OUTDIR/*
+rm -rf $OUTDIR/*
 
 backuptool=mariabackup
 if command -v xtrabackup; then backuptool="xtrabackup --datadir=/var/lib/mysql"; fi

containers/ddev-dbserver/files/docker-entrypoint.sh

@@ -31,24 +31,19 @@ if [ $# = "2" -a "${1:-}" = "restore_snapshot" ] ; then
     echo "Restoring from snapshot directory $snapshot_dir"
     # Ugly macOS .DS_Store in this directory can break the restore
     find ${snapshot_dir} -name .DS_Store -print0 | xargs rm -f
-    sudo rm -rf /var/lib/mysql/*
+    rm -rf /var/lib/mysql/*
   else
     echo "$snapshot_dir does not exist, not attempting restore of snapshot"
     unset snapshot_dir
     exit 101
   fi
 fi
 
-sudo chown -R "$(id -u):$(id -g)" /mysqlbase /var/lib/mysql
-
 server_db_version=$(PATH=$PATH:/usr/sbin:/usr/local/bin:/usr/local/mysql/bin mysqld -V 2>/dev/null | awk '{sub( /\.[0-9]+(-.*)?$/, "", $3); print $3 }')
 
-sudo chown -R "$UID:$(id -g)" /var/lib/mysql
-
 # If we have extra mariadb cnf files,, copy them to where they go.
 if [ -d /mnt/ddev_config/mysql -a "$(echo /mnt/ddev_config/mysql/*.cnf)" != "/mnt/ddev_config/mysql/*.cnf" ] ; then
-  sudo cp /mnt/ddev_config/mysql/*.cnf /etc/mysql/conf.d
-  sudo chmod -R ugo-w /etc/mysql/conf.d
+  echo "!includedir /mnt/ddev_config/mysql" >/etc/mysql/conf.d/ddev.cnf
 fi
 
 export BACKUPTOOL=mariabackup
@@ -64,8 +59,7 @@ if [ ! -f "/var/lib/mysql/db_mariadb_version.txt" ]; then
     fi
     target=${snapshot_dir:-/mysqlbase/}
     name=$(basename $target)
-    sudo rm -rf /var/lib/mysql/* /var/lib/mysql/.[a-z]* && sudo chmod -R ugo+w /var/lib/mysql
-    sudo chmod -R ugo+r $target
+    rm -rf /var/lib/mysql/* /var/lib/mysql/.[a-z]*
     ${BACKUPTOOL} --prepare --skip-innodb-use-native-aio --target-dir "$target" --user=root --password=root --socket=$SOCKET 2>&1 | tee "/var/log/mariabackup_prepare_$name.log"
     ${BACKUPTOOL} --copy-back --skip-innodb-use-native-aio --force-non-empty-directories --target-dir "$target" --user=root --password=root --socket=$SOCKET 2>&1 | tee "/var/log/mariabackup_copy_back_$name.log"
     echo "Database initialized from ${target}"
@@ -95,8 +89,7 @@ fi
 echo $server_db_version >/var/lib/mysql/db_mariadb_version.txt
 
 cp -r /home/{.my.cnf,.bashrc} ~/
-sudo mkdir -p /mnt/ddev-global-cache/bashhistory/${HOSTNAME}
-sudo chown -R "$(id -u):$(id -g)" /mnt/ddev-global-cache/ ~/.my.cnf
+mkdir -p /mnt/ddev-global-cache/bashhistory/${HOSTNAME} || true
 
 echo
 echo 'MySQL init process done. Ready for start up.'

containers/ddev-dbserver/test/basic_database.bats

@@ -1,58 +1,22 @@
 #!/usr/bin/env bats
 
 # Run these tests from the repo root directory, for example
-# bats tests
+# ./test/bats tests
 
-function setup {
-    CONTAINER_NAME="testserver"
-    HOSTPORT=33000
-    MYTMPDIR="${HOME}/tmp/testserver-sh_${RANDOM}_$$"
-    outdir="${HOME}/tmp/mariadb_testserver/output_${RANDOM}_$$"
-    VOLUME="dbserver_test-${RANDOM}_$$"
-
-    export MOUNTUID=33
-    export MOUNTGID=33
+load functions.sh
 
-    # Homebrew mysql client realy really wants /usr/local/etc/my.cnf.d
-    if [ "${OS:-$(uname)}" != "Windows_NT" ] && [ ! -d /usr/local/etc/my.cnf.d ]; then
-        mkdir -p /usr/local/etc/my.cnf.d || sudo mkdir -p /usr/local/etc/my.cnf.d
-    fi
-    docker rm -f ${CONTAINER_NAME} 2>/dev/null || true
-
-    echo "# Starting container using: docker run -u "$MOUNTUID:$MOUNTGID" -v $VOLUME:/var/lib/mysql --name=$CONTAINER_NAME -p $HOSTPORT:3306 -d $IMAGE"
-    docker run -u "$MOUNTUID:$MOUNTGID" -v $VOLUME:/var/lib/mysql --name=$CONTAINER_NAME -p $HOSTPORT:3306 -d $IMAGE
-    containercheck
-}
+function setup {
+  basic_setup
 
-function teardown {
-    docker stop $CONTAINER_NAME
-    docker rm $CONTAINER_NAME
-    docker volume rm $VOLUME || true
+  echo "# Starting container using: docker run --rm -u "$MOUNTUID:$MOUNTGID" -v $VOLUME:/var/lib/mysql --name=$CONTAINER_NAME -p $HOSTPORT:3306 -d $IMAGE"
+  docker run -u "$MOUNTUID:$MOUNTGID" -v $VOLUME:/var/lib/mysql:nocopy --name=$CONTAINER_NAME -p $HOSTPORT:3306 -d $IMAGE
+  containercheck
 }
 
 
-# Wait for container to be ready.
-function containercheck {
-	for i in {60..0};
-	do
-		# status contains uptime and health in parenthesis, sed to return health
-		status="$(docker ps --format "{{.Status}}" --filter "name=$CONTAINER_NAME" | sed  's/.*(\(.*\)).*/\1/')"
-		if [[ "$status" == "healthy" ]]
-		then
-			return 0
-		fi
-		sleep 1
-	done
-	echo "# --- ddev-dbserver FAIL: information"
-	docker ps -a
-	docker logs $CONTAINER_NAME
-	docker inspect $CONTAINER_NAME
-	return 1
-}
-
 @test "test user root and db access for ${DB_TYPE} ${DB_VERSION}" {
-    mysql --user=root --password=root --database=mysql --host=127.0.0.1 --port=$HOSTPORT -e "SELECT 1;"
-    mysql -udb -pdb --database=db --host=127.0.0.1 --port=$HOSTPORT -e "SHOW TABLES;"
+  mysql --user=root --password=root --database=mysql --host=127.0.0.1 --port=$HOSTPORT -e "SELECT 1;"
+  mysql -udb -pdb --database=db --host=127.0.0.1 --port=$HOSTPORT -e "SHOW TABLES;"
 }
 
 @test "make sure trigger capability works correctly on ${DB_TYPE} ${DB_VERSION}" {

containers/ddev-dbserver/test/custom_config.bats

@@ -3,55 +3,19 @@
 # Run these tests from the repo root directory, for example
 # bats tests
 
-function setup {
-    CONTAINER_NAME="testserver"
-    HOSTPORT=33000
-    MYTMPDIR="${HOME}/tmp/testserver-sh_${RANDOM}_$$"
-    outdir="${HOME}/tmp/mariadb_testserver/output_${RANDOM}_$$"
-    VOLUME="dbserver_test-${RANDOM}_$$"
-
-    export MOUNTUID=33
-    export MOUNTGID=33
-
-    # Homebrew mysql client realy really wants /usr/local/etc/my.cnf.d
-    if [ "${OS:-$(uname)}" != "Windows_NT" ] && [ ! -d /usr/local/etc/my.cnf.d ]; then
-        mkdir -p /usr/local/etc/my.cnf.d || sudo mkdir -p /usr/local/etc/my.cnf.d
-    fi
-    docker rm -f ${CONTAINER_NAME} 2>/dev/null || true
-
-    echo "# Starting image with database image $IMAGE"
-    docker run -u "$MOUNTUID:$MOUNTGID" -v $VOLUME:/var/lib/mysql --mount "type=bind,src=$PWD/test/testdata,target=/mnt/ddev_config" --name=$CONTAINER_NAME -p $HOSTPORT:3306 -d $IMAGE
-    containercheck
-}
-
-function teardown {
-    docker rm -f $CONTAINER_NAME
-    docker volume rm $VOLUME || true
-}
+load functions.sh
 
+function setup {
+  basic_setup
 
-# Wait for container to be ready.
-function containercheck {
-	for i in {60..0};
-	do
-		# status contains uptime and health in parenthesis, sed to return health
-		status="$(docker ps --format "{{.Status}}" --filter "name=$CONTAINER_NAME" | sed  's/.*(\(.*\)).*/\1/')"
-		if [[ "$status" == "healthy" ]]
-		then
-			return 0
-		fi
-		sleep 1
-	done
-	echo "# --- ddev-dbserver FAIL: information"
-	docker ps -a
-	docker logs $CONTAINER_NAME
-	docker inspect $CONTAINER_NAME
-	return 1
+  echo "# Starting container using: docker run --rm -u "$MOUNTUID:$MOUNTGID" --rm -v $VOLUME:/var/lib/mysql --mount "type=bind,src=$PWD/test/testdata,target=/mnt/ddev_config" --name=$CONTAINER_NAME -p $HOSTPORT:3306 -d $IMAGE"
+  docker run --rm -u "$MOUNTUID:$MOUNTGID" --rm -v $VOLUME:/var/lib/mysql --mount "type=bind,src=$PWD/test/testdata,target=/mnt/ddev_config" --name=$CONTAINER_NAME -p $HOSTPORT:3306 -d $IMAGE
+  containercheck
 }
 
 @test "test with mysql/utf.cnf override ${DB_TYPE} ${DB_VERSION}" {
-    docker exec $CONTAINER_NAME sh -c 'grep collation-server /mnt/ddev_config/mysql/utf.cnf'
-    mysql --user=root --password=root --skip-column-names --host=127.0.0.1 --port=$HOSTPORT -e "SHOW GLOBAL VARIABLES like \"collation_server\";" | grep "utf8_general_ci"
+  docker exec $CONTAINER_NAME sh -c 'grep collation-server /mnt/ddev_config/mysql/utf.cnf'
+  mysql --user=root --password=root --skip-column-names --host=127.0.0.1 --port=$HOSTPORT -e "SHOW GLOBAL VARIABLES like \"collation_server\";" | grep "utf8_general_ci"
 }
 
 

containers/ddev-dbserver/test/functions.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+function basic_setup {
+    export CONTAINER_NAME="testserver"
+    export HOSTPORT=33000
+    export MYTMPDIR="${HOME}/tmp/testserver-sh_${RANDOM}_$$"
+    export outdir="${HOME}/tmp/mariadb_testserver/output_${RANDOM}_$$"
+    export VOLUME="dbserver_test-${RANDOM}_$$"
+
+    export MOUNTUID=33
+    export MOUNTGID=33
+
+    # Homebrew mysql client realy really wants /usr/local/etc/my.cnf.d
+    if [ "${OS:-$(uname)}" != "Windows_NT" ] && [ ! -d /usr/local/etc/my.cnf.d ]; then
+        mkdir -p /usr/local/etc/my.cnf.d || sudo mkdir -p /usr/local/etc/my.cnf.d
+    fi
+    docker rm -f ${CONTAINER_NAME} 2>/dev/null || true
+
+    # Initialize the volume with the correct ownership
+    docker run --rm -v "${VOLUME}:/var/lib/mysql:nocopy" busybox chown -R ${MOUNTUID}:${MOUNTGID} /var/lib/mysql
+}
+
+function teardown {
+  docker rm -f ${CONTAINER_NAME}
+  docker volume rm $VOLUME || true
+}
+
+# Wait for container to be ready.
+function containercheck {
+  for i in {15..0}; do
+    # fail if we can't find the container
+    if ! docker inspect ${CONTAINER_NAME} >/dev/null; then
+      break
+    fi
+
+    status="$(docker inspect ${CONTAINER_NAME} | jq -r '.[0].State.Status')"
+    if [ "${status}" != "running" ]; then
+      break
+    fi
+    health="$(docker inspect --format '{{json .State.Health }}' ${CONTAINER_NAME} | jq -r .Status)"
+    case ${health} in
+    healthy)
+      return 0
+      ;;
+    *)
+      sleep 1
+      ;;
+    esac
+  done
+  echo "# --- ddev-dbserver FAIL -----"
+  return 1
+}

containers/ddev-ssh-agent/Dockerfile

@@ -1,6 +1,6 @@
 FROM debian:buster-slim
 
-RUN apt-get update && apt-get install -y bash expect file openssh-client socat sudo psmisc && sudo apt autoclean
+RUN apt-get update && apt-get install -y bash expect file openssh-client socat psmisc && apt autoclean
 
 # Copy container files
 COPY files /

containers/ddev-ssh-agent/files/entry.sh

@@ -32,7 +32,7 @@ debug_msg ()
   fi
 }
 
-sudo mkdir -p $SSH_KEY_DIR && sudo chown $UID $SSH_KEY_DIR && chmod 700 $SSH_KEY_DIR
+mkdir -p $SSH_KEY_DIR && chown $UID $SSH_KEY_DIR && chmod 700 $SSH_KEY_DIR
 mkdir -p $SOCKET_DIR
 
 case "$1" in