Use docker for CI replica tests, deprecate dbdeployer (#1529)

* use docker for ci tests

* set charset server

* remove unusued cibuild-gh-ost-replica-tests

* add doc for how to run localtests

Author: meiji163

Diff for: .github/workflows/replica-tests.yml

@@ -3,33 +3,12 @@ name: migration tests
 on: [pull_request]
 
 jobs:
-  build:
-
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        version: [mysql-5.7.25,mysql-8.0.16,PerconaServer-8.0.21]
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Go
-      uses: actions/setup-go@v5
-      with:
-        go-version-file: go.mod
-
-    - name: migration tests
-      env:
-        TEST_MYSQL_VERSION: ${{ matrix.version }}
-      run:  script/cibuild-gh-ost-replica-tests
-
   docker-tests:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
-        image: ['mysql:8.4.3']
+        image: ['mysql/mysql-server:5.7.41','mysql:8.0.41','mysql:8.4.3','percona/percona-server:8.0.41-32']
     env:
       TEST_MYSQL_IMAGE: ${{ matrix.image }}
 
@@ -45,5 +24,3 @@ jobs:
       - name: Teardown environment
         if: always()
         run: script/docker-gh-ost-replica-tests down
-
-

Diff for: doc/local-tests.md

@@ -2,7 +2,7 @@
 
 `gh-ost` is continuously tested in production via `--test-on-replica alter='engine=innodb'`. These tests check the GitHub workload and usage, but not necessarily the general case.
 
-Local tests are an additional layer of tests. They will eventually be part of continuous integration tests.
+Local tests are an additional layer of tests used for continuous integration tests and local development.
 
 Local tests test explicit use cases, such as column renames, mix of time zones, special types and alters. Traits of a single test:
 
@@ -19,4 +19,18 @@ Tests are found under [localtests](https://github.com/github/gh-ost/tree/master/
 
 New data-integrity, synchronization issues or otherwise concerns are expected to be tested by new test cases.
 
-While this is merged work is still ongoing.
+## Run with docker compose
+
+Local tests can be run locally with docker compose using the helper script [script/docker-gh-ost-replica-tests](https://github.com/github/gh-ost/tree/master/script/docker-gh-ost-replica-tests). 
+
+Example usage:
+```shell
+# create primary-replica containers with specified mysql image
+TEST_MYSQL_IMAGE="mysql-server:8.0.16" ./script/docker-gh-ost-replica-tests up
+
+# run all tests
+./script/docker-gh-ost-replica-tests run
+
+# cleanup containers
+./script/docker-gh-ost-replica-tests down
+```

Diff for: localtests/docker-compose.yml

@@ -2,25 +2,27 @@ services:
   mysql-primary:
     image: $TEST_MYSQL_IMAGE
     container_name: mysql-primary
-    command: --server-id=1 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON $MYSQL_NATIVE_PASSWORD_FLAG
+    command: --server-id=1 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG
     environment:
       MYSQL_ROOT_PASSWORD: opensesame
       MYSQL_ROOT_HOST: '%'
       MYSQL_DATABASE: test
       MYSQL_TCP_PORT: 3307
+      INIT_ROCKSDB: 1 # for percona-server
     ports:
       - '3307:3307'
     expose:
       - '3307'
   mysql-replica:
     image: $TEST_MYSQL_IMAGE
     container_name: mysql-replica
-    command: --server-id=2 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON $MYSQL_NATIVE_PASSWORD_FLAG
+    command: --server-id=2 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG
     environment:
       MYSQL_ROOT_PASSWORD: opensesame
       MYSQL_ROOT_HOST: '%'
       MYSQL_DATABASE: test
       MYSQL_TCP_PORT: 3308
+      INIT_ROCKSDB: 1 # for percona-server
     ports:
       - '3308:3308'
     expose:

Diff for: script/cibuild-gh-ost-replica-tests

@@ -19,8 +19,7 @@ fi
 poll_mysql() {
     CTR=0
     cmd="gh-ost-test-mysql-$1"
-    while ! $cmd -e "select 1;" > /dev/null 2>&1
-    do
+    while ! $cmd -e "select 1;" >/dev/null 2>&1; do
         sleep 1
         CTR=$((CTR + 1))
         if [ $CTR -gt 30 ]; then
@@ -34,23 +33,22 @@ poll_mysql() {
 
 mysql-source() {
     if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then
-        gh-ost-test-mysql-master --ssl-mode=required "$@";
+        gh-ost-test-mysql-master --ssl-mode=required "$@"
     else
-        gh-ost-test-mysql-master $@;
+        gh-ost-test-mysql-master "$@"
     fi
 }
 
 mysql-replica() {
     if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then
-        gh-ost-test-mysql-replica --ssl-mode=required "$@";
+        gh-ost-test-mysql-replica --ssl-mode=required "$@"
     else
-        gh-ost-test-mysql-replica $@;
+        gh-ost-test-mysql-replica "$@"
     fi
 }
 
-
 setup() {
-    [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.39"
+    [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.41"
 
     echo "Starting MySQL $TEST_MYSQL_IMAGE containers..."
     compose_file="$GH_OST_ROOT/localtests/docker-compose.yml"
@@ -60,7 +58,7 @@ setup() {
         MYSQL_PASSWORD_HASHING_ALGORITHM="caching_sha2_password"
         MYSQL_SHA2_RSA_KEYS_FLAG="--caching-sha2-password-auto-generate-rsa-keys=ON"
     fi
-    (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_SHA2_RSA_KEYS_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" envsubst < "$compose_file") > "$compose_file.tmp"
+    (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_SHA2_RSA_KEYS_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" envsubst <"$compose_file") >"$compose_file.tmp"
     docker compose -f "$compose_file.tmp" up -d --wait
 
     echo "Waiting for MySQL..."