ci(br): support run group br&lightning tests

br/tests/README.md

@@ -87,8 +87,10 @@ After executing the tests, run `make br_coverage` to get a coverage report at
 
 ## Writing new tests
 
-New integration tests can be written as shell scripts in `tests/TEST_NAME/run.sh`.
+1. New integration tests can be written as shell scripts in `tests/TEST_NAME/run.sh`.
 The script should exit with a nonzero error code on failure.
+2. Add TEST_NAME to existing group in [run_group.sh](./run_group.sh)(Recommended), or add a new group for it.
+3. If you add a new group, the name of the new group must be added to CI [br-integration-test](https://github.com/PingCAP-QE/ci/blob/main/pipelines/pingcap/tidb/latest/pull_br_integration_test.groovy).
 
 Several convenient commands are provided:
 

br/tests/_utils/br_tikv_outage_util

@@ -1,3 +1,6 @@
+
+cur_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
 wait_file_exist() {
     timeout=0
     until [ -e "$1" ]; do
@@ -35,7 +38,7 @@ single_point_fault() {
 
 load() {
     run_sql "create database if not exists $TEST_NAME"
-    go-ycsb load mysql -P tests/"$TEST_NAME"/workload -p mysql.host="$TIDB_IP" -p mysql.port="$TIDB_PORT" -p mysql.user=root -p mysql.db="$TEST_NAME"
+    go-ycsb load mysql -P $cur_dir/../"$TEST_NAME"/workload -p mysql.host="$TIDB_IP" -p mysql.port="$TIDB_PORT" -p mysql.user=root -p mysql.db="$TEST_NAME"
     run_sql 'use '$TEST_NAME'; show tables'
 }
 

br/tests/_utils/generate_certs

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 set -eu
+cur_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 mkdir -p $TEST_DIR/certs
 openssl ecparam -out "$TEST_DIR/certs/ca.key" -name prime256v1 -genkey
@@ -24,7 +25,7 @@ openssl x509 -req -sha256 -days 2 -in "$TEST_DIR/certs/ca.csr" -signkey "$TEST_D
 for cluster in tidb pd tikv lightning tiflash curl ticdc br; do
     openssl ecparam -out "$TEST_DIR/certs/$cluster.key" -name prime256v1 -genkey
     openssl req -new -batch -sha256 -subj '/CN=localhost' -key "$TEST_DIR/certs/$cluster.key" -out "$TEST_DIR/certs/$cluster.csr"
-    openssl x509 -req -sha256 -days 1 -extensions EXT -extfile "tests/config/ipsan.cnf" \
+    openssl x509 -req -sha256 -days 1 -extensions EXT -extfile "${cur_dir}/../config/ipsan.cnf" \
         -in "$TEST_DIR/certs/$cluster.csr" \
         -CA "$TEST_DIR/certs/ca.pem" \
         -CAkey "$TEST_DIR/certs/ca.key" \

br/tests/_utils/run_br

@@ -16,7 +16,7 @@
 
 set -eux
 
-bin/br.test -test.coverprofile="$TEST_DIR/cov.$TEST_NAME.$$.out.log" DEVEL "$@" \
+br.test -test.coverprofile="$TEST_DIR/cov.$TEST_NAME.$$.out.log" DEVEL "$@" \
     -L "debug" \
     --ca "$TEST_DIR/certs/ca.pem" \
     --cert "$TEST_DIR/certs/br.pem" \

br/tests/_utils/run_cdc

@@ -16,7 +16,7 @@
 
 set -eux
 
-bin/cdc \
+cdc \
     --ca "$TEST_DIR/certs/ca.pem" \
     --cert "$TEST_DIR/certs/ticdc.pem" \
     --key "$TEST_DIR/certs/ticdc.key" \

br/tests/_utils/run_lightning

@@ -16,16 +16,18 @@
 
 set -eux
 
-echo "[$(date)] <<<<<< RUNNING TEST FOR: tests/$TEST_NAME $@ >>>>>>" >> "$TEST_DIR/lightning.log"
-bin/tidb-lightning.test -test.coverprofile="$TEST_DIR/cov.$TEST_NAME.$$.out" DEVEL \
+cur_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+echo "[$(date)] <<<<<< RUNNING TEST FOR: $cur_dir/../$TEST_NAME $@ >>>>>>" >> "$TEST_DIR/lightning.log"
+tidb-lightning.test -test.coverprofile="$TEST_DIR/cov.$TEST_NAME.$$.out" DEVEL \
     --ca "$TEST_DIR/certs/ca.pem" \
     --cert "$TEST_DIR/certs/lightning.pem" \
     --key "$TEST_DIR/certs/lightning.key" \
     --log-file "$TEST_DIR/lightning.log" \
     --tidb-port 4000 \
     --pd-urls '127.0.0.1:2379' \
-    --config "tests/$TEST_NAME/config.toml" \
-    -d "tests/$TEST_NAME/data" \
+    --config "$cur_dir/../$TEST_NAME/config.toml" \
+    -d "$cur_dir/../$TEST_NAME/data" \
     --sorted-kv-dir "$TEST_DIR/$TEST_NAME.sorted" \
     --enable-checkpoint=0 \
     --check-requirements=0 \

br/tests/_utils/run_lightning_ctl

@@ -16,14 +16,16 @@
 
 set -eux
 
-bin/tidb-lightning-ctl.test -test.coverprofile="$TEST_DIR/cov.ctl.$TEST_NAME.$$.out" DEVEL \
+cur_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+tidb-lightning-ctl.test -test.coverprofile="$TEST_DIR/cov.ctl.$TEST_NAME.$$.out" DEVEL \
     --ca "$TEST_DIR/certs/ca.pem" \
     --cert "$TEST_DIR/certs/lightning.pem" \
     --key "$TEST_DIR/certs/lightning.key" \
     --log-file "$TEST_DIR/lightning.log" \
     --tidb-port 4000 \
     --pd-urls '127.0.0.1:2379' \
-    -d "tests/$TEST_NAME/data" \
+    -d "$cur_dir/../$TEST_NAME/data" \
     --sorted-kv-dir "$TEST_DIR/$TEST_NAME.sorted" \
     --enable-checkpoint=0 \
     --check-requirements=0 \

br/tests/_utils/run_services

@@ -29,6 +29,8 @@ export TIKV_COUNT=3
 export TIFLASH_HTTP="127.0.0.1:8125"
 export TIKV_PIDS="${TEST_DIR:?}/tikv_pids.txt"
 
+cur_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
 cleanup_data() {
     # Clean up data
     for svc in "br" "tidb" "tiflash" "tikv" "pd"; do
@@ -74,7 +76,7 @@ start_services() {
 start_pd() {
     echo "Starting PD..."
     mkdir -p "$TEST_DIR/pd"
-    bin/pd-server \
+    pd-server \
         --client-urls "https://$PD_ADDR" \
         --peer-urls "https://$PD_PEER_ADDR" \
         --log-file "$TEST_DIR/pd.log" \
@@ -122,7 +124,7 @@ kv_outage() {
             if [ -e "${TIKV_PIDS}_$i" ]; then
                 TIKV_CONFIG=$(cat "${TIKV_PIDS}_$i" | awk '{print $2}')
             else
-                TIKV_CONFIG=${TIKV_CONFIG:-"tests/config/tikv.toml"}
+                TIKV_CONFIG=${TIKV_CONFIG:-"$cur_dir/../config/tikv.toml"}
             fi
             start_tikv "$i"
         done
@@ -137,7 +139,7 @@ start_tikv() {
     i=$1
     echo "Starting TiKV($i)..."
     mkdir -p "$TEST_DIR/tikv${i}"
-    bin/tikv-server \
+    tikv-server \
         --pd "$PD_ADDR" \
         -A "$TIKV_ADDR$i" \
         --status-addr "$TIKV_STATUS_ADDR$i" \
@@ -163,7 +165,7 @@ ensure_tikv() {
 
 start_tidb() {
     echo "Starting TiDB..."
-    bin/tidb-server \
+    tidb-server \
         -P 4000 \
         --status 10080 \
         --advertise-address="127.0.0.1" \
@@ -188,9 +190,9 @@ start_services_impl() {
     stop_services || true
     cleanup_data || true
 
-    TIDB_CONFIG=${TIDB_CONFIG:-"tests/config/tidb.toml"}
-    TIKV_CONFIG=${TIKV_CONFIG:-"tests/config/tikv.toml"}
-    PD_CONFIG=${PD_CONFIG:-"tests/config/pd.toml"}
+    TIDB_CONFIG=${TIDB_CONFIG:-"$cur_dir/../config/tidb.toml"}
+    TIKV_CONFIG=${TIKV_CONFIG:-"$cur_dir/../config/tikv.toml"}
+    PD_CONFIG=${PD_CONFIG:-"$cur_dir/../config/pd.toml"}
     RUN_TIFLASH=true
 
     while [[ $# -gt 0 ]]
@@ -242,8 +244,8 @@ start_services_impl() {
 
 start_tiflash() {
     echo "Starting TiFlash..."
-    tests/_utils/make_tiflash_config
-    LD_LIBRARY_PATH=bin/ bin/tiflash server --config-file="$TEST_DIR/tiflash.toml" &
+    make_tiflash_config
+    tiflash server --config-file="$TEST_DIR/tiflash.toml" &
 
     i=0
     while ! run_curl https://$TIFLASH_HTTP 1>/dev/null 2>&1; do

br/tests/br_azblob/_run.sh

@@ -20,6 +20,7 @@ set -eux
 DB="$TEST_NAME"
 TABLE="usertable"
 DB_COUNT=3
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 AZBLOB_ENDPOINT="http://127.0.0.1:10000/devstoreaccount1"
 # azurite default account
@@ -33,7 +34,7 @@ mkdir -p "$TEST_DIR/$DB"
 # Fill in the database
 for i in $(seq $DB_COUNT); do
     run_sql "CREATE DATABASE $DB${i};"
-    go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
+    go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
 done
 
 for i in $(seq $DB_COUNT); do

br/tests/br_crypter/run.sh

@@ -18,11 +18,12 @@ set -eu
 DB="$TEST_NAME"
 TABLE="usertable"
 DB_COUNT=3
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 function create_db_with_table(){
     for i in $(seq $DB_COUNT); do
         run_sql "CREATE DATABASE $DB${i};"
-        go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
+        go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
     done
 }
 

br/tests/br_crypter2/run.sh

@@ -18,11 +18,12 @@ set -eu
 DB="$TEST_NAME"
 TABLE="usertable"
 DB_COUNT=3
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 function create_db_with_table(){
     for i in $(seq $DB_COUNT); do
         run_sql "CREATE DATABASE $DB${i};"
-        go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
+        go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
     done
 }
 

br/tests/br_db_online_newkv/run.sh

@@ -58,7 +58,7 @@ for i in $(seq $TIKV_COUNT); do
       -A "$TIKV_ADDR$i" \
       --status-addr "$TIKV_STATUS_ADDR$i" \
       --log-file "$TEST_DIR/restore-tikv${i}.log" \
-      -C "tests/config/restore-tikv.toml" \
+      -C "$CUR/../config/restore-tikv.toml" \
       -s "$TEST_DIR/restore-tikv${i}" &
 done
 sleep 5

br/tests/br_debug_meta/run.sh

@@ -17,9 +17,10 @@
 set -eu
 DB="$TEST_NAME"
 TABLE="usertable"
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 run_sql "CREATE DATABASE $DB;"
-go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
+go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
 
 table_region_sql="SELECT COUNT(*) FROM information_schema.tikv_region_status WHERE db_name = '$DB' AND table_name = '$TABLE';"
 for i in $(seq 10); do

br/tests/br_full/run.sh

@@ -18,10 +18,11 @@ set -eu
 DB="$TEST_NAME"
 TABLE="usertable"
 DB_COUNT=3
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 for i in $(seq $DB_COUNT); do
     run_sql "CREATE DATABASE $DB${i};"
-    go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
+    go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
 done
 
 for i in $(seq $DB_COUNT); do

br/tests/br_full_cluster_restore/run.sh

@@ -3,11 +3,12 @@
 set -eu
 
 # we need to keep backup data after restart service
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 backup_dir=$TEST_DIR/keep_${TEST_NAME}
 incr_backup_dir=${backup_dir}_inc
 res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
 br_log_file=$TEST_DIR/br.log
-source tests/_utils/run_services
+source $CUR/../_utils/run_services
 
 function run_sql_as() {
 	user=$1
@@ -25,7 +26,7 @@ function run_sql_as() {
 restart_services
 
 unset BR_LOG_TO_TERM
-run_sql_file tests/${TEST_NAME}/full_data.sql
+run_sql_file $CUR/full_data.sql
 run_br backup full --log-file $br_log_file -s "local://$backup_dir"
 
 run_sql "SELECT user FROM mysql.user WHERE JSON_EXTRACT(user_attributes, '$.resource_group') != '';"

br/tests/br_full_ddl/run.sh

@@ -22,9 +22,10 @@ LOG=/$TEST_DIR/backup.log
 RESTORE_LOG=LOG=/$TEST_DIR/restore.log
 BACKUP_STAT=/$TEST_DIR/backup_stat
 RESOTRE_STAT=/$TEST_DIR/restore_stat
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 run_sql "CREATE DATABASE $DB;"
-go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
+go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
 
 row_count_ori=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}')
 

br/tests/br_full_index/run.sh

@@ -19,10 +19,11 @@ DB="$TEST_NAME"
 TABLE="usertable"
 DB_COUNT=3
 LOG=/$TEST_DIR/backup.log
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 for i in $(seq $DB_COUNT); do
     run_sql "CREATE DATABASE $DB${i};"
-    go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
+    go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
 done
 
 for i in $(seq $DB_COUNT); do

br/tests/br_gcs/run.sh

@@ -18,6 +18,7 @@ set -eux
 DB="$TEST_NAME"
 TABLE="usertable"
 DB_COUNT=3
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 GCS_HOST="localhost"
 GCS_PORT=21808
@@ -51,7 +52,7 @@ mkdir -p "$TEST_DIR/$DB"
 # Fill in the database
 for i in $(seq $DB_COUNT); do
     run_sql "CREATE DATABASE $DB${i};"
-    go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
+    go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
 done
 
 # we need start a oauth server or gcs client will failed to handle request.
@@ -66,10 +67,10 @@ EOF
 )
 
 # save CREDENTIALS to file
-echo $KEY > "tests/$TEST_NAME/config.json"
+echo $KEY > "$CUR/config.json"
 
 # export test CREDENTIALS for gcs oauth
-export GOOGLE_APPLICATION_CREDENTIALS="tests/$TEST_NAME/config.json"
+export GOOGLE_APPLICATION_CREDENTIALS="$CUR/config.json"
 
 # create gcs bucket
 curl -XPOST http://$GCS_HOST:$GCS_PORT/storage/v1/b -d '{"name":"test"}'

br/tests/br_history/run.sh

@@ -18,10 +18,11 @@ set -eu
 DB="$TEST_NAME"
 TABLE="usertable"
 DB_COUNT=3
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 for i in $(seq $DB_COUNT); do
     run_sql "CREATE DATABASE $DB${i};"
-    go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
+    go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
 done
 
 for i in $(seq $DB_COUNT); do

br/tests/br_incremental/run.sh

@@ -19,10 +19,11 @@ DB="$TEST_NAME"
 TABLE="usertable"
 AUTO_ID_TABLE="a"
 ROW_COUNT=10
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 run_sql "CREATE DATABASE $DB;"
 
-go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
+go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
 row_count_ori_full=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}')
 
 run_sql "CREATE TABLE IF NOT EXISTS ${DB}.${AUTO_ID_TABLE} (c1 INT);"
@@ -31,7 +32,7 @@ run_sql "CREATE TABLE IF NOT EXISTS ${DB}.${AUTO_ID_TABLE} (c1 INT);"
 echo "full backup start..."
 run_br --pd $PD_ADDR backup db -s "local://$TEST_DIR/$DB/full" --db $DB
 
-go-ycsb run mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
+go-ycsb run mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
 
 for i in $(seq $ROW_COUNT); do
     run_sql "INSERT INTO ${DB}.${AUTO_ID_TABLE}(c1) VALUES ($i);"

br/tests/br_incremental_ddl/run.sh

@@ -15,10 +15,11 @@
 # limitations under the License.
 
 set -eu
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 DB="$TEST_NAME"
 TABLE="usertable"
 ROW_COUNT=100
-PATH="tests/$TEST_NAME:bin:$PATH"
+PATH="$CUR:bin:$PATH"
 LOG=/$TEST_DIR/backup.log
 
 echo "load data..."

br/tests/br_incremental_index/run.sh

@@ -15,10 +15,11 @@
 # limitations under the License.
 
 set -eu
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 DB="$TEST_NAME"
 TABLE="usertable"
 ROW_COUNT=100
-PATH="tests/$TEST_NAME:bin:$PATH"
+PATH="$CUR:bin:$PATH"
 
 echo "load data..."
 # create database