ci: reuse some code across scripts (risingwavelabs#8926)

Little-Wallace · Apr 10, 2023 · fc5836f · fc5836f
1 parent 10190e8
commit fc5836f
Show file tree

Hide file tree

Showing 31 changed files with 133 additions and 253 deletions.
diff --git a/ci/scripts/build-other.sh b/ci/scripts/build-other.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 
 echo "--- Build Java connector node"

diff --git a/ci/scripts/build-simulation.sh b/ci/scripts/build-simulation.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 echo "--- Generate RiseDev CI config"
 cp ci/risedev-components.ci.env risedev-components.user.env

diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 while getopts 't:p:' opt; do
     case ${opt} in

diff --git a/ci/scripts/check.sh b/ci/scripts/check.sh
@@ -5,8 +5,8 @@ set -euo pipefail
 
 # Check ci bash scripts contains `set -euo pipefail`.
 for script in ci/**/*.sh; do
-    # skip .env.sh
-    if [[ "$script" == *".env.sh" ]]; then
+    # skip .env.sh and common.sh
+    if [[ "$script" == *"common.sh" ]] || [[ "$script" == *".env.sh" ]]; then
         continue
     fi
     if ! grep -Fq 'set -euo pipefail' "$script"; then
@@ -15,7 +15,7 @@ for script in ci/**/*.sh; do
     fi
 done
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 echo "--- Run clippy check (dev, all features)"
 cargo clippy --all-targets --all-features --locked -- -D warnings

diff --git a/ci/scripts/common.env.sh b/ci/scripts/common.env.sh
diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
@@ -0,0 +1,80 @@
+export CARGO_TERM_COLOR=always
+export PROTOC_NO_VENDOR=true
+export CARGO_HOME=/risingwave/.cargo
+export RISINGWAVE_CI=true
+export RUST_BACKTRACE=1
+export ENABLE_TELEMETRY=false
+
+if [ -n "${BUILDKITE_COMMIT:-}" ]; then
+  export GIT_SHA=$BUILDKITE_COMMIT
+fi
+
+# Arguments:
+#   $1: cargo build `profile` of the binaries
+#   $2: risedev-components `env` to use
+# 
+# Download risingwave and risedev-dev, and put them in target/debug
+function download_and_prepare_rw() {
+  echo "--- Download RisingWave binaries and prepare environment"
+  if [ -z "$1" ]; then
+    echo "download_and_prepare_rw: missing argument profile"
+    exit 1
+  fi
+  if [ -z "$2" ]; then
+    echo "download_and_prepare_rw: missing argument env"
+    exit 1
+  fi
+  # env is either common or source
+  if [ "$2" != "common" ] && [ "$2" != "source" ]; then
+    echo "download_and_prepare_rw: invalid argument env"
+    exit 1
+  fi
+
+  profile=$1
+  env=$2
+
+  echo -e "\033[33mDownload artifacts\033[0m"
+
+  mkdir -p target/debug
+  buildkite-agent artifact download risingwave-"$profile" target/debug/
+  buildkite-agent artifact download risedev-dev-"$profile" target/debug/
+
+  mv target/debug/risingwave-"$profile" target/debug/risingwave
+  mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
+
+  chmod +x ./target/debug/risingwave
+  chmod +x ./target/debug/risedev-dev
+
+  echo -e "\033[33mGenerate RiseDev CI config\033[0m"
+
+  if [ "$env" = "common" ]; then
+    cp ci/risedev-components.ci.env risedev-components.user.env
+  elif [ "$env" = "source" ]; then
+    cp ci/risedev-components.ci.source.env risedev-components.user.env
+  fi
+
+  echo -e "\033[33mPrepare RiseDev dev cluster\033[0m"
+
+  cargo make pre-start-dev
+  cargo make link-all-in-one-binaries
+}
+
+# Arguments:
+#   $1: cargo build `profile` of the binaries
+function download_java_binding() {
+  echo "--- Download java binding"
+  if [ -z "$1" ]; then
+    echo "download_java_binding: missing argument profile"
+    exit 1
+  fi
+
+  profile=$1
+
+  echo -e "\033[33mDownload artifacts\033[0m"
+
+  mkdir -p target/debug
+  buildkite-agent artifact download librisingwave_java_binding.so-"$profile" target/debug
+  mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so
+
+  export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug
+}
diff --git a/ci/scripts/cron-e2e-test.sh b/ci/scripts/cron-e2e-test.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 export RUN_COMPACTION=1;
 export RUN_META_BACKUP=1;
 export RUN_DELETE_RANGE=1;

diff --git a/ci/scripts/cron-fuzz-test.sh b/ci/scripts/cron-fuzz-test.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 export RUN_SQLSMITH=1
 export SQLSMITH_COUNT=1000
 export TEST_NUM=100

diff --git a/ci/scripts/deterministic-e2e-test.sh b/ci/scripts/deterministic-e2e-test.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 source ci/scripts/pr.env.sh
 
 echo "--- Download artifacts"

diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 echo "--- Download artifacts"
 buildkite-agent artifact download risingwave_simulation .

diff --git a/ci/scripts/deterministic-scale-test.sh b/ci/scripts/deterministic-scale-test.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 echo "--- Download artifacts"
 buildkite-agent artifact download scale-test.tar.zst .

diff --git a/ci/scripts/deterministic-unit-test.sh b/ci/scripts/deterministic-unit-test.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 echo "--- Generate RiseDev CI config"
 cp ci/risedev-components.ci.env risedev-components.user.env

diff --git a/ci/scripts/docslt.sh b/ci/scripts/docslt.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 echo "--- Extract DocSlt end-to-end tests"
 cargo run --bin risedev-docslt

diff --git a/ci/scripts/e2e-iceberg-sink-test.sh b/ci/scripts/e2e-iceberg-sink-test.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 while getopts 'p:' opt; do
     case ${opt} in
@@ -21,34 +21,17 @@ while getopts 'p:' opt; do
 done
 shift $((OPTIND -1))
 
-echo "--- Download artifacts"
-mkdir -p target/debug
-buildkite-agent artifact download risingwave-"$profile" target/debug/
-buildkite-agent artifact download risedev-dev-"$profile" target/debug/
-buildkite-agent artifact download librisingwave_java_binding.so-"$profile" target/debug
-mv target/debug/risingwave-"$profile" target/debug/risingwave
-mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
-mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so
+download_and_prepare_rw "$profile" source
+
+download_java_binding "$profile"
 
-export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug
 export RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT=stream_chunk
 
 echo "--- Download connector node package"
 buildkite-agent artifact download risingwave-connector.tar.gz ./
 mkdir ./connector-node
 tar xf ./risingwave-connector.tar.gz -C ./connector-node
 
-echo "--- Adjust permission"
-chmod +x ./target/debug/risingwave
-chmod +x ./target/debug/risedev-dev
-
-echo "--- Generate RiseDev CI config"
-cp ci/risedev-components.ci.source.env risedev-components.user.env
-
-echo "--- Prepare RiseDev dev cluster"
-cargo make pre-start-dev
-cargo make link-all-in-one-binaries
-
 echo "--- starting risingwave cluster with connector node"
 mkdir -p .risingwave/log
 ./connector-node/start-service.sh -p 50051 > .risingwave/log/connector-sink.log 2>&1 &

diff --git a/ci/scripts/e2e-sink-test.sh b/ci/scripts/e2e-sink-test.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 while getopts 'p:' opt; do
     case ${opt} in
@@ -21,16 +21,10 @@ while getopts 'p:' opt; do
 done
 shift $((OPTIND -1))
 
-echo "--- Download artifacts"
-mkdir -p target/debug
-buildkite-agent artifact download risingwave-"$profile" target/debug/
-buildkite-agent artifact download risedev-dev-"$profile" target/debug/
-buildkite-agent artifact download librisingwave_java_binding.so-"$profile" target/debug
-mv target/debug/risingwave-"$profile" target/debug/risingwave
-mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
-mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so
+download_and_prepare_rw "$profile" source
+
+download_java_binding "$profile"
 
-export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug
 # TODO: Switch to stream_chunk encoding once it's completed, and then remove json encoding as well as this env var.
 export RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT=stream_chunk
 
@@ -39,18 +33,6 @@ buildkite-agent artifact download risingwave-connector.tar.gz ./
 mkdir ./connector-node
 tar xf ./risingwave-connector.tar.gz -C ./connector-node
 
-
-echo "--- Adjust permission"
-chmod +x ./target/debug/risingwave
-chmod +x ./target/debug/risedev-dev
-
-echo "--- Generate RiseDev CI config"
-cp ci/risedev-components.ci.source.env risedev-components.user.env
-
-echo "--- Prepare RiseDev dev cluster"
-cargo make pre-start-dev
-cargo make link-all-in-one-binaries
-
 # prepare environment mysql sink
 mysql --host=mysql --port=3306 -u root -p123456 -e "CREATE DATABASE IF NOT EXISTS test;"
 # grant access to `test` for ci test user

diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 # prepare environment
 export CONNECTOR_RPC_ENDPOINT="localhost:50051"
@@ -24,17 +24,9 @@ while getopts 'p:' opt; do
 done
 shift $((OPTIND -1))
 
-echo "--- Download artifacts"
-mkdir -p target/debug
-buildkite-agent artifact download risingwave-"$profile" target/debug/
-buildkite-agent artifact download risedev-dev-"$profile" target/debug/
-buildkite-agent artifact download librisingwave_java_binding.so-"$profile" target/debug
-mv target/debug/risingwave-"$profile" target/debug/risingwave
-mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
-mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so
-
-export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug
+download_and_prepare_rw "$profile" source
 
+download_java_binding "$profile"
 
 echo "--- Download connector node package"
 buildkite-agent artifact download risingwave-connector.tar.gz ./
@@ -46,16 +38,7 @@ cp src/connector/src/test_data/simple-schema.avsc ./avro-simple-schema.avsc
 cp src/connector/src/test_data/complex-schema.avsc ./avro-complex-schema.avsc
 cp src/connector/src/test_data/complex-schema ./proto-complex-schema
 
-echo "--- Adjust permission"
-chmod +x ./target/debug/risingwave
-chmod +x ./target/debug/risedev-dev
-
-echo "--- Generate RiseDev CI config"
-cp ci/risedev-components.ci.source.env risedev-components.user.env
 
-echo "--- Prepare RiseDev dev cluster"
-cargo make pre-start-dev
-cargo make link-all-in-one-binaries
 
 echo "--- e2e, ci-1cn-1fe, mysql & postgres cdc"
 

diff --git a/ci/scripts/e2e-test-parallel-for-opendal.sh b/ci/scripts/e2e-test-parallel-for-opendal.sh
@@ -3,7 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
+source ci/scripts/common.sh
 
 while getopts 'p:' opt; do
     case ${opt} in
@@ -21,24 +21,10 @@ while getopts 'p:' opt; do
 done
 shift $((OPTIND -1))
 
+download_and_prepare_rw "$profile" common
+
 echo "--- Download artifacts"
-mkdir -p target/debug
-buildkite-agent artifact download risingwave-"$profile" target/debug/
-buildkite-agent artifact download risedev-dev-"$profile" target/debug/
 buildkite-agent artifact download "e2e_test/generated/*" ./
-mv target/debug/risingwave-"$profile" target/debug/risingwave
-mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
-
-echo "--- Adjust permission"
-chmod +x ./target/debug/risingwave
-chmod +x ./target/debug/risedev-dev
-
-echo "--- Generate RiseDev CI config"
-cp ci/risedev-components.ci.env risedev-components.user.env
-
-echo "--- Prepare RiseDev dev cluster"
-cargo make pre-start-dev
-cargo make link-all-in-one-binaries
 
 host_args="-h localhost -p 4565 -h localhost -p 4566 -h localhost -p 4567"