From fc5836f08cd2ad6a71571c23eb72f7658038790a Mon Sep 17 00:00:00 2001 From: xxchan Date: Mon, 10 Apr 2023 08:33:22 +0200 Subject: [PATCH] ci: reuse some code across scripts (#8926) --- ci/scripts/build-other.sh | 2 +- ci/scripts/build-simulation.sh | 2 +- ci/scripts/build.sh | 2 +- ci/scripts/check.sh | 6 +- ci/scripts/common.env.sh | 10 --- ci/scripts/common.sh | 80 +++++++++++++++++++ ci/scripts/cron-e2e-test.sh | 2 +- ci/scripts/cron-fuzz-test.sh | 2 +- ci/scripts/deterministic-e2e-test.sh | 2 +- ci/scripts/deterministic-recovery-test.sh | 2 +- ci/scripts/deterministic-scale-test.sh | 2 +- ci/scripts/deterministic-unit-test.sh | 2 +- ci/scripts/docslt.sh | 2 +- ci/scripts/e2e-iceberg-sink-test.sh | 25 +----- ci/scripts/e2e-sink-test.sh | 26 +----- ci/scripts/e2e-source-test.sh | 23 +----- ci/scripts/e2e-test-parallel-for-opendal.sh | 20 +---- ci/scripts/e2e-test-parallel-in-memory.sh | 20 +---- ci/scripts/e2e-test-parallel.sh | 20 +---- ci/scripts/e2e-test.sh | 2 +- ci/scripts/java-binding-test.sh | 25 +----- ci/scripts/misc-check.sh | 2 +- ci/scripts/pr-fuzz-test.sh | 2 +- ci/scripts/pr-unit-test.sh | 2 +- ci/scripts/regress-test.sh | 19 +---- ci/scripts/release.sh | 2 +- ci/scripts/run-e2e-test.sh | 16 +--- ci/scripts/run-fuzz-test.sh | 18 +---- .../s3-source-test-for-opendal-fs-engine.sh | 23 +----- ci/scripts/s3-source-test.sh | 23 +----- ci/scripts/unit-test.sh | 2 +- 31 files changed, 133 insertions(+), 253 deletions(-) delete mode 100644 ci/scripts/common.env.sh create mode 100644 ci/scripts/common.sh diff --git a/ci/scripts/build-other.sh b/ci/scripts/build-other.sh index 0d9e3a6aae1d..b04276ce7a7a 100755 --- a/ci/scripts/build-other.sh +++ b/ci/scripts/build-other.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh echo "--- Build Java connector node" diff --git a/ci/scripts/build-simulation.sh b/ci/scripts/build-simulation.sh index 93f7e9e22d97..ac0c41539bea 100755 --- a/ci/scripts/build-simulation.sh +++ b/ci/scripts/build-simulation.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh echo "--- Generate RiseDev CI config" cp ci/risedev-components.ci.env risedev-components.user.env diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index ca8e826cbf62..f0039d4666f0 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh while getopts 't:p:' opt; do case ${opt} in diff --git a/ci/scripts/check.sh b/ci/scripts/check.sh index 725c09f60753..9d52cf34f333 100755 --- a/ci/scripts/check.sh +++ b/ci/scripts/check.sh @@ -5,8 +5,8 @@ set -euo pipefail # Check ci bash scripts contains `set -euo pipefail`. for script in ci/**/*.sh; do - # skip .env.sh - if [[ "$script" == *".env.sh" ]]; then + # skip .env.sh and common.sh + if [[ "$script" == *"common.sh" ]] || [[ "$script" == *".env.sh" ]]; then continue fi if ! grep -Fq 'set -euo pipefail' "$script"; then @@ -15,7 +15,7 @@ for script in ci/**/*.sh; do fi done -source ci/scripts/common.env.sh +source ci/scripts/common.sh echo "--- Run clippy check (dev, all features)" cargo clippy --all-targets --all-features --locked -- -D warnings diff --git a/ci/scripts/common.env.sh b/ci/scripts/common.env.sh deleted file mode 100644 index dbaded683b35..000000000000 --- a/ci/scripts/common.env.sh +++ /dev/null @@ -1,10 +0,0 @@ -export CARGO_TERM_COLOR=always -export PROTOC_NO_VENDOR=true -export CARGO_HOME=/risingwave/.cargo -export RISINGWAVE_CI=true -export RUST_BACKTRACE=1 -export ENABLE_TELEMETRY=false - -if [ -n "${BUILDKITE_COMMIT:-}" ]; then - export GIT_SHA=$BUILDKITE_COMMIT -fi diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh new file mode 100644 index 000000000000..0a91c69ac6c4 --- /dev/null +++ b/ci/scripts/common.sh @@ -0,0 +1,80 @@ +export CARGO_TERM_COLOR=always +export PROTOC_NO_VENDOR=true +export CARGO_HOME=/risingwave/.cargo +export RISINGWAVE_CI=true +export RUST_BACKTRACE=1 +export ENABLE_TELEMETRY=false + +if [ -n "${BUILDKITE_COMMIT:-}" ]; then + export GIT_SHA=$BUILDKITE_COMMIT +fi + +# Arguments: +# $1: cargo build `profile` of the binaries +# $2: risedev-components `env` to use +# +# Download risingwave and risedev-dev, and put them in target/debug +function download_and_prepare_rw() { + echo "--- Download RisingWave binaries and prepare environment" + if [ -z "$1" ]; then + echo "download_and_prepare_rw: missing argument profile" + exit 1 + fi + if [ -z "$2" ]; then + echo "download_and_prepare_rw: missing argument env" + exit 1 + fi + # env is either common or source + if [ "$2" != "common" ] && [ "$2" != "source" ]; then + echo "download_and_prepare_rw: invalid argument env" + exit 1 + fi + + profile=$1 + env=$2 + + echo -e "\033[33mDownload artifacts\033[0m" + + mkdir -p target/debug + buildkite-agent artifact download risingwave-"$profile" target/debug/ + buildkite-agent artifact download risedev-dev-"$profile" target/debug/ + + mv target/debug/risingwave-"$profile" target/debug/risingwave + mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev + + chmod +x ./target/debug/risingwave + chmod +x ./target/debug/risedev-dev + + echo -e "\033[33mGenerate RiseDev CI config\033[0m" + + if [ "$env" = "common" ]; then + cp ci/risedev-components.ci.env risedev-components.user.env + elif [ "$env" = "source" ]; then + cp ci/risedev-components.ci.source.env risedev-components.user.env + fi + + echo -e "\033[33mPrepare RiseDev dev cluster\033[0m" + + cargo make pre-start-dev + cargo make link-all-in-one-binaries +} + +# Arguments: +# $1: cargo build `profile` of the binaries +function download_java_binding() { + echo "--- Download java binding" + if [ -z "$1" ]; then + echo "download_java_binding: missing argument profile" + exit 1 + fi + + profile=$1 + + echo -e "\033[33mDownload artifacts\033[0m" + + mkdir -p target/debug + buildkite-agent artifact download librisingwave_java_binding.so-"$profile" target/debug + mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so + + export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug +} diff --git a/ci/scripts/cron-e2e-test.sh b/ci/scripts/cron-e2e-test.sh index 1bf57d4b18bd..2e8c56c3c1d5 100755 --- a/ci/scripts/cron-e2e-test.sh +++ b/ci/scripts/cron-e2e-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh export RUN_COMPACTION=1; export RUN_META_BACKUP=1; export RUN_DELETE_RANGE=1; diff --git a/ci/scripts/cron-fuzz-test.sh b/ci/scripts/cron-fuzz-test.sh index 2398f4c571c6..2912546d14cb 100755 --- a/ci/scripts/cron-fuzz-test.sh +++ b/ci/scripts/cron-fuzz-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh export RUN_SQLSMITH=1 export SQLSMITH_COUNT=1000 export TEST_NUM=100 diff --git a/ci/scripts/deterministic-e2e-test.sh b/ci/scripts/deterministic-e2e-test.sh index bc2c9936138e..d061fd5f76dd 100755 --- a/ci/scripts/deterministic-e2e-test.sh +++ b/ci/scripts/deterministic-e2e-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh source ci/scripts/pr.env.sh echo "--- Download artifacts" diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh index 6ff9f51c4ce4..096cdd1eb25e 100755 --- a/ci/scripts/deterministic-recovery-test.sh +++ b/ci/scripts/deterministic-recovery-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh echo "--- Download artifacts" buildkite-agent artifact download risingwave_simulation . diff --git a/ci/scripts/deterministic-scale-test.sh b/ci/scripts/deterministic-scale-test.sh index 0159331e2de7..0bb525bed931 100755 --- a/ci/scripts/deterministic-scale-test.sh +++ b/ci/scripts/deterministic-scale-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh echo "--- Download artifacts" buildkite-agent artifact download scale-test.tar.zst . diff --git a/ci/scripts/deterministic-unit-test.sh b/ci/scripts/deterministic-unit-test.sh index 55465cd18823..509810d03e88 100755 --- a/ci/scripts/deterministic-unit-test.sh +++ b/ci/scripts/deterministic-unit-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh echo "--- Generate RiseDev CI config" cp ci/risedev-components.ci.env risedev-components.user.env diff --git a/ci/scripts/docslt.sh b/ci/scripts/docslt.sh index 79a2388cea78..9a8c27d97298 100755 --- a/ci/scripts/docslt.sh +++ b/ci/scripts/docslt.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh echo "--- Extract DocSlt end-to-end tests" cargo run --bin risedev-docslt diff --git a/ci/scripts/e2e-iceberg-sink-test.sh b/ci/scripts/e2e-iceberg-sink-test.sh index 011a6c2f41f0..1b366cc9e0a5 100755 --- a/ci/scripts/e2e-iceberg-sink-test.sh +++ b/ci/scripts/e2e-iceberg-sink-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh while getopts 'p:' opt; do case ${opt} in @@ -21,16 +21,10 @@ while getopts 'p:' opt; do done shift $((OPTIND -1)) -echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ -buildkite-agent artifact download librisingwave_java_binding.so-"$profile" target/debug -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev -mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so +download_and_prepare_rw "$profile" source + +download_java_binding "$profile" -export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug export RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT=stream_chunk echo "--- Download connector node package" @@ -38,17 +32,6 @@ buildkite-agent artifact download risingwave-connector.tar.gz ./ mkdir ./connector-node tar xf ./risingwave-connector.tar.gz -C ./connector-node -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev - -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.source.env risedev-components.user.env - -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries - echo "--- starting risingwave cluster with connector node" mkdir -p .risingwave/log ./connector-node/start-service.sh -p 50051 > .risingwave/log/connector-sink.log 2>&1 & diff --git a/ci/scripts/e2e-sink-test.sh b/ci/scripts/e2e-sink-test.sh index a98130fec016..056207ff0d97 100755 --- a/ci/scripts/e2e-sink-test.sh +++ b/ci/scripts/e2e-sink-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh while getopts 'p:' opt; do case ${opt} in @@ -21,16 +21,10 @@ while getopts 'p:' opt; do done shift $((OPTIND -1)) -echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ -buildkite-agent artifact download librisingwave_java_binding.so-"$profile" target/debug -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev -mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so +download_and_prepare_rw "$profile" source + +download_java_binding "$profile" -export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug # TODO: Switch to stream_chunk encoding once it's completed, and then remove json encoding as well as this env var. export RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT=stream_chunk @@ -39,18 +33,6 @@ buildkite-agent artifact download risingwave-connector.tar.gz ./ mkdir ./connector-node tar xf ./risingwave-connector.tar.gz -C ./connector-node - -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev - -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.source.env risedev-components.user.env - -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries - # prepare environment mysql sink mysql --host=mysql --port=3306 -u root -p123456 -e "CREATE DATABASE IF NOT EXISTS test;" # grant access to `test` for ci test user diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh index 4883e049acda..6470f92d2d7f 100755 --- a/ci/scripts/e2e-source-test.sh +++ b/ci/scripts/e2e-source-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh # prepare environment export CONNECTOR_RPC_ENDPOINT="localhost:50051" @@ -24,17 +24,9 @@ while getopts 'p:' opt; do done shift $((OPTIND -1)) -echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ -buildkite-agent artifact download librisingwave_java_binding.so-"$profile" target/debug -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev -mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so - -export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug +download_and_prepare_rw "$profile" source +download_java_binding "$profile" echo "--- Download connector node package" buildkite-agent artifact download risingwave-connector.tar.gz ./ @@ -46,16 +38,7 @@ cp src/connector/src/test_data/simple-schema.avsc ./avro-simple-schema.avsc cp src/connector/src/test_data/complex-schema.avsc ./avro-complex-schema.avsc cp src/connector/src/test_data/complex-schema ./proto-complex-schema -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev - -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.source.env risedev-components.user.env -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries echo "--- e2e, ci-1cn-1fe, mysql & postgres cdc" diff --git a/ci/scripts/e2e-test-parallel-for-opendal.sh b/ci/scripts/e2e-test-parallel-for-opendal.sh index 3a1a80445a45..5760bd3a4242 100755 --- a/ci/scripts/e2e-test-parallel-for-opendal.sh +++ b/ci/scripts/e2e-test-parallel-for-opendal.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh while getopts 'p:' opt; do case ${opt} in @@ -21,24 +21,10 @@ while getopts 'p:' opt; do done shift $((OPTIND -1)) +download_and_prepare_rw "$profile" common + echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ buildkite-agent artifact download "e2e_test/generated/*" ./ -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev - -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev - -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.env risedev-components.user.env - -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries host_args="-h localhost -p 4565 -h localhost -p 4566 -h localhost -p 4567" diff --git a/ci/scripts/e2e-test-parallel-in-memory.sh b/ci/scripts/e2e-test-parallel-in-memory.sh index d161bc6f6bef..03e39e093ffb 100755 --- a/ci/scripts/e2e-test-parallel-in-memory.sh +++ b/ci/scripts/e2e-test-parallel-in-memory.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh while getopts 'p:' opt; do case ${opt} in @@ -21,23 +21,7 @@ while getopts 'p:' opt; do done shift $((OPTIND -1)) -echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev - -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev - -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.env risedev-components.user.env - -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries +download_and_prepare_rw "$profile" common host_args="-h localhost -p 4565 -h localhost -p 4566 -h localhost -p 4567" diff --git a/ci/scripts/e2e-test-parallel.sh b/ci/scripts/e2e-test-parallel.sh index 10ebe815295c..4cfa58e5ab56 100755 --- a/ci/scripts/e2e-test-parallel.sh +++ b/ci/scripts/e2e-test-parallel.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh while getopts 'p:' opt; do case ${opt} in @@ -21,24 +21,10 @@ while getopts 'p:' opt; do done shift $((OPTIND -1)) +download_and_prepare_rw "$profile" common + echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ buildkite-agent artifact download "e2e_test/generated/*" ./ -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev - -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev - -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.env risedev-components.user.env - -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries host_args="-h localhost -p 4565 -h localhost -p 4566 -h localhost -p 4567" diff --git a/ci/scripts/e2e-test.sh b/ci/scripts/e2e-test.sh index 773993ec12fa..23e5db6ec55c 100755 --- a/ci/scripts/e2e-test.sh +++ b/ci/scripts/e2e-test.sh @@ -3,6 +3,6 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh source ci/scripts/pr.env.sh source ci/scripts/run-e2e-test.sh diff --git a/ci/scripts/java-binding-test.sh b/ci/scripts/java-binding-test.sh index 8818bd7d8a41..878a8b779364 100755 --- a/ci/scripts/java-binding-test.sh +++ b/ci/scripts/java-binding-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh while getopts 'p:' opt; do case ${opt} in @@ -21,26 +21,9 @@ while getopts 'p:' opt; do done shift $((OPTIND -1)) -echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ -buildkite-agent artifact download librisingwave_java_binding.so-"$profile" target/debug -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev -mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so - -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev -chmod +x ./target/debug/librisingwave_java_binding.so - -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.source.env risedev-components.user.env - -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries +download_and_prepare_rw "$profile" source + +download_java_binding "$profile" echo "--- starting risingwave cluster" cargo make ci-start java-binding-demo diff --git a/ci/scripts/misc-check.sh b/ci/scripts/misc-check.sh index e29b20c743c3..16bdbd929432 100755 --- a/ci/scripts/misc-check.sh +++ b/ci/scripts/misc-check.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh echo "--- Check protobuf code format && Lint protobuf" cd proto diff --git a/ci/scripts/pr-fuzz-test.sh b/ci/scripts/pr-fuzz-test.sh index add474896a96..f19bd0d8312b 100755 --- a/ci/scripts/pr-fuzz-test.sh +++ b/ci/scripts/pr-fuzz-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh set +e # Set features, depending on our workflow diff --git a/ci/scripts/pr-unit-test.sh b/ci/scripts/pr-unit-test.sh index 127e983fee4f..25f11f446206 100755 --- a/ci/scripts/pr-unit-test.sh +++ b/ci/scripts/pr-unit-test.sh @@ -3,6 +3,6 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh source ci/scripts/pr.env.sh ./ci/scripts/run-unit-test.sh diff --git a/ci/scripts/regress-test.sh b/ci/scripts/regress-test.sh index c134400c6e97..9bee7617a074 100755 --- a/ci/scripts/regress-test.sh +++ b/ci/scripts/regress-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh while getopts 'p:' opt; do case ${opt} in @@ -21,27 +21,14 @@ while getopts 'p:' opt; do done shift $((OPTIND -1)) +download_and_prepare_rw "$profile" common + echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ buildkite-agent artifact download risingwave_regress_test-"$profile" target/debug/ -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev mv target/debug/risingwave_regress_test-"$profile" target/debug/risingwave_regress_test -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev chmod +x ./target/debug/risingwave_regress_test -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.env risedev-components.user.env - -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries - echo "--- Postgres regress test" apt-get update -yy apt-get -y install locales diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index cbbf8892df7e..62716760d52d 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -20,7 +20,7 @@ mvn -v echo "--- Install rust" curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain $(cat ./rust-toolchain) -y source "$HOME/.cargo/env" -source ci/scripts/common.env.sh +source ci/scripts/common.sh echo "--- Install protoc3" curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip diff --git a/ci/scripts/run-e2e-test.sh b/ci/scripts/run-e2e-test.sh index 2140503ba191..a5a3a8126fba 100755 --- a/ci/scripts/run-e2e-test.sh +++ b/ci/scripts/run-e2e-test.sh @@ -19,27 +19,15 @@ while getopts 'p:' opt; do done shift $((OPTIND -1)) +download_and_prepare_rw "$profile" common + echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ buildkite-agent artifact download "e2e_test/generated/*" ./ buildkite-agent artifact download risingwave_e2e_extended_mode_test-"$profile" target/debug/ -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev mv target/debug/risingwave_e2e_extended_mode_test-"$profile" target/debug/risingwave_e2e_extended_mode_test -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev chmod +x ./target/debug/risingwave_e2e_extended_mode_test -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.env risedev-components.user.env - -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries echo "--- e2e, ci-3cn-1fe, streaming" RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ diff --git a/ci/scripts/run-fuzz-test.sh b/ci/scripts/run-fuzz-test.sh index 1172dff93d9b..cfb3bc5ac43f 100755 --- a/ci/scripts/run-fuzz-test.sh +++ b/ci/scripts/run-fuzz-test.sh @@ -20,26 +20,12 @@ if [[ "$RUN_SQLSMITH" -eq "1" ]]; then done shift $((OPTIND -1)) + download_and_prepare_rw "$profile" common + echo "--- Download artifacts" - mkdir -p target/debug - buildkite-agent artifact download risingwave-"$profile" target/debug/ - buildkite-agent artifact download risedev-dev-"$profile" target/debug/ - mv target/debug/risingwave-"$profile" target/debug/risingwave - mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev buildkite-agent artifact download risingwave_simulation . chmod +x ./risingwave_simulation - echo "--- Adjust permission" - chmod +x ./target/debug/risingwave - chmod +x ./target/debug/risedev-dev - - echo "--- Generate RiseDev CI config" - cp ci/risedev-components.ci.env risedev-components.user.env - - echo "--- Prepare RiseDev dev cluster" - cargo make pre-start-dev - cargo make link-all-in-one-binaries - echo "--- Run sqlsmith tests" NEXTEST_PROFILE=ci cargo nextest run run_sqlsmith_on_frontend --features "failpoints sync_point enable_sqlsmith_unit_test" 2> >(tee); diff --git a/ci/scripts/s3-source-test-for-opendal-fs-engine.sh b/ci/scripts/s3-source-test-for-opendal-fs-engine.sh index 20fd77483a7a..6fbbdb35e0e4 100755 --- a/ci/scripts/s3-source-test-for-opendal-fs-engine.sh +++ b/ci/scripts/s3-source-test-for-opendal-fs-engine.sh @@ -2,7 +2,7 @@ set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh while getopts 'p:s:' opt; do case ${opt} in @@ -23,26 +23,7 @@ while getopts 'p:s:' opt; do done shift $((OPTIND -1)) - - -echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ - -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev - -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev - -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.env risedev-components.user.env - -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries +download_and_prepare_rw "$profile" common echo "--- starting risingwave cluster with connector node" cargo make ci-start ci-3cn-3fe-opendal-fs-backend diff --git a/ci/scripts/s3-source-test.sh b/ci/scripts/s3-source-test.sh index 4d482568cad2..710ba63b6fd6 100755 --- a/ci/scripts/s3-source-test.sh +++ b/ci/scripts/s3-source-test.sh @@ -2,7 +2,7 @@ set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh while getopts 'p:s:' opt; do case ${opt} in @@ -23,26 +23,7 @@ while getopts 'p:s:' opt; do done shift $((OPTIND -1)) - - -echo "--- Download artifacts" -mkdir -p target/debug -buildkite-agent artifact download risingwave-"$profile" target/debug/ -buildkite-agent artifact download risedev-dev-"$profile" target/debug/ - -mv target/debug/risingwave-"$profile" target/debug/risingwave -mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev - -echo "--- Adjust permission" -chmod +x ./target/debug/risingwave -chmod +x ./target/debug/risedev-dev - -echo "--- Generate RiseDev CI config" -cp ci/risedev-components.ci.env risedev-components.user.env - -echo "--- Prepare RiseDev dev cluster" -cargo make pre-start-dev -cargo make link-all-in-one-binaries +download_and_prepare_rw "$profile" source echo "--- starting risingwave cluster with connector node" cargo make ci-start ci-1cn-1fe diff --git a/ci/scripts/unit-test.sh b/ci/scripts/unit-test.sh index d391c11bc869..0602e8c0df5a 100755 --- a/ci/scripts/unit-test.sh +++ b/ci/scripts/unit-test.sh @@ -3,7 +3,7 @@ # Exits as soon as any line fails. set -euo pipefail -source ci/scripts/common.env.sh +source ci/scripts/common.sh source ci/scripts/run-unit-test.sh echo "--- Show sccache stats"