Makefile.toml

extend = [
  { path = "src/risedevtool/grafana.toml" },
  { path = "src/risedevtool/prometheus.toml" },
  { path = "src/risedevtool/minio.toml" },
  { path = "src/risedevtool/etcd.toml" },
  { path = "src/risedevtool/tempo.toml" },
  { path = "src/risedevtool/kafka.toml" },
  { path = "src/risedevtool/gcloud-pubsub.toml" },
  { path = "src/risedevtool/redis.toml" },
  { path = "src/risedevtool/connector.toml" },
  { path = "src/risedevtool/risedev-components.toml" },
  { path = "src/sqlparser/test_runner/sqlparser_test.toml" },
  { path = "src/frontend/planner_test/planner_test.toml" },
  { path = "src/tests/compaction_test/Makefile.toml" },
  { path = "src/storage/backup/integration_tests/Makefile.toml" },
  { path = "src/java_binding/make-java-binding.toml" },
  { path = "src/stream/tests/integration_tests/integration_test.toml" },
]

env_files = ["./risedev-components.user.env"]

env_scripts = [
  '''
#!@duckscript

# only duckscript can modify env variables in cargo-make
# duckscript doc: https://github.com/sagiegurari/duckscript/blob/master/docs/sdk.md

set_env ENABLE_TELEMETRY "false"

is_sanitizer_enabled = get_env ENABLE_SANITIZER
is_hdfs_backend = get_env ENABLE_HDFS
is_release = get_env ENABLE_RELEASE_PROFILE
is_not_release = not ${is_release}
is_dynamic_linking = get_env ENABLE_DYNAMIC_LINKING
is_hummock_trace = get_env ENABLE_HUMMOCK_TRACE
is_python_udf_enabled = get_env ENABLE_PYTHON_UDF

if ${is_sanitizer_enabled}
  set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS "-Zbuild-std --target ${CARGO_MAKE_RUST_TARGET_TRIPLE}"
  set_env RISEDEV_BUILD_TARGET_DIR "${CARGO_MAKE_RUST_TARGET_TRIPLE}/"
  set_env RISEDEV_RUSTFLAGS "-Ctarget-cpu=native --cfg tokio_unstable -Zsanitizer=thread"
else
  set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS ""
  set_env RISEDEV_BUILD_TARGET_DIR ""
end

set_env RISEDEV_CARGO_BUILD_CRATE "risingwave_cmd_all"
set_env RISEDEV_CTL_RUN_CMD "-- risectl"

if ${is_hdfs_backend}
  set_env BUILD_HDFS_BACKEND_CMD "-p risingwave_object_store --features hdfs-backend"
else
  set_env BUILD_HDFS_BACKEND_CMD ""
end

if ${is_not_release} and ${is_dynamic_linking}
  set_env RISINGWAVE_FEATURE_FLAGS "--features rw-dynamic-link --no-default-features"
else
  set_env RISINGWAVE_FEATURE_FLAGS "--features rw-static-link"
end

if ${is_python_udf_enabled}
  flags = get_env RISINGWAVE_FEATURE_FLAGS
  set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features embedded-python-udf"
end

if ${is_hummock_trace}
  set_env BUILD_HUMMOCK_TRACE_CMD "-p risingwave_storage --features hm-trace"
else
  set_env BUILD_HUMMOCK_TRACE_CMD ""
end

is_ci = get_env RISINGWAVE_CI
is_not_ci = not ${is_ci}

if ${is_not_ci}
  query_log_path = get_env RW_QUERY_LOG_PATH
  no_query_log_path = not ${query_log_path}

  if ${no_query_log_path}
    set_env RW_QUERY_LOG_PATH "${PREFIX_LOG}"
  fi

  rust_log = get_env RUST_LOG
  no_rust_log = not ${rust_log}

  if ${no_rust_log}
    set_env RUST_LOG "pgwire_query_log=info"
  else
    set_env RUST_LOG "pgwire_query_log=info,${rust_log}"
  end
end
''',
]


[config]
default_to_workspace = false
skip_core_tasks = true
skip_git_env_info = true
skip_rust_env_info = false
skip_crate_env_info = true
min_version = "0.36.10"

[tasks.clean-full]
category = "Misc"
description = "Clean all downloaded binaries and all data, config and logs by deleting .risingwave folder"
script = '''
#!@duckscript
rm -rf "${PREFIX}"
'''

[tasks.clean-data]
category = "RiseDev - Start/Stop"
description = "Clean all data, config and logs"
script = '''
#!@duckscript
rm -rf "${PREFIX_DATA}"
rm -rf "${PREFIX_LOG}"
rm -rf "${PREFIX_CONFIG}"
rm -rf "${PREFIX_PROFILING}"
'''


[tasks.l]
alias = "logs"

[tasks.logs]
category = "RiseDev - Start/Stop"
description = "Open logs with VSCode or dump in console"
script = '''
#!/usr/bin/env bash

code "${PREFIX_LOG}" 2> /dev/null
if [[ $? -eq 0 ]]; then
  exit 0
fi

for out_file in ${PREFIX_LOG}/*.log
do
  echo "~~~ Dump log file $out_file"
  echo ""
  cat "$out_file" | tail -n 300
done

echo ""
echo ""
echo ""
'''


[tasks.ls]
category = "Misc"
description = "List all nodes in the cluster"
script = '''
#!/usr/bin/env bash
tmux list-windows -t risedev | grep -v active | cut -d'(' -f1
'''

[tasks.lsw]
category = "Misc"
description = "List --watch all nodes in the cluster"
script = '''
#!/usr/bin/env bash
watch -n 1 "tmux list-windows -t risedev | grep -v active | cut -d'(' -f1"
'''

[tasks.del]
alias = "delete"

[tasks.delete]
category = "RiseDev - Start/Stop"
description = "Delete a node in the cluster"
script = '''
#!/usr/bin/env bash

risedev_ls () {
  tmux list-windows -t risedev | grep -v active | cut -d'(' -f1
}

err () {
  echo "Available nodes are"
  risedev_ls
  exit 1
}

if [[ -z "$1" ]]; then
  echo "Please pass a parameter to this script, defining which node you want to delete"
  err
fi

if [[ -z $(risedev_ls | grep "$1" ) ]]; then
  echo "Please select a node that is currently running"
  err
fi

tmux kill-window -t $1
'''

[tasks.f]
alias = "follow"

[tasks.follow]
category = "Misc"
description = "Follows the end of the logs of a specific component"
script = '''
#!/usr/bin/env bash
set -e

if [[ -z "$1" ]]; then
  echo "Please pass a parameter to this script, defining which logs you want to follow"
  echo "Available logs are..."
  ls ${PREFIX_LOG}
  exit 1
fi

if [[ ! -f ${PREFIX_LOG}/$1 ]]; then
  echo "selected file does not exist"
  echo "Available logs are..."
  ls ${PREFIX_LOG}
  exit 1
fi

tail -f -n 5 ${PREFIX_LOG}/$1
'''

[tasks.check-logs]
private = true
category = "RiseDev - CI"
description = "Check if there is panic in log or significant log size issue"
script = '''
#!/usr/bin/env bash
set -e

echo "~~~ Check logs"
du -ah ${PREFIX_LOG}

for out_file in ${PREFIX_LOG}/*.log
do
  if grep "panicked at" "$out_file" -C 100; then
    echo "$(tput setaf 1)\"panicked at\" found in $out_file$(tput sgr0), please check if there's any bugs in this PR."
    echo "You may find \"risedev-logs\" artifacts and download logs after all workflows finish."
    exit 1
  fi
done

if (( "$(du -sk ${PREFIX_LOG} | cut -f1)" > 3000 )) ; then
    echo "$(tput setaf 1)ERROR: log size is significantly large ($(du -sh ${PREFIX_LOG} | cut -f1)).$(tput sgr0) Please disable unnecessary logs."
    exit 1
fi
'''

[tasks.doc]
category = "Misc"
description = "Open rustdoc for risingwave"
dependencies = ["build-docs"]
script = '''
#!/usr/bin/env bash
set -e

python -mwebbrowser file://$(pwd)/target/doc/index.html
'''

[tasks.link-all-in-one-binaries]
private = true
category = "RiseDev - Build"
description = "Link all-in-one cmds to RiseDev bin"
script = '''
#!/usr/bin/env bash
set -e

rm -rf "${PREFIX_BIN}/risingwave"
mkdir -p "${PREFIX_BIN}/risingwave"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/meta-node"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/compute-node"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/frontend-node"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/compactor"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/risectl"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/playground"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/standalone"
'''

[tasks.codesign-playground]
private = true
category = "RiseDev - Build"
description = "Codesign playground binary to support coredump"
# If core dump is enabled by RiseDev and we're on an Apple Silicon platform,
# codesign the binary before running.
# https://developer.apple.com/forums/thread/694233?answerId=695943022#695943022
condition = { env_set = [
  "ENABLE_COREDUMP",
], env = { "SYSTEM" = "darwin-arm64" } }
script = '''
#!/usr/bin/env bash

set -ex
codesign -s - -f --entitlements scripts/coredump/coredump.entitlements "target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave"
'''

[tasks.codesign-binaries]
private = true
category = "RiseDev - Build"
description = "Codesign all binaries to support coredump"
# If core dump is enabled by RiseDev and we're on an Apple Silicon platform,
# codesign the binary before running.
# https://developer.apple.com/forums/thread/694233?answerId=695943022#695943022
condition = { env_set = [
  "ENABLE_COREDUMP",
], env = { "SYSTEM" = "darwin-arm64" } }
script = '''
#!/usr/bin/env bash
set -e

binaries=("risingwave")

set -ex
echo -n "${binaries[*]}" | parallel -d ' ' \
  "codesign -s - -f --entitlements scripts/coredump/coredump.entitlements \"target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/{}\""
'''

[tasks.link-user-bin]
private = true
category = "RiseDev - Build"
description = "Link all binaries to .bin"
script = '''
#!/usr/bin/env bash
set -e

rm -rf "${PREFIX_USR_BIN}"
mkdir -p "${PREFIX_USR_BIN}"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/meta-node"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/compute-node"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/frontend-node"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/risectl"
ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_USR_BIN}/playground"
'''

[tasks.post-build-risingwave]
category = "RiseDev - Build"
description = "Copy RisngWave binaries to bin"
condition = { env_set = ["ENABLE_BUILD_RUST"] }
dependencies = [
  "link-all-in-one-binaries",
  "link-user-bin",
  "codesign-binaries",
]

[tasks.b]
alias = "build-risingwave"

[tasks.build]
alias = "build-risingwave"

[tasks.extract-dashboard-artifact]
private = true
category = "RiseDev - Build"
description = "Extract dashboard artifact"
condition = { env_not_set = ["ENABLE_BUILD_DASHBOARD"] }
script = '''
#!/usr/bin/env bash

# we allow this script to fail

echo "Extracting dashboard artifacts to ${PREFIX_UI}"

rm -rf "${PREFIX_UI}"
git worktree prune
git worktree add "${PREFIX_UI}" origin/dashboard-artifact
'''

[tasks.export-dashboard]
private = true
category = "RiseDev - Build"
description = "Build dashboard"
condition = { env_set = [
  "ENABLE_BUILD_DASHBOARD",
], files_modified = { input = [
  "./dashboard/**/*.js",
  "./dashboard/**/*.ts*",
  "./dashboard/package.json",
  "./dashboard/next.config.js",
], output = [
  "./dashboard/out/**/*.html",
] } }
script = """
#!/usr/bin/env bash
set -e

rm -rf "${PREFIX_UI}"
cd dashboard && npm run build-static
cd .. && ln -s "$(pwd)/dashboard/out" "${PREFIX_UI}"
"""

[tasks.build-risingwave]
category = "RiseDev - Build"
description = "Build Rust components"
condition = { env_true = ["ENABLE_BUILD_RUST"] }
script = '''
#!/usr/bin/env bash

set -e
echo "$(tput setaf 4)$(tput bold)[Reminder]$(tput sgr0) risedev will only build $(tput setaf 4)risingwave_cmd(_all) and risedev$(tput sgr0) crates."

[[ -z "${RISEDEV_RUSTFLAGS}" ]] || export RUSTFLAGS="${RISEDEV_RUSTFLAGS}"
echo + RUSTFLAGS="${RUSTFLAGS:-<not set>}"
set -xe
cargo build -p ${RISEDEV_CARGO_BUILD_CRATE} -p risedev ${BUILD_HDFS_BACKEND_CMD}\
${BUILD_HUMMOCK_TRACE_CMD}\
            --profile "${RISINGWAVE_BUILD_PROFILE}" \
            ${RISINGWAVE_FEATURE_FLAGS} \
            ${RISEDEV_CARGO_BUILD_EXTRA_ARGS}
'''

[tasks.build-risingwave-playground]
category = "RiseDev - Build"
description = "Build RisingWave playground"
condition = { env_true = ["ENABLE_BUILD_RUST"] }
script = '''
#!/usr/bin/env bash

set -e
[[ -z "${RISEDEV_RUSTFLAGS}" ]] || export RUSTFLAGS="${RISEDEV_RUSTFLAGS}"
echo + RUSTFLAGS="${RUSTFLAGS:-<not set>}"
set -xe

cargo build -p risingwave_cmd_all \
          --profile "${RISINGWAVE_BUILD_PROFILE}" \
          ${RISINGWAVE_FEATURE_FLAGS} \
          ${RISEDEV_CARGO_BUILD_EXTRA_ARGS}
'''

[tasks.clean]
private = true
category = "RiseDev - Build"
description = "Clean all build targets"
dependencies = ["clean-rust", "clean-java"]

[tasks.clean-rust]
private = true
category = "RiseDev - Build"
description = "Clean Rust targets"
condition = { env_set = ["ENABLE_BUILD_RUST"] }
script = '''
#!/usr/bin/env bash
set -e
cargo clean
'''

[tasks.clean-java]
private = true
category = "RiseDev - Build"
description = "Clean Rust targets"
condition = { env_set = ["ENABLE_RW_CONNECTOR", "ENABLE_BUILD_RW_CONNECTOR"] }
script = '''
#!/usr/bin/env bash
set -e

if command -v mvn &> /dev/null; then
  MAVEN_PATH="$(command -v mvn)"
else
  MAVEN_PATH="${PREFIX_BIN}/maven/bin/mvn"
fi

cd "${JAVA_DIR}"
"${MAVEN_PATH}" clean
'''

[tasks.build-docs]
private = true
category = "RiseDev - Build"
description = "Build Rust docs"
condition = { env_set = ["ENABLE_BUILD_RUST"] }
env = { RUSTDOCFLAGS = "--markdown-css ../../docs/rustdoc/rust.css --markdown-no-toc --index-page docs/rustdoc/index.md -Zunstable-options" }
script = '''
#!/usr/bin/env bash

set -e

cargo doc --workspace --no-deps --document-private-items
'''

[tasks.prepare-config]
private = true
category = "RiseDev - Prepare"
description = "Copy necessary configuration files to RiseDev"
script = '''
#!/usr/bin/env bash

set -e

echo > "${PREFIX_CONFIG}/risingwave.toml"
cp "src/risedevtool/run_command.sh" "${PREFIX_BIN}/run_command.sh"
cp "src/risedevtool/welcome.sh" "${PREFIX_BIN}/welcome.sh"
'''

[tasks.download-all]
private = true
category = "Misc"
description = "Download all available components at once"
dependencies = [
  "download-maven",
  "download-etcd",
  "download-grafana",
  "download-tempo",
  "download-kafka",
  "download-mcli",
  "download-minio",
  "download-prometheus",
  "download-pubsub",
  "download-redis",
]

[tasks.create-user-profiles-file]
private = true
category = "RiseDev - Prepare"
description = "Create user profiles file if not exists"
condition = { files_not_exist = [
  "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/risedev-profiles.user.yml",
] }
script = '''
#!/usr/bin/env bash

set -e
cat <<\EOF > "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/risedev-profiles.user.yml"
# This file is used to store extra profiles besides the ones defined in `risedev.yml`.
# It is automatically created by RiseDev.
#
# To use this:
# - add new profiles just like the ones defined under the `profile` section in `risedev.yml`
# - develop the cluster with the new profiles by `./risedev dev <new-profile-name>`

my-default-example:
  config-path: src/config/my_config.toml
  steps:
    - use: meta-node
    - use: compute-node
    - use: frontend
EOF
'''

[tasks.pre-start-dev]
category = "RiseDev - Prepare"
description = "Prepare dev cluster by downloading necessary tools and build required components"
dependencies = [
  "create-user-profiles-file",
  "download-all",
  "build-risingwave",
  "build-connector-node",
  "post-build-risingwave",
  "extract-dashboard-artifact",
  "export-dashboard",
  "prepare-config",
]

[tasks.pre-start-benchmark]
category = "RiseDev - Prepare"
description = "Download necessary tools to deploy a benchmark env"
dependencies = [
  "download-minio",
  "download-mcli",
  "download-etcd",
  "download-grafana",
  "download-prometheus",
  "download-tempo",
  "download-kafka",
  "download-redis",
]

[tasks.pre-start-playground]
category = "RiseDev - Prepare"
description = "Preparation steps for playground"
dependencies = [
  "build-risingwave-playground",
  "codesign-playground",
  "build-connector-node",
]

[tasks.check-risedev-env-file]
private = true
category = "RiseDev - Prepare"
description = "Check if risedev-env file exists"
script = '''
#!/usr/bin/env bash
set -euo pipefail

RC_ENV_FILE="${PREFIX_CONFIG}/risedev-env"

if [ ! -f "${RC_ENV_FILE}" ]; then
  echo "risedev-env file not found. Did you start cluster using $(tput setaf 4)\`./risedev d\`$(tput sgr0) or $(tput setaf 4)\`./risedev p\`$(tput sgr0)?"
  exit 1
fi
'''

[tasks.psql-env]
category = "RiseDev - Start/Stop"
description = "Dump env configuration for psql"
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]
script = '''
#!/usr/bin/env bash
cat <<EOF > "${PREFIX_CONFIG}/psql-env"
export PGHOST=$RW_FRONTEND_LISTEN_ADDRESS
export PGPORT=$RW_FRONTEND_PORT
export PGUSER=root
export PGDATABASE=dev
EOF

echo "psql environment file is created at ${PREFIX_CONFIG}/psql-env"
echo "You can source this file to use psql with default connection parameters."
echo "   $(tput setaf 4)source ${PREFIX_CONFIG}/psql-env$(tput sgr0)"
'''

[tasks.psql]
category = "RiseDev - Start/Stop"
description = "Run local psql client with default connection parameters. You can pass extra arguments to psql."
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]
script = '''
#!/usr/bin/env bash
psql -h $RW_FRONTEND_LISTEN_ADDRESS -p $RW_FRONTEND_PORT -U root -d dev "$@"
'''

[tasks.ctl]
category = "RiseDev - Start/Stop"
description = "Start RiseCtl"
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]
script = '''
#!/usr/bin/env bash
cargo run -p ${RISEDEV_CARGO_BUILD_CRATE} --profile "${RISINGWAVE_BUILD_PROFILE}" ${RISEDEV_CTL_RUN_CMD} "$@"
test $? -eq 0 || exit 1
'''

[tasks.p]
alias = "playground"

[tasks.playground]
category = "RiseDev - Start/Stop"
description = "🌟 Start a lite RisingWave playground using risingwave all-in-one binary"
dependencies = ["pre-start-playground"]
script = '''
#!/usr/bin/env bash

set -e
if [[ $ENABLE_COREDUMP == "true" ]]; then
  echo "+ ulimit -c unlimited"
  ulimit -c unlimited
fi

set -x
target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave playground $@
'''

[tasks.standalone]
category = "RiseDev - Start/Stop"
description = "🌟 Start a RisingWave standalone instance"
dependencies = ["pre-start-playground"]
script = '''
#!/usr/bin/env bash

set -e
if [[ $ENABLE_COREDUMP == "true" ]]; then
  echo "+ ulimit -c unlimited"
  ulimit -c unlimited
fi

set -x
target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave standalone $@
'''

# TODO(kwannoel): Support `tasks.standalone-dev` as well.
# Then we can reuse risedev to start peripheral services, e.g. compactor, etcd, kafka, etc...
[tasks.standalone-demo-dev]
category = "RiseDev - Start/Stop"
description = "🌟 Start a RisingWave standalone demo."
dependencies = ["build"]
script = "./src/cmd_all/scripts/standalone-demo-dev.sh"

[tasks.standalone-demo-full]
category = "RiseDev - Start/Stop"
description = "🌟 Start a RisingWave standalone demo."
dependencies = ["build"]
script = "./src/cmd_all/scripts/standalone-demo-full.sh"

[tasks.d]
alias = "dev"

[tasks.dev]
category = "RiseDev - Start/Stop"
dependencies = ["pre-start-dev"]
description = "🌟 Start a full RisingWave dev cluster using risedev-dev"
script = '''
#!/usr/bin/env bash

set -e
if [[ $ENABLE_COREDUMP == "true" ]]; then
  echo "+ ulimit -c unlimited"
  ulimit -c unlimited
fi

set -x
target/${BUILD_MODE_DIR}/risedev-dev ${@}
'''

[tasks.kill-risedev]
category = "RiseDev - Start/Stop"
description = "Kill RisingWave dev cluster"
script = '''
#!/usr/bin/env bash

set -euo pipefail

wait_kafka_exit() {
  # Follow kafka-server-stop.sh
  while [[ -n "$(ps ax | grep ' kafka\.Kafka ' | grep java | grep -v grep | awk '{print $1}')" ]]; do
    echo "Waiting for kafka to exit"
    sleep 1
  done
}

wait_zookeeper_exit() {
  # Follow zookeeper-server-stop.sh
  while [[ -n "$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')" ]]; do
    echo "Waiting for zookeeper to exit"
    sleep 1
  done
}

kill_kafka() {
  ${PREFIX_BIN}/kafka/bin/kafka-server-stop.sh
  wait_kafka_exit
}

kill_zookeeper() {
  ${PREFIX_BIN}/kafka/bin/zookeeper-server-stop.sh
  wait_zookeeper_exit
}

# Kill other components
tmux list-windows -t risedev -F "#{window_name} #{pane_id}" \
| grep -v 'kafka' \
| grep -v 'zookeeper' \
| awk '{ print $2 }' \
| xargs -I {} tmux send-keys -t {} C-c C-d

if [[ -n $(tmux list-windows -t risedev | grep kafka) ]];
then
  echo "kill kafka"
  kill_kafka || true

  echo "kill zookeeper"
  kill_zookeeper || true

  # Kill their tmux sessions
  tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d
fi

tmux kill-session -t risedev
test $? -eq 0 || { echo "Failed to stop all RiseDev components."; exit 1; }
'''

[tasks.kill]
alias = "kill-risedev"

[tasks.k]
alias = "kill-risedev"

[tasks.down]
category = "RiseDev - Start/Stop"
description = "Kill RisingWave dev cluster and clean the data"
dependencies = ["k", "clean-data"]

[tasks.install-nextest]
private = true
category = "RiseDev - Check"
install_crate = { min_version = "0.9.51", crate_name = "cargo-nextest", binary = "cargo", test_arg = [
  "nextest",
  "--help",
], install_command = "binstall" }

[tasks.install-llvm-cov]
private = true
category = "RiseDev - Check"
install_crate = { min_version = "0.5.17", crate_name = "cargo-llvm-cov", binary = "cargo", test_arg = [
  "llvm-cov",
  "--help",
], install_command = "binstall" }

[tasks.install-tools]
category = "RiseDev - Check"
script = """
#!/usr/bin/env bash

set -e

for tool in cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-make cargo-udeps
do
  echo "install: $(tput setaf 4)$tool$(tput sgr0)"
  cargo binstall -y --no-symlinks $tool
  echo
done

# Tools that fallback to `cargo install` when using `cargo binstall`.
# We directly use `cargo install` here to be faster.
for tool in typos-cli sqllogictest-bin
do
  echo "install: $(tput setaf 4)$tool$(tput sgr0)"
  cargo install $tool --locked
  echo
done

echo "check: $(tput setaf 4)tmux >= v3.2a$(tput sgr0)"
tmux -V || echo "$(tput setaf 3)tmux$(tput sgr0) not found."
echo

echo "check: $(tput setaf 4)psql >= 14$(tput sgr0)"
psql -V || echo "$(tput setaf 3)psql$(tput sgr0) not found."
echo

echo "check: $(tput setaf 4)cmake$(tput sgr0)"
cmake --version || echo "$(tput setaf 3)cmake$(tput sgr0) not found."
echo

echo "check: $(tput setaf 4)protoc >= 3.12.0$(tput sgr0)"
protoc --version || echo "$(tput setaf 3)protoc$(tput sgr0) not found."
echo

echo "check: $(tput setaf 4)parallel >= 2022XXXX$(tput sgr0)"
parallel --version || echo "$(tput setaf 3)parallel$(tput sgr0) not found."
echo
"""
description = "Install (or upgrade) required tools to do pre-CI check and run e2e tests"

[tasks.warn-on-missing-tools]
private = true
script = """
echo "If any command is not found, run $(tput setaf 4)./risedev install-tools$(tput sgr0) to install required tools."
"""

[tasks.test-cov]
category = "RiseDev - Test"
dependencies = ["install-llvm-cov", "install-nextest"]
script = """
#!/usr/bin/env bash
set -e

export CARGO_TARGET_DIR=target/coverage
cargo llvm-cov nextest --html --workspace --exclude risingwave_simulation
"""
description = "Run unit tests and report coverage"

[tasks.test]
category = "RiseDev - Test"
dependencies = ["install-nextest"]
script = """
#!/usr/bin/env bash
set -e

cargo nextest run --workspace --exclude risingwave_simulation "$@"
"""
description = "🌟 Run unit tests"

[tasks.build-connector-node]
category = "RiseDev - Build"
dependencies = ["prepare"]
condition = { env_set = [
  "ENABLE_BUILD_RW_CONNECTOR",
], files_modified = { input = [
  "./java/connector-node/**/*",
], output = [
  "./java/connector-node/assembly/target/**/*",
] } }
description = "Build RisingWave Connector from source"
script = '''
#!/usr/bin/env bash
set -e

if command -v mvn &> /dev/null; then
  MAVEN_PATH="$(command -v mvn)"
else
  MAVEN_PATH="${PREFIX_BIN}/maven/bin/mvn"
fi

ARTIFACT="risingwave-connector-1.0.0.tar.gz"
TARGET_PATH="${JAVA_DIR}/connector-node/assembly/target/${ARTIFACT}"

echo "Building connector node..."
cd "${JAVA_DIR}"
"${MAVEN_PATH}" --batch-mode --update-snapshots package -Dmaven.test.skip -Dno-build-rust

rm -rf ${PREFIX_BIN}/connector-node
mkdir -p "${PREFIX_BIN}/connector-node"
tar xf ${TARGET_PATH} -C "${PREFIX_BIN}/connector-node"
'''

[tasks.sbuild]
category = "RiseDev - Build"
description = "Build in simulation mode"
dependencies = ["warn-on-missing-tools"]
env = { CARGO_TARGET_DIR = "target/sim" }
script = """
#!/usr/bin/env bash
set -e

cargo build \
  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
  -p risingwave_batch \
  -p risingwave_common \
  -p risingwave_compute \
  -p risingwave_connector \
  -p risingwave_dml \
  -p risingwave_frontend \
  -p risingwave_meta \
  -p risingwave_object_store \
  -p risingwave_storage \
  -p risingwave_stream \
  -p pgwire \
  "$@"
"""

[tasks.stest]
category = "RiseDev - Deterministic Simulation"
description = "Run unit tests in deterministic simulation mode"
dependencies = ["install-nextest"]
env = { CARGO_TARGET_DIR = "target/sim" }
script = """
#!/usr/bin/env bash
set -e

cargo nextest run \
  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
  -p risingwave_batch \
  -p risingwave_common \
  -p risingwave_compute \
  -p risingwave_connector \
  -p risingwave_dml \
  -p risingwave_frontend \
  -p risingwave_meta \
  -p risingwave_object_store \
  -p risingwave_storage \
  -p risingwave_stream \
  -p pgwire \
  "$@"
"""

[tasks.sit-test]
category = "RiseDev - Deterministic Simulation"
description = "Run integration tests in deterministic simulation mode"
dependencies = ["install-nextest"]
env = { CARGO_TARGET_DIR = "target/sim" }
script = """
#!/usr/bin/env bash
set -e

cargo nextest run \
  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
  -p risingwave_simulation \
  "$@"
"""

[tasks.sarchive-it-test]
category = "RiseDev - Deterministic Simulation"
description = "Archive integration tests in deterministic simulation mode"
dependencies = ["install-nextest"]
env = { CARGO_TARGET_DIR = "target/sim" }
script = """
#!/usr/bin/env bash
set -e

cargo nextest archive \
  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
  -p risingwave_simulation \
  --archive-file simulation-it-test.tar.zst \
  "$@"
"""

[tasks.scheck]
category = "RiseDev - Deterministic Simulation"
description = "Run cargo check in deterministic simulation mode"
dependencies = ["warn-on-missing-tools"]
env = { CARGO_TARGET_DIR = "target/sim" }
script = """
#!/usr/bin/env bash
set -e

cargo check \
  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
  -p risingwave_batch \
  -p risingwave_common \
  -p risingwave_compute \
  -p risingwave_connector \
  -p risingwave_dml \
  -p risingwave_frontend \
  -p risingwave_meta \
  -p risingwave_object_store \
  -p risingwave_storage \
  -p risingwave_stream \
  -p pgwire \
  -p risingwave_simulation --tests "$@"
"""

[tasks.sslt]
category = "RiseDev - Deterministic Simulation"
description = "Run e2e tests in deterministic simulation mode"
dependencies = ["warn-on-missing-tools"]
env = { CARGO_TARGET_DIR = "target/sim" }
script = """
#!/usr/bin/env bash
set -e

cargo run \
  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
  -p risingwave_simulation "$@"
"""

[tasks.sslt-build-all]
category = "RiseDev - Deterministic Simulation"
description = "Build deterministic simulation runner and tests"
dependencies = ["warn-on-missing-tools"]
env = { CARGO_TARGET_DIR = "target/sim" }
script = """
#!/usr/bin/env bash
set -e

cargo build \
  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
  -p risingwave_simulation \
  --tests "$@"
"""

[tasks.sslt-cov]
category = "RiseDev - Deterministic Simulation"
description = "Run e2e tests in deterministic simulation mode and report code coverage"
dependencies = ["install-llvm-cov"]
env = { CARGO_TARGET_DIR = "target/sim-cov" }
script = """
#!/usr/bin/env bash
set -e

cargo llvm-cov run \
  --config "target.'cfg(all())'.rustflags = ['--cfg=madsim']" \
  -p risingwave_simulation \
  --html "$@"
"""

[tasks.check-java]
private = true
category = "RiseDev - Check"
description = "Run mvn spotless:check in connector-node"
dependencies = ["warn-on-missing-tools"]
condition = { env_set = ["ENABLE_RW_CONNECTOR", "ENABLE_BUILD_RW_CONNECTOR"] }
script = """
#!/usr/bin/env bash
set -e

if command -v mvn &> /dev/null; then
  MAVEN_PATH="$(command -v mvn)"
else
  MAVEN_PATH="${PREFIX_BIN}/maven/bin/mvn"
fi

cd "${JAVA_DIR}"
"${MAVEN_PATH}" spotless:check -q
"""

[tasks.check-java-fix]
private = true
category = "RiseDev - Check"
description = "Run mvn spotless:apply in connector-node"
dependencies = ["warn-on-missing-tools"]
condition = { env_set = ["ENABLE_RW_CONNECTOR", "ENABLE_BUILD_RW_CONNECTOR"] }
script = """
#!/usr/bin/env bash
set -e

if command -v mvn &> /dev/null; then
  MAVEN_PATH="$(command -v mvn)"
else
  MAVEN_PATH="${PREFIX_BIN}/maven/bin/mvn"
fi

cd "${JAVA_DIR}"
"${MAVEN_PATH}" spotless:apply
"""

[tasks.check-hakari]
private = true
category = "RiseDev - Check"
description = "Run cargo hakari check and attempt to fix"
install_crate = { min_version = "0.9.24", crate_name = "cargo-hakari", binary = "cargo", test_arg = [
  "hakari",
  "--help",
], install_command = "binstall" }
script = """
#!/usr/bin/env bash

echo "Running $(tput setaf 4)cargo hakari$(tput sgr0) checks and attempting to fix"

cargo hakari generate --diff --quiet || cargo hakari generate
cargo hakari verify > /dev/null
test $? -eq 0 || exit 1
"""

[tasks.check-dep-sort]
private = true
category = "RiseDev - Check"
description = "Run cargo sort check and attempt to fix"
install_crate = { min_version = "1.0.9", crate_name = "cargo-sort", binary = "cargo", test_arg = [
  "sort",
  "--help",
], install_command = "binstall" }
script = """
#!/usr/bin/env bash

echo "Running $(tput setaf 4)cargo sort$(tput sgr0) checks and attempting to fix"

# Rewriting Cargo.toml will cause a full rebuild of all crates, so we always check before fix.
(cargo sort -w -c > /dev/null) || cargo sort -w
test $? -eq 0 || { echo "cargo sort check failed. You may run $(tput setaf 4)cargo sort -w$(tput sgr0) to fix it."; exit 1; }
"""

[tasks.check-fmt]
private = true
category = "RiseDev - Check"
description = "Run cargo fmt check and attempt to fix"
script = """
#!/usr/bin/env bash

echo "Running $(tput setaf 4)cargo fmt$(tput sgr0) checks and attempting to fix"
cargo fmt --all
test $? -eq 0 || exit 1
"""

[tasks.check-clippy]
private = true
category = "RiseDev - Check"
description = "Run cargo clippy check"
script = """
#!/usr/bin/env bash

echo "Running $(tput setaf 4)cargo clippy$(tput sgr0) checks"
cargo clippy --all-targets ${RISINGWAVE_FEATURE_FLAGS}
echo "If cargo clippy check failed or generates warning, you may run $(tput setaf 4)cargo clippy --workspace --all-targets --fix$(tput sgr0) to fix it. Note that clippy fix requires manual review, as not all auto fixes are guaranteed to be reasonable."
echo "Alternately, you may run $(tput setaf 4)./risedev cf {package_names}$(tput sgr0) to fix those packages (e.g. frontend, meta)."
"""

[tasks.check-clippy-fix]
private = true
category = "RiseDev - Check"
description = "Run cargo clippy check and fixes all files (including dirty and staged)"
script = """
#!/usr/bin/env bash

echo "Running $(tput setaf 4)cargo clippy$(tput sgr0) checks and attempting to fix"
if [ $# -gt 0 ]; then
  ARGS=("$@")

  echo "Applying clippy --fix for $@ (including dirty and staged files)"
  cargo clippy ${ARGS[@]/#/--package risingwave_} --fix --allow-dirty --allow-staged
else
  echo "Applying clippy --fix for all targets to all files (including dirty and staged files)"
  echo "Tip: run $(tput setaf 4)./risedev cf {package_names}$(tput sgr0) to only check-fix those packages (e.g. frontend, meta)."
  cargo clippy --all-targets ${RISINGWAVE_FEATURE_FLAGS} --fix --allow-dirty --allow-staged
fi
"""

[tasks.check-typos]
private = true
category = "RiseDev - Check"
description = "Run cargo typos-cli check"
install_crate = { min_version = "1.15.0", crate_name = "typos-cli", binary = "typos", test_arg = [
  "--help",
], install_command = "binstall" }
script = """
#!/usr/bin/env bash

if ! typos ; then
  echo "Hint: use 'typos -w' to fix."
fi
"""

[tasks.check-udeps]
private = true
category = "RiseDev - Check"
description = "Check unused dependencies"
env = { RUSTFLAGS = "--cfg tokio_unstable" }
install_crate = { min_version = "0.1.35", crate_name = "cargo-udeps", binary = "cargo", test_arg = [
  "udeps",
  "--help",
], install_command = "binstall" }
script = """
#!/usr/bin/env bash

# TODO: after cargo-machete supports excluding packages, we may use it instead of cargo udeps.
# It's much faster so we can add it to CI and [tasks.check].

echo "Running $(tput setaf 4)cargo udeps$(tput sgr0) checks"

cargo udeps --workspace --all-targets ${RISINGWAVE_FEATURE_FLAGS} --exclude workspace-hack --exclude risingwave_bench --exclude risingwave_udf --exclude risingwave_simulation
"""

[tasks.check-trailing-spaces]
private = true
category = "RiseDev - Check"
description = "Check trailing spaces and attempt to fix"
script = """
#!/usr/bin/env bash

# This can trim trailing spaces in all git-managed text files, including .md, .toml, .sh, etc.
scripts/check/check-trailing-spaces.sh --fix
"""

[tasks.check-fast]
category = "RiseDev - Check"
dependencies = [
  "warn-on-missing-tools",
  "check-hakari",
  "check-dep-sort",
  "check-fmt",
  "check-trailing-spaces",
  "check-typos",
]
description = "Perform part of the pre-CI checks that are fast to run"

[tasks.check]
category = "RiseDev - Check"
dependencies = ["check-fast", "check-clippy", "check-java"]
script = """
#!/usr/bin/env bash

echo "Tip: use the alias $(tput setaf 4)./risedev c$(tput sgr0)."
echo "Good work! You may run $(tput setaf 4)./risedev test$(tput sgr0) or $(tput setaf 4)./risedev test-cov$(tput sgr0) to run unit tests."
"""
description = "🌟 Perform pre-CI checks and automatically fix cargo sort/hakari/fmt warnings"

[tasks.c]
alias = "check"

[tasks.check-fix]
category = "RiseDev - Check"
dependencies = [
  "warn-on-missing-tools",
  "check-fast",
  "check-clippy-fix",
  "check-java-fix",
]
script = """
#!/usr/bin/env bash

echo "Tip: use the alias $(tput setaf 4)./risedev cf$(tput sgr0)."
echo "Good work! You may run $(tput setaf 4)./risedev test$(tput sgr0) or $(tput setaf 4)./risedev test-cov$(tput sgr0) to run unit tests."
"""
description = "Same as check, but also fixes clippy"

[tasks.cf]
alias = "check-fix"


[tasks.install]
category = "Misc"
description = "🌟 Install RiseDev so you can run it from anywhere"
script = """
#!/usr/bin/env bash
set -e


DIR="$(pwd)"
NAME="${1:-risedev}"
INSTALL_PATH="$CARGO_HOME/bin/${NAME}"

read -p "Install to ${INSTALL_PATH}? [y/N] " -r

if [[ ! $REPLY =~ ^[Yy]$ ]]
then
    exit 1
fi

cat <<EOF > "${INSTALL_PATH}"
#!/usr/bin/env bash
set -e
cd "$DIR"
./risedev "\\$@"
EOF
chmod +x "${INSTALL_PATH}"

echo RiseDev installed to $(tput setaf 4)${INSTALL_PATH}$(tput sgr0)
echo If you still feel this is not enough, you may copy $(tput setaf 4)risedev$(tput sgr0) to something like $(tput setaf 4)r+$(tput sgr0).
"""

[tasks.ci-start]
category = "RiseDev - CI"
dependencies = ["clean-data", "pre-start-dev"]
command = "target/${BUILD_MODE_DIR}/risedev-dev"
args = ["${@}"]
description = "Clean data and start a full RisingWave dev cluster using risedev-dev"

[tasks.ci-kill-no-dump-logs]
category = "RiseDev - CI"
dependencies = ["k", "check-logs", "wait-processes-exit"]
description = "Kill cluster and check logs, do not dump logs"

[tasks.ci-kill]
category = "RiseDev - CI"
dependencies = ["k", "l", "check-logs", "wait-processes-exit"]
description = "Kill cluster, dump logs and check logs"

[tasks.wait-processes-exit]
category = "Misc"
description = "Wait for RisingWave processes to exit"
script = """
wait_for_process() {
    process_name="$1"

    while pgrep -x "$process_name" > /dev/null; do
        echo "Process $process_name is still running... Wait for 1 sec"
        sleep 1
    done
}

wait_for_process meta-node
wait_for_process compute-node
wait_for_process frontend
wait_for_process compactor
echo "All processes has exited."
"""

[tasks.slt]
category = "RiseDev - SQLLogicTest"
install_crate = { version = "0.19.1", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
  "--help",
], install_command = "binstall" }
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]
env = { SLT_HOST = "${RW_FRONTEND_LISTEN_ADDRESS}", SLT_PORT = "${RW_FRONTEND_PORT}", SLT_DB = "dev" }
command = "sqllogictest"
args = ["${@}"]
description = "🌟 Run SQLLogicTest"

[tasks.slt-streaming]
category = "RiseDev - SQLLogicTest"
extend = "slt"
args = ["${@}", "./e2e_test/streaming/**/*.slt"]
description = "Run all streaming e2e tests"

[tasks.slt-batch]
category = "RiseDev - SQLLogicTest"
extend = "slt"
args = ["${@}", "./e2e_test/batch/*.slt"]
description = "Run all batch e2e tests"

[tasks.slt-generated]
category = "RiseDev - SQLLogicTest"
extend = "slt"
args = ["${@}", "./e2e_test/generated/**/*.slt"]
description = "Run all generated e2e tests"

[tasks.slt-all]
category = "RiseDev - SQLLogicTest"
run_task = { name = ["slt-streaming", "slt-batch", "slt-generated"] }
description = "Run all e2e tests"

[tasks.docslt]
category = "RiseDev - SQLLogicTest"
description = "Extract SQL examples written in SQLLogicTest syntax from Rust doc comments"
script = '''
#!/usr/bin/env bash
set -e
cargo run -p risedev --bin risedev-docslt -- "$@"
'''

[tasks.compose]
category = "RiseDev - Compose"
description = "Compose a docker-compose.yaml file"
script = '''
#!/usr/bin/env bash
set -e
rm -rf ${PREFIX_DOCKER}/*
mkdir -p "${PREFIX_DOCKER}"
cargo run -p risedev --bin risedev-compose -- -d "${PREFIX_DOCKER}" "$@"
echo docker-compose file generated in $(tput setaf 4)${PREFIX_DOCKER}$(tput sgr0)
'''

[tasks.compose-deploy]
category = "RiseDev - Compose"
description = "Compose a docker-compose.yaml deploy directory"
script = '''
#!/usr/bin/env bash
set -e
rm -rf "${PREFIX_DOCKER}"/*
mkdir -p "${PREFIX_DOCKER}"
cargo run -p risedev --bin risedev-compose -- -d "${PREFIX_DOCKER}" --deploy "$@"
echo docker-compose file and deploy.sh generated in $(tput setaf 4)${PREFIX_DOCKER}$(tput sgr0)
echo If you are ready, run $(tput setaf 4)./risedev apply-compose-deploy$(tput sgr0) to do the deployment.
'''

[tasks.apply-compose-deploy]
category = "RiseDev - Compose"
description = "Run deploy.sh"
script = '''
#!/usr/bin/env bash
set -e

.risingwave/rw-docker/deploy.sh "$@"

echo
echo "Deploy finished!"

cat .risingwave/rw-docker/_message.partial.sh
'''

[tasks.apply-compose-docker]
category = "RiseDev - Compose"
description = "Run deploy.sh"
script = '''
#!/usr/bin/env bash
set -e

cp -a .risingwave/rw-docker/* docker
'''

[tasks.generate-example-config]
private = true
category = "Misc"
description = "Generate default config and write to src/config/example.toml"
script = '''
UPDATE_EXPECT=1 cargo test --package risingwave_common --lib -- config::tests::test_example_up_to_date
'''

[tasks.generate-with-options]
private = true
category = "Misc"
description = "This command extracts WITH options from the Rust code and maintains them in a single source of truth `with_options.yaml`"
script = '''
UPDATE_EXPECT=1 cargo test -p risingwave_connector tests::test_with_options_yaml_up_to_date
'''

[tasks.backwards-compat-test]
category = "RiseDev - Backwards Compatibility Test"
description = "Run backwards compatibility test"
script = "./backwards-compat-tests/scripts/run_local.sh"

# For debugging.
# To show the env for a specific task, use `run_task = "show-env"` for that task.
[tasks.show-env]
private = true
description = "Show cargo-make runtime environment variables"
script = """
#!@duckscript
# https://github.com/sagiegurari/cargo-make/issues/889
vars = dump_variables
echo ${vars}
"""