diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml index 29ebd5e907b4..a04a3701c8a2 100644 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@ -34,6 +34,37 @@ default_env_vars: &default_env_vars DTEST_BRANCH: trunk CCM_MAX_HEAP_SIZE: 1024M CCM_HEAP_NEWSIZE: 256M + # The Ant test target to run, for example: + # REPEATED_UTEST_TARGET: testsome + # REPEATED_UTEST_TARGET: test-jvm-dtest-some + # REPEATED_UTEST_TARGET: test-cdc + # REPEATED_UTEST_TARGET: test-compression + # REPEATED_UTEST_TARGET: test-system-keyspace-directory + REPEATED_UTEST_TARGET: testsome + # The name of JUnit class to be run multiple times, for example: + # REPEATED_UTEST_CLASS: org.apache.cassandra.cql3.ViewTest + # REPEATED_UTEST_CLASS: org.apache.cassandra.distributed.test.PagingTest + REPEATED_UTEST_CLASS: + # The optional specific methods within REPEATED_UTEST_CLASS to be run, for example: + # REPEATED_UTEST_METHODS: testCompoundPartitionKey + # REPEATED_UTEST_METHODS: testCompoundPartitionKey,testStaticTable + # Please note that some Ant targets will ignore the -Dtest.methods argument produced by this. + REPEATED_UTEST_METHODS: + # The number of times that the repeated JUnit test should be run + REPEATED_UTEST_COUNT: 100 + # Whether the test iteration should stop on the first failure + REPEATED_UTEST_STOP_ON_FAILURE: false + # A Python dtest to be run multiple times, for example: + # REPEATED_DTEST_NAME: cqlsh_tests/test_cqlsh.py + # REPEATED_DTEST_NAME: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke + # REPEATED_DTEST_NAME: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke::test_create_index + REPEATED_DTEST_NAME: + # Whether the repeated Python dtest should use vnodes + REPEATED_DTEST_VNODES: false + # The number of times that the repeated Python dtest should be run + REPEATED_DTEST_COUNT: 100 + # Whether the test iteration should stop on the first failure + REPEATED_DTEST_STOP_ON_FAILURE: false j8_par_executor: &j8_par_executor executor: @@ -112,6 +143,20 @@ with_dtests_jobs: &with_dtest_jobs - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests + # Java 8 repeated utest (on request) + - start_j8_repeated-utest: + type: approval + - j8_repeated-utest: + requires: + - start_j8_repeated-utest + - build + # Java 8 repeated dtest (on request) + - start_j8_repeated-dtest: + type: approval + - j8_repeated-dtest: + requires: + - start_j8_repeated-dtest + - build with_dtest_jobs_only: &with_dtest_jobs_only jobs: @@ -267,6 +312,23 @@ jobs: extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' pytest_extra_args: '--execute-upgrade-tests' + j8_repeated-utest: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - log_environment + - run_repeated_utest + + j8_repeated-dtest: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - run_repeated_dtest + commands: log_environment: steps: @@ -584,3 +646,175 @@ commands: - store_artifacts: path: ~/cassandra-dtest/logs destination: dtest_<>_logs + + run_repeated_utest: + steps: + - run: + name: Run repeated utest + no_output_timeout: 15m + command: | + if [ "$REPEATED_UTEST_CLASS" == "" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "$REPEATED_UTEST_COUNT" == "" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "$REPEATED_UTEST_COUNT" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((REPEATED_UTEST_COUNT / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (REPEATED_UTEST_COUNT % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running $REPEATED_UTEST_TARGET $REPEATED_UTEST_CLASS $REPEATED_UTEST_METHODS $count times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=$REPEATED_UTEST_TARGET + class_path=$REPEATED_UTEST_CLASS + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "$REPEATED_UTEST_METHODS" == "" ]; then + methods="" + else + methods="-Dtest.methods=$REPEATED_UTEST_METHODS" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if REPEATED_UTEST_STOP_ON_FAILURE is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ $REPEATED_UTEST_STOP_ON_FAILURE = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + + run_repeated_dtest: + steps: + - run: + name: Run repeated dtest + no_output_timeout: 15m + command: | + if [ "$REPEATED_DTEST_NAME" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "$REPEATED_DTEST_COUNT" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "$REPEATED_DTEST_COUNT" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((REPEATED_DTEST_COUNT / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (REPEATED_DTEST_COUNT % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running $REPEATED_DTEST_NAME $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + stop_on_failure_arg="" + if $REPEATED_UTEST_STOP_ON_FAILURE; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if $REPEATED_DTEST_VNODES; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $REPEATED_DTEST_NAME | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs diff --git a/.circleci/config.yml b/.circleci/config.yml index d5fc879dc657..95aafcb2aa03 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,6 +108,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: @@ -189,6 +198,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests-no-vnodes: @@ -247,6 +265,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_upgradetests-no-vnodes: @@ -346,6 +373,123 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated-dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Run repeated dtest + no_output_timeout: 15m + command: | + if [ "$REPEATED_DTEST_NAME" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "$REPEATED_DTEST_COUNT" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "$REPEATED_DTEST_COUNT" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((REPEATED_DTEST_COUNT / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (REPEATED_DTEST_COUNT % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running $REPEATED_DTEST_NAME $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + stop_on_failure_arg="" + if $REPEATED_UTEST_STOP_ON_FAILURE; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if $REPEATED_DTEST_VNODES; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $REPEATED_DTEST_NAME | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_unit_tests: @@ -438,6 +582,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests-with-vnodes: @@ -496,6 +649,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_jvm_dtests: @@ -588,6 +750,182 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated-utest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run repeated utest + no_output_timeout: 15m + command: | + if [ "$REPEATED_UTEST_CLASS" == "" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "$REPEATED_UTEST_COUNT" == "" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "$REPEATED_UTEST_COUNT" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((REPEATED_UTEST_COUNT / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (REPEATED_UTEST_COUNT % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running $REPEATED_UTEST_TARGET $REPEATED_UTEST_CLASS $REPEATED_UTEST_METHODS $count times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=$REPEATED_UTEST_TARGET + class_path=$REPEATED_UTEST_CLASS + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "$REPEATED_UTEST_METHODS" == "" ]; then + methods="" + else + methods="-Dtest.methods=$REPEATED_UTEST_METHODS" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if REPEATED_UTEST_STOP_ON_FAILURE is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ $REPEATED_UTEST_STOP_ON_FAILURE = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_long: @@ -634,6 +972,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_compression: @@ -726,6 +1073,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtest_jars_build: @@ -804,6 +1160,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: @@ -857,3 +1222,15 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests + - start_j8_repeated-utest: + type: approval + - j8_repeated-utest: + requires: + - start_j8_repeated-utest + - build + - start_j8_repeated-dtest: + type: approval + - j8_repeated-dtest: + requires: + - start_j8_repeated-dtest + - build diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES index f348b2503cc9..7035295201b8 100644 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@ -108,6 +108,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: @@ -189,6 +198,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests-no-vnodes: @@ -247,6 +265,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_upgradetests-no-vnodes: @@ -346,6 +373,123 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated-dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Run repeated dtest + no_output_timeout: 15m + command: | + if [ "$REPEATED_DTEST_NAME" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "$REPEATED_DTEST_COUNT" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "$REPEATED_DTEST_COUNT" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((REPEATED_DTEST_COUNT / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (REPEATED_DTEST_COUNT % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running $REPEATED_DTEST_NAME $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + stop_on_failure_arg="" + if $REPEATED_UTEST_STOP_ON_FAILURE; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if $REPEATED_DTEST_VNODES; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $REPEATED_DTEST_NAME | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_unit_tests: @@ -438,6 +582,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests-with-vnodes: @@ -496,6 +649,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_jvm_dtests: @@ -588,6 +750,182 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated-utest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run repeated utest + no_output_timeout: 15m + command: | + if [ "$REPEATED_UTEST_CLASS" == "" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "$REPEATED_UTEST_COUNT" == "" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "$REPEATED_UTEST_COUNT" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((REPEATED_UTEST_COUNT / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (REPEATED_UTEST_COUNT % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running $REPEATED_UTEST_TARGET $REPEATED_UTEST_CLASS $REPEATED_UTEST_METHODS $count times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=$REPEATED_UTEST_TARGET + class_path=$REPEATED_UTEST_CLASS + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "$REPEATED_UTEST_METHODS" == "" ]; then + methods="" + else + methods="-Dtest.methods=$REPEATED_UTEST_METHODS" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if REPEATED_UTEST_STOP_ON_FAILURE is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ $REPEATED_UTEST_STOP_ON_FAILURE = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_long: @@ -634,6 +972,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_compression: @@ -726,6 +1073,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtest_jars_build: @@ -750,7 +1106,7 @@ jobs: git remote set-branches --add apache '$branch' git fetch --depth 1 apache $branch git checkout $branch - it clean -fd + git clean -fd # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. for x in $(seq 1 3); do ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar @@ -769,7 +1125,7 @@ jobs: # and build the dtest-jar for the branch under test ${ANT_HOME}/bin/ant realclean git checkout origin/$CIRCLE_BRANCH - it clean -fd + git clean -fd for x in $(seq 1 3); do ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar RETURN="$?" @@ -804,6 +1160,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: @@ -857,3 +1222,15 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests + - start_j8_repeated-utest: + type: approval + - j8_repeated-utest: + requires: + - start_j8_repeated-utest + - build + - start_j8_repeated-dtest: + type: approval + - j8_repeated-dtest: + requires: + - start_j8_repeated-dtest + - build diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES index d5fc879dc657..95aafcb2aa03 100644 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@ -108,6 +108,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: @@ -189,6 +198,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests-no-vnodes: @@ -247,6 +265,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_upgradetests-no-vnodes: @@ -346,6 +373,123 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated-dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Run repeated dtest + no_output_timeout: 15m + command: | + if [ "$REPEATED_DTEST_NAME" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "$REPEATED_DTEST_COUNT" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "$REPEATED_DTEST_COUNT" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((REPEATED_DTEST_COUNT / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (REPEATED_DTEST_COUNT % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running $REPEATED_DTEST_NAME $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + stop_on_failure_arg="" + if $REPEATED_UTEST_STOP_ON_FAILURE; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if $REPEATED_DTEST_VNODES; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $REPEATED_DTEST_NAME | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_unit_tests: @@ -438,6 +582,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests-with-vnodes: @@ -496,6 +649,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_jvm_dtests: @@ -588,6 +750,182 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated-utest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run repeated utest + no_output_timeout: 15m + command: | + if [ "$REPEATED_UTEST_CLASS" == "" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "$REPEATED_UTEST_COUNT" == "" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "$REPEATED_UTEST_COUNT" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((REPEATED_UTEST_COUNT / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (REPEATED_UTEST_COUNT % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running $REPEATED_UTEST_TARGET $REPEATED_UTEST_CLASS $REPEATED_UTEST_METHODS $count times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=$REPEATED_UTEST_TARGET + class_path=$REPEATED_UTEST_CLASS + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "$REPEATED_UTEST_METHODS" == "" ]; then + methods="" + else + methods="-Dtest.methods=$REPEATED_UTEST_METHODS" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if REPEATED_UTEST_STOP_ON_FAILURE is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ $REPEATED_UTEST_STOP_ON_FAILURE = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_long: @@ -634,6 +972,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_compression: @@ -726,6 +1073,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtest_jars_build: @@ -804,6 +1160,15 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: @@ -857,3 +1222,15 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests + - start_j8_repeated-utest: + type: approval + - j8_repeated-utest: + requires: + - start_j8_repeated-utest + - build + - start_j8_repeated-dtest: + type: approval + - j8_repeated-dtest: + requires: + - start_j8_repeated-dtest + - build diff --git a/.circleci/generate.sh b/.circleci/generate.sh index 0ba497b2d13f..59a1a9fabd11 100755 --- a/.circleci/generate.sh +++ b/.circleci/generate.sh @@ -19,8 +19,17 @@ BASEDIR=`dirname $0` -circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES +# setup lowres +circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES.tmp +cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp > $BASEDIR/config.yml.LOWRES +rm $BASEDIR/config.yml.LOWRES.tmp + +# setup highres patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.high_res.patch -circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES -rm $BASEDIR/config-2_1.yml.HIGHRES +circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES.tmp +cat $BASEDIR/license.yml $BASEDIR/config.yml.HIGHRES.tmp > $BASEDIR/config.yml.HIGHRES +rm $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config.yml.HIGHRES.tmp + +# copy lower into config.yml to make sure this gets updated +cp $BASEDIR/config.yml.LOWRES $BASEDIR/config.yml diff --git a/.circleci/license.yml b/.circleci/license.yml new file mode 100644 index 000000000000..01d70572088e --- /dev/null +++ b/.circleci/license.yml @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + diff --git a/.circleci/readme.md b/.circleci/readme.md index b3d5ab009d29..c128c9cc07ff 100644 --- a/.circleci/readme.md +++ b/.circleci/readme.md @@ -39,6 +39,8 @@ HIGHRES files, read below for details how to do it manually; 1. make your edits to config-2_1.yml - let it stay at lowres settings 1. generate a valid LOWRES file: `circleci config process config-2_1.yml > config.yml.LOWRES` +1. add the Apache license header to the newly created LOWRES file: + `cat license.yml config.yml.LOWRES > config.yml.LOWRES.new && mv config.yml.LOWRES.new config.yml.LOWRES` 1. then apply the highres patch to config-2_1.yml; `patch -o config-2_1.yml.HIGHRES config-2_1.yml config-2_1.yml.high_res.patch` (this creates a new file `config-2_1.yml.HIGHRES` instead of in-place patching @@ -48,5 +50,7 @@ HIGHRES files, read below for details how to do it manually; the patch file based on the diff (don't commit it though). 1. generate the HIGHRES file: `circleci config process config-2_1.yml.HIGHRES > config.yml.HIGHRES` -1. and remove the temporary patched highres `config-2_1.yml.HIGHRES` +1. remove the temporary patched HIGHRES file: `rm config-2_1.yml.HIGHRES` +1. add the Apache license header to the newly created HIGHRES file: + `cat license.yml config.yml.HIGHRES > config.yml.HIGHRES.new && mv config.yml.HIGHRES.new config.yml.HIGHRES` diff --git a/build.xml b/build.xml index 6d533eb71fb9..76e84ac0434b 100644 --- a/build.xml +++ b/build.xml @@ -1144,7 +1144,7 @@ - +