diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b05495d1867..ca8842759b11 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,85 +14,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -version: 2 jobs: - j8_jvm_upgrade_dtests: + j11_build: docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine distributed Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -125,149 +51,87 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - repeated_jvm_upgrade_dtest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + parallelism: 1 resource_class: medium - working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - - attach_workspace: - at: /home/cassandra - run: - name: Log Environment Information - command: | - echo '*** id ***' + command: 'echo ''*** id ***'' + id - echo '*** cat /proc/cpuinfo ***' + + echo ''*** cat /proc/cpuinfo ***'' + cat /proc/cpuinfo - echo '*** free -m ***' + + echo ''*** free -m ***'' + free -m - echo '*** df -m ***' + + echo ''*** df -m ***'' + df -m - echo '*** ifconfig -a ***' + + echo ''*** ifconfig -a ***'' + ifconfig -a - echo '*** uname -a ***' + + echo ''*** uname -a ***'' + uname -a - echo '*** mount ***' + + echo ''*** mount ***'' + mount - echo '*** env ***' + + echo ''*** env ***'' + env - echo '*** java ***' + + echo ''*** java ***'' + which java + java -version + + ' + name: Log Environment Information - run: - name: Run repeated JUnit test + command: 'git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git + ~/cassandra + + ' + name: Clone Cassandra Repository (via git) + - run: + command: "export PATH=$JAVA_HOME/bin:$PATH\ncd ~/cassandra\n# Loop to prevent + failure due to maven-ant-tasks not downloading a jar..\nfor x in $(seq 1 + 3); do\n ${ANT_HOME}/bin/ant clean realclean jar\n RETURN=\"$?\"\n + \ if [ \"${RETURN}\" -eq \"0\" ]; then\n break\n fi\ndone\n# + Exit, if we didn't build successfully\nif [ \"${RETURN}\" -ne \"0\" ]; then\n + \ echo \"Build failed with exit code: ${RETURN}\"\n exit ${RETURN}\nfi\n" + name: Build Cassandra no_output_timeout: 15m - command: | - if [ "${REPEATED_JVM_UPGRADE_DTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_JVM_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running test-jvm-dtest-some ${REPEATED_JVM_UPGRADE_DTEST_CLASS} ${REPEATED_JVM_UPGRADE_DTEST_METHODS} ${REPEATED_JVM_UPGRADE_DTEST_COUNT} times" - - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - target=test-jvm-dtest-some - class_path=${REPEATED_JVM_UPGRADE_DTEST_CLASS} - class_name="${class_path##*.}" - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" - else - name="-Dtest.name=$class_path" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_JVM_UPGRADE_DTEST_METHODS}" == "" ]; then - methods="" - else - methods="-Dtest.methods=${REPEATED_JVM_UPGRADE_DTEST_METHODS}" - fi - - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" - for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/test-jvm-dtest-some-${REPEATED_JVM_UPGRADE_DTEST_CLASS}.txt - - # move the XML output files - source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - - (exit ${exit_code}) - fi - fi - - store_test_results: - path: /tmp/results/repeated_utest/output - - store_artifacts: - path: /tmp/results/repeated_utest/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utest/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utest/logs - destination: logs + - run: + command: 'export PATH=$JAVA_HOME/bin:$PATH + + cd ~/cassandra + + ant eclipse-warnings + + ' + name: Run eclipse-warnings + - persist_to_workspace: + paths: + - cassandra + - .m2 + root: /home/cassandra + working_directory: ~/ + j11_cqlsh-dtests-py2-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -300,69 +164,91 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_cqlsh-dtests-py2-with-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j8_with_vnodes) + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests + --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw + --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw + /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw + > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match + any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci + tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes + > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt + | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + name: Determine Tests to Run (j11_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: - name: Run dtests (j8_with_vnodes) + command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; + then\n export CQLSH_PYTHON=/usr/bin/python2.7\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests + --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j11_without_vnodes) no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then - export CQLSH_PYTHON=/usr/bin/python2.7 - fi - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest_j11_without_vnodes path: /tmp/dtest - destination: dtest_j8_with_vnodes - store_artifacts: + destination: dtest_j11_without_vnodes_logs path: ~/cassandra-dtest/logs - destination: dtest_j8_with_vnodes_logs + working_directory: ~/ + j11_cqlsh-dtests-py2-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -395,83 +281,91 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j11_unit_tests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Determine unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" + ' + name: Clone Cassandra dtest Repository (via git) + - run: + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.6/bin/activate + + export PATH=$JAVA_HOME/bin:$PATH + + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + + pip3 uninstall -y cqlsh + + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes + --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only + --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif + [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n + \ grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes + || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset + -eo pipefail && circleci tests split --split-by=timings --timings-type=classname + /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat + /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m - run: - name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; + then\n export CQLSH_PYTHON=/usr/bin/python2.7\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 + --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j11_with_vnodes) no_output_timeout: 15m - store_test_results: - path: /tmp/cassandra/build/test/output/ + path: /tmp/results - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml + destination: dtest_j11_with_vnodes + path: /tmp/dtest - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs + destination: dtest_j11_with_vnodes_logs + path: ~/cassandra-dtest/logs + working_directory: ~/ + j11_cqlsh-dtests-py3-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -507,94 +401,88 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - repeated_upgrade_dtest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Run repeated Python dtest + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests + --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw + --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw + /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw + > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match + any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci + tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes + > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt + | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + - run: + command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; + then\n export CQLSH_PYTHON=/usr/bin/python3.6\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests + --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j11_without_vnodes) no_output_timeout: 15m - command: | - if [ "${REPEATED_UPGRADE_DTEST_NAME}" == "" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" == "" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UPGRADE_DTEST_NAME} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - stop_on_failure_arg="" - if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if false; then - vnodes_args="--use-vnodes --num-tokens=16" - fi - - upgrade_arg="" - if true; then - upgrade_arg="--execute-upgrade-tests" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} | tee /tmp/dtest/stdout.txt - fi - fi - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest_j11_without_vnodes path: /tmp/dtest - destination: dtest - store_artifacts: + destination: dtest_j11_without_vnodes_logs path: ~/cassandra-dtest/logs - destination: dtest_logs + working_directory: ~/ + j11_cqlsh-dtests-py3-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -627,69 +515,91 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_cqlsh-dtests-py38-no-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + parallelism: 20 + resource_class: xlarge + shell: /bin/bash -eo pipefail -l steps: - attach_workspace: at: /home/cassandra - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.8/bin/activate + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j8_without_vnodes) + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes + --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only + --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif + [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n + \ grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes + || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset + -eo pipefail && circleci tests split --split-by=timings --timings-type=classname + /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat + /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + name: Determine Tests to Run (j11_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: - name: Run dtests (j8_without_vnodes) + command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; + then\n export CQLSH_PYTHON=/usr/bin/python3.6\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 + --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j11_with_vnodes) no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt - - source ~/env3.8/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then - export CQLSH_PYTHON=/usr/bin/python3.8 - fi - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest_j11_with_vnodes path: /tmp/dtest - destination: dtest_j8_without_vnodes - store_artifacts: + destination: dtest_j11_with_vnodes_logs path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + working_directory: ~/ + j11_cqlsh-dtests-py38-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -722,69 +632,91 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j11_cqlsh-dtests-py3-with-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.8/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j11_with_vnodes) + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests + --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw + --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw + /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw + > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match + any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci + tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes + > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt + | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + name: Determine Tests to Run (j11_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" - run: - name: Run dtests (j11_with_vnodes) + command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; + then\n export CQLSH_PYTHON=/usr/bin/python3.8\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests + --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j11_without_vnodes) no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" - cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then - export CQLSH_PYTHON=/usr/bin/python3.6 - fi - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest_j11_without_vnodes path: /tmp/dtest - destination: dtest_j11_with_vnodes - store_artifacts: + destination: dtest_j11_without_vnodes_logs path: ~/cassandra-dtest/logs - destination: dtest_j11_with_vnodes_logs + working_directory: ~/ + j11_cqlsh-dtests-py38-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -820,67 +752,88 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j11_cqlsh-dtests-py3-no-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.8/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j11_without_vnodes) + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes + --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only + --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif + [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n + \ grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes + || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset + -eo pipefail && circleci tests split --split-by=timings --timings-type=classname + /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat + /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + name: Determine Tests to Run (j11_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" - run: - name: Run dtests (j11_without_vnodes) + command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; + then\n export CQLSH_PYTHON=/usr/bin/python3.8\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 + --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j11_with_vnodes) no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" - cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then - export CQLSH_PYTHON=/usr/bin/python3.6 - fi - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest_j11_with_vnodes path: /tmp/dtest - destination: dtest_j11_without_vnodes - store_artifacts: + destination: dtest_j11_with_vnodes_logs path: ~/cassandra-dtest/logs - destination: dtest_j11_without_vnodes_logs + working_directory: ~/ + j11_cqlshlib_tests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -916,67 +869,30 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j11_cqlsh-dtests-py38-with-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + parallelism: 1 resource_class: medium - working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.8/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j11_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" - - run: - name: Run dtests (j11_with_vnodes) - no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" - cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + command: 'export PATH=$JAVA_HOME/bin:$PATH - source ~/env3.8/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then - export CQLSH_PYTHON=/usr/bin/python3.8 - fi + time mv ~/cassandra /tmp - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + cd /tmp/cassandra/pylib + + ./cassandra-cqlsh-tests.sh .. + + ' + name: Run cqlshlib Unit Tests + no_output_timeout: 15m - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j11_with_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j11_with_vnodes_logs + path: /tmp/cassandra/pylib + working_directory: ~/ + j11_dtests-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -1012,67 +928,128 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_cqlsh-dtests-py3-with-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 25 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: + command: 'echo ''*** id ***'' + + id + + echo ''*** cat /proc/cpuinfo ***'' + + cat /proc/cpuinfo + + echo ''*** free -m ***'' + + free -m + + echo ''*** df -m ***'' + + df -m + + echo ''*** ifconfig -a ***'' + + ifconfig -a + + echo ''*** uname -a ***'' + + uname -a + + echo ''*** mount ***'' + + mount + + echo ''*** env ***'' + + env + + echo ''*** java ***'' + + which java + + java -version + + ' + name: Log Environment Information + - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j8_with_vnodes) + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests + --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw + --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw + /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw + > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match + any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci + tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes + > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt + | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + name: Determine Tests to Run (j11_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: - name: Run dtests (j8_with_vnodes) + command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests + --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j11_without_vnodes) no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then - export CQLSH_PYTHON=/usr/bin/python3.6 - fi - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest_j11_without_vnodes path: /tmp/dtest - destination: dtest_j8_with_vnodes - store_artifacts: + destination: dtest_j11_without_vnodes_logs path: ~/cassandra-dtest/logs - destination: dtest_j8_with_vnodes_logs + working_directory: ~/ + j11_dtests-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -1105,213 +1082,131 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_cqlsh-dtests-py2-no-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + parallelism: 25 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - - run: - name: Run dtests (j8_without_vnodes) - no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + command: 'echo ''*** id ***'' - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then - export CQLSH_PYTHON=/usr/bin/python2.7 - fi + id + + echo ''*** cat /proc/cpuinfo ***'' - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_without_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs - environment: - - ANT_HOME: /usr/share/ant - - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j11_repeated_dtest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' cat /proc/cpuinfo - echo '*** free -m ***' + + echo ''*** free -m ***'' + free -m - echo '*** df -m ***' + + echo ''*** df -m ***'' + df -m - echo '*** ifconfig -a ***' + + echo ''*** ifconfig -a ***'' + ifconfig -a - echo '*** uname -a ***' + + echo ''*** uname -a ***'' + uname -a - echo '*** mount ***' + + echo ''*** mount ***'' + mount - echo '*** env ***' + + echo ''*** env ***'' + env - echo '*** java ***' + + echo ''*** java ***'' + which java + java -version + + ' + name: Log Environment Information - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Run repeated Python dtest + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes + --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only + --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif + [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n + \ grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes + || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset + -eo pipefail && circleci tests split --split-by=timings --timings-type=classname + /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat + /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + - run: + command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 + --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j11_with_vnodes) no_output_timeout: 15m - command: | - if [ "${REPEATED_DTEST_NAME}" == "" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" == "" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_DTEST_NAME} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - stop_on_failure_arg="" - if ${REPEATED_DTEST_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if ${REPEATED_DTEST_VNODES}; then - vnodes_args="--use-vnodes --num-tokens=16" - fi - - upgrade_arg="" - if false; then - upgrade_arg="--execute-upgrade-tests" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt - fi - fi - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest_j11_with_vnodes path: /tmp/dtest - destination: dtest - store_artifacts: + destination: dtest_j11_with_vnodes_logs path: ~/cassandra-dtest/logs - destination: dtest_logs + working_directory: ~/ + j11_jvm_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -1347,94 +1242,112 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_repeated_dtest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 26 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + command: '# reminder: this code (along with all the steps) is independently + executed on every circle container + + # so the goal here is to get the circleci script to return the tests *this* + container will run + + # which we do via the `circleci` cli tool. + + + rm -fr ~/cassandra-dtest/upgrade_tests + + echo "***java tests***" + + + # get all of our unit test filenames + + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" + > /tmp/all_java_unit_tests.txt + + + # split up the unit tests into groups based on the number of containers + we have + + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename + --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt + > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" + | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + ' + name: Determine distributed Tests to Run + no_output_timeout: 15m - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze + command: 'echo ''*** id ***'' + + id + + echo ''*** cat /proc/cpuinfo ***'' + + cat /proc/cpuinfo + + echo ''*** free -m ***'' + + free -m + + echo ''*** df -m ***'' + + df -m + + echo ''*** ifconfig -a ***'' + + ifconfig -a + + echo ''*** uname -a ***'' + + uname -a + + echo ''*** mount ***'' + + mount + + echo ''*** env ***'' + + env + + echo ''*** java ***'' + + which java + + java -version + + ' + name: Log Environment Information - run: - name: Run repeated Python dtest + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd + /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\ntest_timeout=$(grep + 'name=\"test.distributed.timeout\"' build.xml | awk -F'\"' '{print $4}' + || true)\nif [ -z \"$test_timeout\" ]; then\n test_timeout=$(grep 'name=\"test.timeout\"' + build.xml | awk -F'\"' '{print $4}')\nfi\nant testclasslist -Dtest.timeout=\"$test_timeout\" + -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed\n" + name: Run Unit Tests (testclasslist) no_output_timeout: 15m - command: | - if [ "${REPEATED_DTEST_NAME}" == "" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" == "" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_DTEST_NAME} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - stop_on_failure_arg="" - if ${REPEATED_DTEST_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if ${REPEATED_DTEST_VNODES}; then - vnodes_args="--use-vnodes --num-tokens=16" - fi - - upgrade_arg="" - if false; then - upgrade_arg="--execute-upgrade-tests" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt - fi - fi - store_test_results: - path: /tmp/results + path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/dtest - destination: dtest + destination: junitxml + path: /tmp/cassandra/build/test/output - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs + destination: logs + path: /tmp/cassandra/build/test/logs + working_directory: ~/ + j11_repeated_dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -1467,69 +1380,128 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j11_cqlsh-dtests-py2-with-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: + command: 'echo ''*** id ***'' + + id + + echo ''*** cat /proc/cpuinfo ***'' + + cat /proc/cpuinfo + + echo ''*** free -m ***'' + + free -m + + echo ''*** df -m ***'' + + df -m + + echo ''*** ifconfig -a ***'' + + ifconfig -a + + echo ''*** uname -a ***'' + + uname -a + + echo ''*** mount ***'' + + mount + + echo ''*** env ***'' + + env + + echo ''*** java ***'' + + which java + + java -version + + ' + name: Log Environment Information + - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j11_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" - - run: - name: Run dtests (j11_with_vnodes) + command: "if [ \"${REPEATED_DTEST_NAME}\" == \"\" ]; then\n echo \"Repeated + dtest name hasn't been defined, exiting without running any test\"\nelif + [ \"${REPEATED_DTEST_COUNT}\" == \"\" ]; then\n echo \"Repeated dtest + count hasn't been defined, exiting without running any test\"\nelif [ \"${REPEATED_DTEST_COUNT}\" + -le 0 ]; then\n echo \"Repeated dtest count is lesser or equals than zero, + exiting without running any test\"\nelse\n\n # Calculate the number of + test iterations to be run by the current parallel runner.\n # Since we + are running the same test multiple times there is no need to use `circleci + tests split`.\n count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL))\n + \ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); + then\n count=$((count+1))\n fi\n\n if (($count <= 0)); then\n echo + \"No tests to run in this runner\"\n else\n echo \"Running ${REPEATED_DTEST_NAME} + $count times\"\n\n source ~/env3.6/bin/activate\n export PATH=$JAVA_HOME/bin:$PATH\n\n + \ java -version\n cd ~/cassandra-dtest\n mkdir -p /tmp/dtest\n\n + \ echo \"env: $(env)\"\n echo \"** done env\"\n mkdir -p /tmp/results/dtests\n\n + \ stop_on_failure_arg=\"\"\n if ${REPEATED_DTEST_STOP_ON_FAILURE}; + then\n stop_on_failure_arg=\"-x\"\n fi\n\n vnodes_args=\"\"\n + \ if ${REPEATED_DTEST_VNODES}; then\n vnodes_args=\"--use-vnodes + --num-tokens=16\"\n fi\n\n upgrade_arg=\"\"\n if false; then\n + \ upgrade_arg=\"--execute-upgrade-tests\"\n fi\n\n # we need the + \"set -o pipefail\" here so that the exit code that circleci will actually + use is from pytest and not the exit code from tee\n set -o pipefail && + cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg + $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} + | tee /tmp/dtest/stdout.txt\n fi\nfi\n" + name: Run repeated Python dtest no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" - cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then - export CQLSH_PYTHON=/usr/bin/python2.7 - fi - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest path: /tmp/dtest - destination: dtest_j11_with_vnodes - store_artifacts: + destination: dtest_logs path: ~/cassandra-dtest/logs - destination: dtest_j11_with_vnodes_logs + working_directory: ~/ + j11_repeated_utest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -1565,70 +1537,113 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j11_dtests-with-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Log Environment Information - command: | - echo '*** id ***' + command: 'echo ''*** id ***'' + id - echo '*** cat /proc/cpuinfo ***' + + echo ''*** cat /proc/cpuinfo ***'' + cat /proc/cpuinfo - echo '*** free -m ***' + + echo ''*** free -m ***'' + free -m - echo '*** df -m ***' + + echo ''*** df -m ***'' + df -m - echo '*** ifconfig -a ***' + + echo ''*** ifconfig -a ***'' + ifconfig -a - echo '*** uname -a ***' + + echo ''*** uname -a ***'' + uname -a - echo '*** mount ***' + + echo ''*** mount ***'' + mount - echo '*** env ***' + + echo ''*** env ***'' + env - echo '*** java ***' + + echo ''*** java ***'' + which java + java -version + + ' + name: Log Environment Information - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j11_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" - - run: - name: Run dtests (j11_with_vnodes) + command: "if [ \"${REPEATED_UTEST_CLASS}\" == \"\" ]; then\n echo \"Repeated + utest class name hasn't been defined, exiting without running any test\"\nelif + [ \"${REPEATED_UTEST_COUNT}\" == \"\" ]; then\n echo \"Repeated utest + count hasn't been defined, exiting without running any test\"\nelif [ \"${REPEATED_UTEST_COUNT}\" + -le 0 ]; then\n echo \"Repeated utest count is lesser or equals than zero, + exiting without running any test\"\nelse\n\n # Calculate the number of + test iterations to be run by the current parallel runner.\n # Since we + are running the same test multiple times there is no need to use `circleci + tests split`.\n count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL))\n + \ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); + then\n count=$((count+1))\n fi\n\n if (($count <= 0)); then\n echo + \"No tests to run in this runner\"\n else\n echo \"Running ${REPEATED_UTEST_TARGET} + ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} + times\"\n\n set -x\n export PATH=$JAVA_HOME/bin:$PATH\n time mv + ~/cassandra /tmp\n cd /tmp/cassandra\n if [ -d ~/dtest_jars ]; then\n + \ cp ~/dtest_jars/dtest* /tmp/cassandra/build/\n fi\n\n target=${REPEATED_UTEST_TARGET}\n + \ class_path=${REPEATED_UTEST_CLASS}\n class_name=\"${class_path##*.}\"\n\n + \ # Prepare the -Dtest.name argument.\n # It can be the fully qualified + class name or the short class name, depending on the target.\n if [[ + $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target + == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" + ]]; then\n name=\"-Dtest.name=$class_name\"\n else\n name=\"-Dtest.name=$class_path\"\n + \ fi\n\n # Prepare the -Dtest.methods argument, which is optional\n + \ if [ \"${REPEATED_UTEST_METHODS}\" == \"\" ]; then\n methods=\"\"\n + \ else\n methods=\"-Dtest.methods=${REPEATED_UTEST_METHODS}\"\n fi\n\n + \ # Run the test target as many times as requested collecting the exit + code,\n # stopping the iteration only if stop_on_failure is set.\n exit_code=\"$?\"\n + \ for i in $(seq -w 1 $count); do\n\n echo \"Running test iteration + $i of $count\"\n\n # run the test\n status=\"passes\"\n if + !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | + tee stdout.txt ); then\n status=\"fails\"\n exit_code=1\n + \ fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utest/stdout/${status}/${i}\n + \ mkdir -p $dest\n mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt\n\n + \ # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utest/output/${status}/${i}\n + \ mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; + then\n mv $source/* $dest/\n fi\n\n # move the log files\n + \ source=build/test/logs\n dest=/tmp/results/repeated_utest/logs/${status}/${i}\n + \ mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; + then\n mv $source/* $dest/\n fi\n\n # maybe stop iterations + on test failure\n if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] + && (( $exit_code > 0 )); then\n break\n fi\n done\n\n (exit + ${exit_code})\n fi\nfi\n" + name: Run repeated JUnit test no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: - path: /tmp/results + path: /tmp/results/repeated_utest/output - store_artifacts: - path: /tmp/dtest - destination: dtest_j11_with_vnodes + destination: stdout + path: /tmp/results/repeated_utest/stdout - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j11_with_vnodes_logs + destination: junitxml + path: /tmp/results/repeated_utest/output + - store_artifacts: + destination: logs + path: /tmp/results/repeated_utest/logs + working_directory: ~/ + j11_unit_tests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -1664,48 +1679,112 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_dtests-no-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 36 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze + command: '# reminder: this code (along with all the steps) is independently + executed on every circle container + + # so the goal here is to get the circleci script to return the tests *this* + container will run + + # which we do via the `circleci` cli tool. + + + rm -fr ~/cassandra-dtest/upgrade_tests + + echo "***java tests***" + + + # get all of our unit test filenames + + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" + > /tmp/all_java_unit_tests.txt + + + # split up the unit tests into groups based on the number of containers + we have + + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename + --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt + > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" + | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + ' + name: Determine unit Tests to Run + no_output_timeout: 15m - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: 'echo ''*** id ***'' + + id + + echo ''*** cat /proc/cpuinfo ***'' + + cat /proc/cpuinfo + + echo ''*** free -m ***'' + + free -m + + echo ''*** df -m ***'' + + df -m + + echo ''*** ifconfig -a ***'' + + ifconfig -a + + echo ''*** uname -a ***'' + + uname -a + + echo ''*** mount ***'' + + mount + + echo ''*** env ***'' + + env + + echo ''*** java ***'' + + which java + + java -version + + ' + name: Log Environment Information - run: - name: Run dtests (j8_without_vnodes) + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd + /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\ntest_timeout=$(grep + 'name=\"test.unit.timeout\"' build.xml | awk -F'\"' '{print $4}' || true)\nif + [ -z \"$test_timeout\" ]; then\n test_timeout=$(grep 'name=\"test.timeout\"' + build.xml | awk -F'\"' '{print $4}')\nfi\nant testclasslist -Dtest.timeout=\"$test_timeout\" + -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit\n" + name: Run Unit Tests (testclasslist) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: - path: /tmp/results + path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_without_vnodes + destination: junitxml + path: /tmp/cassandra/build/test/output - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + destination: logs + path: /tmp/cassandra/build/test/logs + working_directory: ~/ + j8_build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -1740,48 +1819,84 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgradetests-no-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + parallelism: 1 resource_class: medium - working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze + command: 'echo ''*** id ***'' + + id + + echo ''*** cat /proc/cpuinfo ***'' + + cat /proc/cpuinfo + + echo ''*** free -m ***'' + + free -m + + echo ''*** df -m ***'' + + df -m + + echo ''*** ifconfig -a ***'' + + ifconfig -a + + echo ''*** uname -a ***'' + + uname -a + + echo ''*** mount ***'' + + mount + + echo ''*** env ***'' + + env + + echo ''*** java ***'' + + which java + + java -version + + ' + name: Log Environment Information - run: - name: Determine Tests to Run (j8_upgradetests_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n" + command: 'git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git + ~/cassandra + + ' + name: Clone Cassandra Repository (via git) - run: - name: Run dtests (j8_upgradetests_without_vnodes) + command: "export PATH=$JAVA_HOME/bin:$PATH\ncd ~/cassandra\n# Loop to prevent + failure due to maven-ant-tasks not downloading a jar..\nfor x in $(seq 1 + 3); do\n ${ANT_HOME}/bin/ant clean realclean jar\n RETURN=\"$?\"\n + \ if [ \"${RETURN}\" -eq \"0\" ]; then\n break\n fi\ndone\n# + Exit, if we didn't build successfully\nif [ \"${RETURN}\" -ne \"0\" ]; then\n + \ echo \"Build failed with exit code: ${RETURN}\"\n exit ${RETURN}\nfi\n" + name: Build Cassandra no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_upgradetests_without_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_upgradetests_without_vnodes_logs + - run: + command: 'export PATH=$JAVA_HOME/bin:$PATH + + cd ~/cassandra + + ant eclipse-warnings + + ' + name: Run eclipse-warnings + - persist_to_workspace: + paths: + - cassandra + - .m2 + root: /home/cassandra + working_directory: ~/ + j8_cqlsh-dtests-py2-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -1816,35 +1931,88 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_stress: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 1 steps: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (stress-test) - command: | + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' + name: Clone Cassandra dtest Repository (via git) + - run: + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - ant stress-test + + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + + pip3 uninstall -y cqlsh + + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies + - run: + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests + --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw + --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw + /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw + > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match + any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci + tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes + > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt + | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + - run: + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; + then\n export CQLSH_PYTHON=/usr/bin/python2.7\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests + --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - store_test_results: - path: /tmp/cassandra/build/test/output/ + path: /tmp/results - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml + destination: dtest_j8_without_vnodes + path: /tmp/dtest - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs + destination: dtest_j8_without_vnodes_logs + path: ~/cassandra-dtest/logs + working_directory: ~/ + j8_cqlsh-dtests-py2-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -1879,81 +2047,88 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_unit_tests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Determine unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" + ' + name: Clone Cassandra dtest Repository (via git) + - run: + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.6/bin/activate + + export PATH=$JAVA_HOME/bin:$PATH + + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + + pip3 uninstall -y cqlsh + + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes + --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only + --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif + [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n + \ grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes + || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset + -eo pipefail && circleci tests split --split-by=timings --timings-type=classname + /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat + /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m - run: - name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; + then\n export CQLSH_PYTHON=/usr/bin/python2.7\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 + --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j8_with_vnodes) no_output_timeout: 15m - store_test_results: - path: /tmp/cassandra/build/test/output/ + path: /tmp/results - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml + destination: dtest_j8_with_vnodes + path: /tmp/dtest - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs + destination: dtest_j8_with_vnodes_logs + path: ~/cassandra-dtest/logs + working_directory: ~/ + j8_cqlsh-dtests-py3-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -1988,81 +2163,88 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j11_jvm_dtests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 1 steps: - attach_workspace: at: /home/cassandra - run: - name: Determine distributed Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" + ' + name: Clone Cassandra dtest Repository (via git) + - run: + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.6/bin/activate + + export PATH=$JAVA_HOME/bin:$PATH + + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + + pip3 uninstall -y cqlsh + + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests + --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw + --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw + /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw + > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match + any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci + tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes + > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt + | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m - run: - name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; + then\n export CQLSH_PYTHON=/usr/bin/python3.6\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests + --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - store_test_results: - path: /tmp/cassandra/build/test/output/ + path: /tmp/results - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml + destination: dtest_j8_without_vnodes + path: /tmp/dtest - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs + destination: dtest_j8_without_vnodes_logs + path: ~/cassandra-dtest/logs + working_directory: ~/ + j8_cqlsh-dtests-py3-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -2095,73 +2277,90 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - CASSANDRA_USE_JDK11: true - j11_build: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 1 steps: + - attach_workspace: + at: /home/cassandra - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Clone Cassandra Repository (via git) - command: | - git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' + name: Clone Cassandra dtest Repository (via git) - run: - name: Build Cassandra - command: | + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant clean realclean jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - no_output_timeout: 15m + + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + + pip3 uninstall -y cqlsh + + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Run eclipse-warnings - command: | - export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - ant eclipse-warnings - - persist_to_workspace: - root: /home/cassandra - paths: - - cassandra - - .m2 + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes + --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only + --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif + [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n + \ grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes + || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset + -eo pipefail && circleci tests split --split-by=timings --timings-type=classname + /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat + /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + - run: + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; + then\n export CQLSH_PYTHON=/usr/bin/python3.6\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 + --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + - store_test_results: + path: /tmp/results + - store_artifacts: + destination: dtest_j8_with_vnodes + path: /tmp/dtest + - store_artifacts: + destination: dtest_j8_with_vnodes_logs + path: ~/cassandra-dtest/logs + working_directory: ~/ + j8_cqlsh-dtests-py38-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -2194,70 +2393,90 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - CASSANDRA_USE_JDK11: true - j11_cqlsh-dtests-py2-no-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.8/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j11_without_vnodes) + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests + --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw + --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw + /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw + > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match + any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci + tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes + > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt + | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" - run: - name: Run dtests (j11_without_vnodes) + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; + then\n export CQLSH_PYTHON=/usr/bin/python3.8\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests + --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" - cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then - export CQLSH_PYTHON=/usr/bin/python2.7 - fi - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest_j8_without_vnodes path: /tmp/dtest - destination: dtest_j11_without_vnodes - store_artifacts: + destination: dtest_j8_without_vnodes_logs path: ~/cassandra-dtest/logs - destination: dtest_j11_without_vnodes_logs + working_directory: ~/ + j8_cqlsh-dtests-py38-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -2290,29 +2509,90 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - CASSANDRA_USE_JDK11: true - j11_cqlshlib_tests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 1 steps: - attach_workspace: at: /home/cassandra - run: - name: Run cqlshlib Unit Tests - command: | + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' + name: Clone Cassandra dtest Repository (via git) + - run: + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.8/bin/activate + export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra/pylib - ./cassandra-cqlsh-tests.sh .. + + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + + pip3 uninstall -y cqlsh + + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies + - run: + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes + --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only + --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif + [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n + \ grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes + || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset + -eo pipefail && circleci tests split --split-by=timings --timings-type=classname + /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat + /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + - run: + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.8/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; + then\n export CQLSH_PYTHON=/usr/bin/python3.8\nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 + --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j8_with_vnodes) no_output_timeout: 15m - store_test_results: - path: /tmp/cassandra/pylib + path: /tmp/results + - store_artifacts: + destination: dtest_j8_with_vnodes + path: /tmp/dtest + - store_artifacts: + destination: dtest_j8_with_vnodes_logs + path: ~/cassandra-dtest/logs + working_directory: ~/ + j8_cqlshlib_tests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -2345,150 +2625,32 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - CASSANDRA_USE_JDK11: true - j11_repeated_utest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + parallelism: 1 resource_class: medium - working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run repeated JUnit test + command: 'export PATH=$JAVA_HOME/bin:$PATH + + time mv ~/cassandra /tmp + + cd /tmp/cassandra/pylib + + ./cassandra-cqlsh-tests.sh .. + + ' + name: Run cqlshlib Unit Tests no_output_timeout: 15m - command: | - if [ "${REPEATED_UTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times" - - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - target=${REPEATED_UTEST_TARGET} - class_path=${REPEATED_UTEST_CLASS} - class_name="${class_path##*.}" - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" - else - name="-Dtest.name=$class_path" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_UTEST_METHODS}" == "" ]; then - methods="" - else - methods="-Dtest.methods=${REPEATED_UTEST_METHODS}" - fi - - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" - for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt - - # move the XML output files - source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - - (exit ${exit_code}) - fi - fi - store_test_results: - path: /tmp/results/repeated_utest/output - - store_artifacts: - path: /tmp/results/repeated_utest/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utest/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utest/logs - destination: logs + path: /tmp/cassandra/pylib + working_directory: ~/ + j8_dtest_jars_build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -2521,51 +2683,159 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - CASSANDRA_USE_JDK11: true - j8_dtests-with-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + parallelism: 1 resource_class: medium + shell: /bin/bash -eo pipefail -l + steps: + - attach_workspace: + at: /home/cassandra + - run: + command: "export PATH=$JAVA_HOME/bin:$PATH\ncd ~/cassandra\nmkdir ~/dtest_jars\ngit + remote add apache git://github.com/apache/cassandra.git\nfor branch in cassandra-2.2 + cassandra-3.0 cassandra-3.11 cassandra-4.0 trunk; do\n # check out the + correct cassandra version:\n git remote set-branches --add apache '$branch'\n + \ git fetch --depth 1 apache $branch\n git checkout $branch\n git clean + -fd\n # Loop to prevent failure due to maven-ant-tasks not downloading + a jar..\n for x in $(seq 1 3); do\n ${ANT_HOME}/bin/ant realclean; + ${ANT_HOME}/bin/ant jar dtest-jar\n RETURN=\"$?\"\n if [ \"${RETURN}\" + -eq \"0\" ]; then\n cp build/dtest*.jar ~/dtest_jars\n break\n + \ fi\n done\n # Exit, if we didn't build successfully\n if [ \"${RETURN}\" + -ne \"0\" ]; then\n echo \"Build failed with exit code: ${RETURN}\"\n + \ exit ${RETURN}\n fi\ndone\n# and build the dtest-jar for the branch + under test\n${ANT_HOME}/bin/ant realclean\ngit checkout origin/$CIRCLE_BRANCH\ngit + clean -fd\nfor x in $(seq 1 3); do\n ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant + jar dtest-jar\n RETURN=\"$?\"\n if [ \"${RETURN}\" -eq \"0\" ]; then\n + \ cp build/dtest*.jar ~/dtest_jars\n break\n fi\ndone\n# + Exit, if we didn't build successfully\nif [ \"${RETURN}\" -ne \"0\" ]; then\n + \ echo \"Build failed with exit code: ${RETURN}\"\n exit ${RETURN}\nfi\nls + -l ~/dtest_jars\n" + name: Build Cassandra DTest jars + no_output_timeout: 15m + - persist_to_workspace: + paths: + - dtest_jars + root: /home/cassandra working_directory: ~/ + j8_dtests-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + parallelism: 25 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j8_with_vnodes) + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests + --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw + --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw + /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw + > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match + any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci + tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes + > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt + | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: - name: Run dtests (j8_with_vnodes) + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests + --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest_j8_without_vnodes path: /tmp/dtest - destination: dtest_j8_with_vnodes - store_artifacts: + destination: dtest_j8_without_vnodes_logs path: ~/cassandra-dtest/logs - destination: dtest_j8_with_vnodes_logs + working_directory: ~/ + j8_dtests-with-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -2600,67 +2870,87 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j11_cqlsh-dtests-py38-no-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 25 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.8/bin/activate + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j11_without_vnodes) + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes + --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only + --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif + [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n + \ grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes + || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset + -eo pipefail && circleci tests split --split-by=timings --timings-type=classname + /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat + /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat + /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + name: Determine Tests to Run (j8_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" - run: - name: Run dtests (j11_without_vnodes) + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd + ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** + done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" + here so that the exit code that circleci will actually use is from pytest + and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset + -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 + --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j8_with_vnodes) no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" - cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt - - source ~/env3.8/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then - export CQLSH_PYTHON=/usr/bin/python3.8 - fi - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest_j8_with_vnodes path: /tmp/dtest - destination: dtest_j11_without_vnodes - store_artifacts: + destination: dtest_j8_with_vnodes_logs path: ~/cassandra-dtest/logs - destination: dtest_j11_without_vnodes_logs + working_directory: ~/ + j8_jvm_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -2693,84 +2983,114 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - CASSANDRA_USE_JDK11: true - j8_jvm_dtests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + parallelism: 26 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 1 steps: - attach_workspace: at: /home/cassandra - run: - name: Determine distributed Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run + command: '# reminder: this code (along with all the steps) is independently + executed on every circle container + + # so the goal here is to get the circleci script to return the tests *this* + container will run + # which we do via the `circleci` cli tool. + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" + > /tmp/all_java_unit_tests.txt + + + # split up the unit tests into groups based on the number of containers + we have + + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename + --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt + > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" + | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + ' + name: Determine distributed Tests to Run no_output_timeout: 15m - run: - name: Log Environment Information - command: | - echo '*** id ***' + command: 'echo ''*** id ***'' + id - echo '*** cat /proc/cpuinfo ***' + + echo ''*** cat /proc/cpuinfo ***'' + cat /proc/cpuinfo - echo '*** free -m ***' + + echo ''*** free -m ***'' + free -m - echo '*** df -m ***' + + echo ''*** df -m ***'' + df -m - echo '*** ifconfig -a ***' + + echo ''*** ifconfig -a ***'' + ifconfig -a - echo '*** uname -a ***' + + echo ''*** uname -a ***'' + uname -a - echo '*** mount ***' + + echo ''*** mount ***'' + mount - echo '*** env ***' + + echo ''*** env ***'' + env - echo '*** java ***' + + echo ''*** java ***'' + which java + java -version + + ' + name: Log Environment Information - run: + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd + /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\ntest_timeout=$(grep + 'name=\"test.distributed.timeout\"' build.xml | awk -F'\"' '{print $4}' + || true)\nif [ -z \"$test_timeout\" ]; then\n test_timeout=$(grep 'name=\"test.timeout\"' + build.xml | awk -F'\"' '{print $4}')\nfi\nant testclasslist -Dtest.timeout=\"$test_timeout\" + -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed\n" name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/cassandra/build/test/output destination: junitxml + path: /tmp/cassandra/build/test/output - store_artifacts: - path: /tmp/cassandra/build/test/logs destination: logs + path: /tmp/cassandra/build/test/logs + working_directory: ~/ + j8_jvm_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -2805,70 +3125,112 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_build: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 12 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 1 steps: + - attach_workspace: + at: /home/cassandra - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Clone Cassandra Repository (via git) - command: | - git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra - - run: - name: Build Cassandra - command: | - export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant clean realclean jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - no_output_timeout: 15m + command: '# reminder: this code (along with all the steps) is independently + executed on every circle container + + # so the goal here is to get the circleci script to return the tests *this* + container will run + + # which we do via the `circleci` cli tool. + + + rm -fr ~/cassandra-dtest/upgrade_tests + + echo "***java tests***" + + + # get all of our unit test filenames + + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" + > /tmp/all_java_unit_tests.txt + + + # split up the unit tests into groups based on the number of containers + we have + + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename + --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt + > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" + | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + ' + name: Determine distributed Tests to Run + no_output_timeout: 15m - run: - name: Run eclipse-warnings - command: | - export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - ant eclipse-warnings - - persist_to_workspace: - root: /home/cassandra - paths: - - cassandra - - .m2 + command: 'echo ''*** id ***'' + + id + + echo ''*** cat /proc/cpuinfo ***'' + + cat /proc/cpuinfo + + echo ''*** free -m ***'' + + free -m + + echo ''*** df -m ***'' + + df -m + + echo ''*** ifconfig -a ***'' + + ifconfig -a + + echo ''*** uname -a ***'' + + uname -a + + echo ''*** mount ***'' + + mount + + echo ''*** env ***'' + + env + + echo ''*** java ***'' + + which java + + java -version + + ' + name: Log Environment Information + - run: + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd + /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\ntest_timeout=$(grep + 'name=\"test.distributed.timeout\"' build.xml | awk -F'\"' '{print $4}' + || true)\nif [ -z \"$test_timeout\" ]; then\n test_timeout=$(grep 'name=\"test.timeout\"' + build.xml | awk -F'\"' '{print $4}')\nfi\nant testclasslist -Dtest.timeout=\"$test_timeout\" + -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed\n" + name: Run Unit Tests (testclasslist) + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + destination: junitxml + path: /tmp/cassandra/build/test/output + - store_artifacts: + destination: logs + path: /tmp/cassandra/build/test/logs + working_directory: ~/ + j8_repeated_dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -2903,67 +3265,84 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_cqlsh-dtests-py3-no-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - - run: - name: Run dtests (j8_without_vnodes) + command: "if [ \"${REPEATED_DTEST_NAME}\" == \"\" ]; then\n echo \"Repeated + dtest name hasn't been defined, exiting without running any test\"\nelif + [ \"${REPEATED_DTEST_COUNT}\" == \"\" ]; then\n echo \"Repeated dtest + count hasn't been defined, exiting without running any test\"\nelif [ \"${REPEATED_DTEST_COUNT}\" + -le 0 ]; then\n echo \"Repeated dtest count is lesser or equals than zero, + exiting without running any test\"\nelse\n\n # Calculate the number of + test iterations to be run by the current parallel runner.\n # Since we + are running the same test multiple times there is no need to use `circleci + tests split`.\n count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL))\n + \ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); + then\n count=$((count+1))\n fi\n\n if (($count <= 0)); then\n echo + \"No tests to run in this runner\"\n else\n echo \"Running ${REPEATED_DTEST_NAME} + $count times\"\n\n source ~/env3.6/bin/activate\n export PATH=$JAVA_HOME/bin:$PATH\n\n + \ java -version\n cd ~/cassandra-dtest\n mkdir -p /tmp/dtest\n\n + \ echo \"env: $(env)\"\n echo \"** done env\"\n mkdir -p /tmp/results/dtests\n\n + \ stop_on_failure_arg=\"\"\n if ${REPEATED_DTEST_STOP_ON_FAILURE}; + then\n stop_on_failure_arg=\"-x\"\n fi\n\n vnodes_args=\"\"\n + \ if ${REPEATED_DTEST_VNODES}; then\n vnodes_args=\"--use-vnodes + --num-tokens=16\"\n fi\n\n upgrade_arg=\"\"\n if false; then\n + \ upgrade_arg=\"--execute-upgrade-tests\"\n fi\n\n # we need the + \"set -o pipefail\" here so that the exit code that circleci will actually + use is from pytest and not the exit code from tee\n set -o pipefail && + cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg + $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} + | tee /tmp/dtest/stdout.txt\n fi\nfi\n" + name: Run repeated Python dtest no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then - export CQLSH_PYTHON=/usr/bin/python3.6 - fi - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest path: /tmp/dtest - destination: dtest_j8_without_vnodes - store_artifacts: + destination: dtest_logs path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + working_directory: ~/ + j8_repeated_utest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -2998,67 +3377,113 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_cqlsh-dtests-py38-with-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.8/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j8_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - - run: - name: Run dtests (j8_with_vnodes) - no_output_timeout: 15m - command: | - echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + command: 'echo ''*** id ***'' - source ~/env3.8/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then - export CQLSH_PYTHON=/usr/bin/python3.8 - fi + id + + echo ''*** cat /proc/cpuinfo ***'' + + cat /proc/cpuinfo + + echo ''*** free -m ***'' + + free -m + + echo ''*** df -m ***'' + + df -m + + echo ''*** ifconfig -a ***'' + + ifconfig -a + + echo ''*** uname -a ***'' + + uname -a + + echo ''*** mount ***'' + + mount + + echo ''*** env ***'' + + env + + echo ''*** java ***'' + + which java java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + + ' + name: Log Environment Information + - run: + command: "if [ \"${REPEATED_UTEST_CLASS}\" == \"\" ]; then\n echo \"Repeated + utest class name hasn't been defined, exiting without running any test\"\nelif + [ \"${REPEATED_UTEST_COUNT}\" == \"\" ]; then\n echo \"Repeated utest + count hasn't been defined, exiting without running any test\"\nelif [ \"${REPEATED_UTEST_COUNT}\" + -le 0 ]; then\n echo \"Repeated utest count is lesser or equals than zero, + exiting without running any test\"\nelse\n\n # Calculate the number of + test iterations to be run by the current parallel runner.\n # Since we + are running the same test multiple times there is no need to use `circleci + tests split`.\n count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL))\n + \ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); + then\n count=$((count+1))\n fi\n\n if (($count <= 0)); then\n echo + \"No tests to run in this runner\"\n else\n echo \"Running ${REPEATED_UTEST_TARGET} + ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} + times\"\n\n set -x\n export PATH=$JAVA_HOME/bin:$PATH\n time mv + ~/cassandra /tmp\n cd /tmp/cassandra\n if [ -d ~/dtest_jars ]; then\n + \ cp ~/dtest_jars/dtest* /tmp/cassandra/build/\n fi\n\n target=${REPEATED_UTEST_TARGET}\n + \ class_path=${REPEATED_UTEST_CLASS}\n class_name=\"${class_path##*.}\"\n\n + \ # Prepare the -Dtest.name argument.\n # It can be the fully qualified + class name or the short class name, depending on the target.\n if [[ + $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target + == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" + ]]; then\n name=\"-Dtest.name=$class_name\"\n else\n name=\"-Dtest.name=$class_path\"\n + \ fi\n\n # Prepare the -Dtest.methods argument, which is optional\n + \ if [ \"${REPEATED_UTEST_METHODS}\" == \"\" ]; then\n methods=\"\"\n + \ else\n methods=\"-Dtest.methods=${REPEATED_UTEST_METHODS}\"\n fi\n\n + \ # Run the test target as many times as requested collecting the exit + code,\n # stopping the iteration only if stop_on_failure is set.\n exit_code=\"$?\"\n + \ for i in $(seq -w 1 $count); do\n\n echo \"Running test iteration + $i of $count\"\n\n # run the test\n status=\"passes\"\n if + !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | + tee stdout.txt ); then\n status=\"fails\"\n exit_code=1\n + \ fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utest/stdout/${status}/${i}\n + \ mkdir -p $dest\n mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt\n\n + \ # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utest/output/${status}/${i}\n + \ mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; + then\n mv $source/* $dest/\n fi\n\n # move the log files\n + \ source=build/test/logs\n dest=/tmp/results/repeated_utest/logs/${status}/${i}\n + \ mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; + then\n mv $source/* $dest/\n fi\n\n # maybe stop iterations + on test failure\n if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] + && (( $exit_code > 0 )); then\n break\n fi\n done\n\n (exit + ${exit_code})\n fi\nfi\n" + name: Run repeated JUnit test + no_output_timeout: 15m - store_test_results: - path: /tmp/results + path: /tmp/results/repeated_utest/output - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_with_vnodes + destination: stdout + path: /tmp/results/repeated_utest/stdout - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_with_vnodes_logs + destination: junitxml + path: /tmp/results/repeated_utest/output + - store_artifacts: + destination: logs + path: /tmp/results/repeated_utest/logs + working_directory: ~/ + j8_unit_tests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -3093,35 +3518,112 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_long: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 36 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 1 steps: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (long-test) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - ant long-test + command: '# reminder: this code (along with all the steps) is independently + executed on every circle container + + # so the goal here is to get the circleci script to return the tests *this* + container will run + + # which we do via the `circleci` cli tool. + + + rm -fr ~/cassandra-dtest/upgrade_tests + + echo "***java tests***" + + + # get all of our unit test filenames + + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" + > /tmp/all_java_unit_tests.txt + + + # split up the unit tests into groups based on the number of containers + we have + + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename + --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt + > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" + | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + ' + name: Determine unit Tests to Run + no_output_timeout: 15m + - run: + command: 'echo ''*** id ***'' + + id + + echo ''*** cat /proc/cpuinfo ***'' + + cat /proc/cpuinfo + + echo ''*** free -m ***'' + + free -m + + echo ''*** df -m ***'' + + df -m + + echo ''*** ifconfig -a ***'' + + ifconfig -a + + echo ''*** uname -a ***'' + + uname -a + + echo ''*** mount ***'' + + mount + + echo ''*** env ***'' + + env + + echo ''*** java ***'' + + which java + + java -version + + ' + name: Log Environment Information + - run: + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd + /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\ntest_timeout=$(grep + 'name=\"test.unit.timeout\"' build.xml | awk -F'\"' '{print $4}' || true)\nif + [ -z \"$test_timeout\" ]; then\n test_timeout=$(grep 'name=\"test.timeout\"' + build.xml | awk -F'\"' '{print $4}')\nfi\nant testclasslist -Dtest.timeout=\"$test_timeout\" + -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit\n" + name: Run Unit Tests (testclasslist) no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/cassandra/build/test/output destination: junitxml + path: /tmp/cassandra/build/test/output - store_artifacts: - path: /tmp/cassandra/build/test/logs destination: logs + path: /tmp/cassandra/build/test/logs + working_directory: ~/ + j8_upgradetests-no-vnodes: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -3156,135 +3658,90 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_system_keyspace_directory: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 100 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Determine unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" + ' + name: Clone Cassandra dtest Repository (via git) + - run: + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + + source ~/env3.6/bin/activate + + export PATH=$JAVA_HOME/bin:$PATH + + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + + pip3 uninstall -y cqlsh + + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version + command: "# reminder: this code (along with all the steps) is independently + executed on every circle container\n# so the goal here is to get the circleci + script to return the tests *this* container will run\n# which we do via + the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport + PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected + DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py + --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection + all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw + --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw + /tmp/all_dtest_tests_j8_upgradetests_without_vnodes\nelse\n grep -e '' + /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes + || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset + -eo pipefail && circleci tests split --split-by=timings --timings-type=classname + /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt\ncat + /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\\n' ' ' + > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n" + name: Determine Tests to Run (j8_upgradetests_without_vnodes) + no_output_timeout: 5m - run: - name: Run Unit Tests (testclasslist-system-keyspace-directory) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat + /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource + ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n + \ export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho + \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we + need the \"set -o pipefail\" here so that the exit code that circleci will + actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat + /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nset -o + pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only + --upgrade-target-version-only --upgrade-version-selection all --log-cli-level=DEBUG + --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS + 2>&1 | tee /tmp/dtest/stdout.txt\n" + name: Run dtests (j8_upgradetests_without_vnodes) no_output_timeout: 15m - store_test_results: - path: /tmp/cassandra/build/test/output/ + path: /tmp/results - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml + destination: dtest_j8_upgradetests_without_vnodes + path: /tmp/dtest - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - ANT_HOME: /usr/share/ant - - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_cqlshlib_tests: + destination: dtest_j8_upgradetests_without_vnodes_logs + path: ~/cassandra-dtest/logs + working_directory: ~/ + repeated_jvm_upgrade_dtest: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Run cqlshlib Unit Tests - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra/pylib - ./cassandra-cqlsh-tests.sh .. - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/pylib environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -3319,35 +3776,114 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_fqltool: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 1 steps: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (fqltool-test) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - ant fqltool-test + command: 'echo ''*** id ***'' + + id + + echo ''*** cat /proc/cpuinfo ***'' + + cat /proc/cpuinfo + + echo ''*** free -m ***'' + + free -m + + echo ''*** df -m ***'' + + df -m + + echo ''*** ifconfig -a ***'' + + ifconfig -a + + echo ''*** uname -a ***'' + + uname -a + + echo ''*** mount ***'' + + mount + + echo ''*** env ***'' + + env + + echo ''*** java ***'' + + which java + + java -version + + ' + name: Log Environment Information + - run: + command: "if [ \"${REPEATED_JVM_UPGRADE_DTEST_CLASS}\" == \"\" ]; then\n + \ echo \"Repeated utest class name hasn't been defined, exiting without + running any test\"\nelif [ \"${REPEATED_JVM_UPGRADE_DTEST_COUNT}\" == \"\" + ]; then\n echo \"Repeated utest count hasn't been defined, exiting without + running any test\"\nelif [ \"${REPEATED_JVM_UPGRADE_DTEST_COUNT}\" -le 0 + ]; then\n echo \"Repeated utest count is lesser or equals than zero, exiting + without running any test\"\nelse\n\n # Calculate the number of test iterations + to be run by the current parallel runner.\n # Since we are running the + same test multiple times there is no need to use `circleci tests split`.\n + \ count=$((${REPEATED_JVM_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL))\n if + (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); + then\n count=$((count+1))\n fi\n\n if (($count <= 0)); then\n echo + \"No tests to run in this runner\"\n else\n echo \"Running test-jvm-dtest-some + ${REPEATED_JVM_UPGRADE_DTEST_CLASS} ${REPEATED_JVM_UPGRADE_DTEST_METHODS} + ${REPEATED_JVM_UPGRADE_DTEST_COUNT} times\"\n\n set -x\n export PATH=$JAVA_HOME/bin:$PATH\n + \ time mv ~/cassandra /tmp\n cd /tmp/cassandra\n if [ -d ~/dtest_jars + ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\n fi\n\n + \ target=test-jvm-dtest-some\n class_path=${REPEATED_JVM_UPGRADE_DTEST_CLASS}\n + \ class_name=\"${class_path##*.}\"\n\n # Prepare the -Dtest.name argument.\n + \ # It can be the fully qualified class name or the short class name, + depending on the target.\n if [[ $target == \"test\" || \\\n $target + == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n + \ $target == \"test-system-keyspace-directory\" ]]; then\n name=\"-Dtest.name=$class_name\"\n + \ else\n name=\"-Dtest.name=$class_path\"\n fi\n\n # Prepare + the -Dtest.methods argument, which is optional\n if [ \"${REPEATED_JVM_UPGRADE_DTEST_METHODS}\" + == \"\" ]; then\n methods=\"\"\n else\n methods=\"-Dtest.methods=${REPEATED_JVM_UPGRADE_DTEST_METHODS}\"\n + \ fi\n\n # Run the test target as many times as requested collecting + the exit code,\n # stopping the iteration only if stop_on_failure is + set.\n exit_code=\"$?\"\n for i in $(seq -w 1 $count); do\n\n echo + \"Running test iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n + \ if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true + | tee stdout.txt ); then\n status=\"fails\"\n exit_code=1\n + \ fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utest/stdout/${status}/${i}\n + \ mkdir -p $dest\n mv stdout.txt $dest/test-jvm-dtest-some-${REPEATED_JVM_UPGRADE_DTEST_CLASS}.txt\n\n + \ # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utest/output/${status}/${i}\n + \ mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; + then\n mv $source/* $dest/\n fi\n\n # move the log files\n + \ source=build/test/logs\n dest=/tmp/results/repeated_utest/logs/${status}/${i}\n + \ mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; + then\n mv $source/* $dest/\n fi\n\n # maybe stop iterations + on test failure\n if [[ ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} + = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\n\n + \ (exit ${exit_code})\n fi\nfi\n" + name: Run repeated JUnit test no_output_timeout: 15m - store_test_results: - path: /tmp/cassandra/build/test/output/ + path: /tmp/results/repeated_utest/output + - store_artifacts: + destination: stdout + path: /tmp/results/repeated_utest/stdout - store_artifacts: - path: /tmp/cassandra/build/test/output destination: junitxml + path: /tmp/results/repeated_utest/output - store_artifacts: - path: /tmp/cassandra/build/test/logs destination: logs + path: /tmp/results/repeated_utest/logs + working_directory: ~/ + repeated_upgrade_dtest: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -3382,70 +3918,83 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j11_dtests-no-vnodes: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: + command: 'git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO + ~/cassandra-dtest + + ' name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + command: '# note, this should be super quick as all dependencies should be + pre-installed in the docker image + + # if additional dependencies were added to requirmeents.txt and the docker + image hasn''t been updated + + # we''d have to install it here at runtime -- which will make things slow, + so do yourself a favor and + + # rebuild the docker image! (it automatically pulls the latest requirements.txt + on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + + ' + name: Configure virtualenv and python Dependencies - run: - name: Determine Tests to Run (j11_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" - - run: - name: Run dtests (j11_without_vnodes) + command: "if [ \"${REPEATED_UPGRADE_DTEST_NAME}\" == \"\" ]; then\n echo + \"Repeated dtest name hasn't been defined, exiting without running any test\"\nelif + [ \"${REPEATED_UPGRADE_DTEST_COUNT}\" == \"\" ]; then\n echo \"Repeated + dtest count hasn't been defined, exiting without running any test\"\nelif + [ \"${REPEATED_UPGRADE_DTEST_COUNT}\" -le 0 ]; then\n echo \"Repeated dtest + count is lesser or equals than zero, exiting without running any test\"\nelse\n\n + \ # Calculate the number of test iterations to be run by the current parallel + runner.\n # Since we are running the same test multiple times there is + no need to use `circleci tests split`.\n count=$((${REPEATED_UPGRADE_DTEST_COUNT} + / CIRCLE_NODE_TOTAL))\n if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} + % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\n fi\n\n if (($count + <= 0)); then\n echo \"No tests to run in this runner\"\n else\n echo + \"Running ${REPEATED_UPGRADE_DTEST_NAME} $count times\"\n\n source ~/env3.6/bin/activate\n + \ export PATH=$JAVA_HOME/bin:$PATH\n\n java -version\n cd ~/cassandra-dtest\n + \ mkdir -p /tmp/dtest\n\n echo \"env: $(env)\"\n echo \"** done + env\"\n mkdir -p /tmp/results/dtests\n\n stop_on_failure_arg=\"\"\n + \ if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then\n stop_on_failure_arg=\"-x\"\n + \ fi\n\n vnodes_args=\"\"\n if false; then\n vnodes_args=\"--use-vnodes + --num-tokens=16\"\n fi\n\n upgrade_arg=\"\"\n if true; then\n upgrade_arg=\"--execute-upgrade-tests\"\n + \ fi\n\n # we need the \"set -o pipefail\" here so that the exit code + that circleci will actually use is from pytest and not the exit code from + tee\n set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args + --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml + -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} + | tee /tmp/dtest/stdout.txt\n fi\nfi\n" + name: Run repeated Python dtest no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: + destination: dtest path: /tmp/dtest - destination: dtest_j11_without_vnodes - store_artifacts: + destination: dtest_logs path: ~/cassandra-dtest/logs - destination: dtest_j11_without_vnodes_logs + working_directory: ~/ + utests_compression: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -3478,84 +4027,115 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - - CASSANDRA_USE_JDK11: true - utests_compression: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + parallelism: 20 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Determine unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run + command: '# reminder: this code (along with all the steps) is independently + executed on every circle container + + # so the goal here is to get the circleci script to return the tests *this* + container will run + # which we do via the `circleci` cli tool. + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" + > /tmp/all_java_unit_tests.txt + + + # split up the unit tests into groups based on the number of containers + we have + + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename + --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt + > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" + | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + ' + name: Determine unit Tests to Run no_output_timeout: 15m - run: - name: Log Environment Information - command: | - echo '*** id ***' + command: 'echo ''*** id ***'' + id - echo '*** cat /proc/cpuinfo ***' + + echo ''*** cat /proc/cpuinfo ***'' + cat /proc/cpuinfo - echo '*** free -m ***' + + echo ''*** free -m ***'' + free -m - echo '*** df -m ***' + + echo ''*** df -m ***'' + df -m - echo '*** ifconfig -a ***' + + echo ''*** ifconfig -a ***'' + ifconfig -a - echo '*** uname -a ***' + + echo ''*** uname -a ***'' + uname -a - echo '*** mount ***' + + echo ''*** mount ***'' + mount - echo '*** env ***' + + echo ''*** env ***'' + env - echo '*** java ***' + + echo ''*** java ***'' + which java + java -version + + ' + name: Log Environment Information - run: + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd + /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\ntest_timeout=$(grep + 'name=\"test.unit.timeout\"' build.xml | awk -F'\"' '{print $4}' || true)\nif + [ -z \"$test_timeout\" ]; then\n test_timeout=$(grep 'name=\"test.timeout\"' + build.xml | awk -F'\"' '{print $4}')\nfi\nant testclasslist-compression + -Dtest.timeout=\"$test_timeout\" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + \ -Dtest.classlistprefix=unit\n" name: Run Unit Tests (testclasslist-compression) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/cassandra/build/test/output destination: junitxml + path: /tmp/cassandra/build/test/output - store_artifacts: - path: /tmp/cassandra/build/test/logs destination: logs + path: /tmp/cassandra/build/test/logs + working_directory: ~/ + utests_fqltool: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -3590,147 +4170,30 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_repeated_utest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ + parallelism: 1 + resource_class: xlarge shell: /bin/bash -eo pipefail -l - parallelism: 4 steps: - attach_workspace: at: /home/cassandra - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run repeated JUnit test + command: "export PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif + [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\nant + fqltool-test\n" + name: Run Unit Tests (fqltool-test) no_output_timeout: 15m - command: | - if [ "${REPEATED_UTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times" - - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - target=${REPEATED_UTEST_TARGET} - class_path=${REPEATED_UTEST_CLASS} - class_name="${class_path##*.}" - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" - else - name="-Dtest.name=$class_path" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_UTEST_METHODS}" == "" ]; then - methods="" - else - methods="-Dtest.methods=${REPEATED_UTEST_METHODS}" - fi - - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" - for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt - - # move the XML output files - source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - - (exit ${exit_code}) - fi - fi - store_test_results: - path: /tmp/results/repeated_utest/output - - store_artifacts: - path: /tmp/results/repeated_utest/stdout - destination: stdout + path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/results/repeated_utest/output destination: junitxml + path: /tmp/cassandra/build/test/output - store_artifacts: - path: /tmp/results/repeated_utest/logs destination: logs + path: /tmp/cassandra/build/test/logs + working_directory: ~/ + utests_long: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -3765,67 +4228,88 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtest_jars_build: + parallelism: 1 + resource_class: xlarge + shell: /bin/bash -eo pipefail -l + steps: + - attach_workspace: + at: /home/cassandra + - run: + command: "export PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif + [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\nant + long-test\n" + name: Run Unit Tests (long-test) + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + destination: junitxml + path: /tmp/cassandra/build/test/output + - store_artifacts: + destination: logs + path: /tmp/cassandra/build/test/logs + working_directory: ~/ + utests_stress: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_UTEST_TARGET: testsome + - REPEATED_UTEST_CLASS: null + - REPEATED_UTEST_METHODS: null + - REPEATED_UTEST_COUNT: 100 + - REPEATED_UTEST_STOP_ON_FAILURE: false + - REPEATED_DTEST_NAME: null + - REPEATED_DTEST_VNODES: false + - REPEATED_DTEST_COUNT: 100 + - REPEATED_DTEST_STOP_ON_FAILURE: false + - REPEATED_UPGRADE_DTEST_NAME: null + - REPEATED_UPGRADE_DTEST_COUNT: 100 + - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_JVM_UPGRADE_DTEST_CLASS: null + - REPEATED_JVM_UPGRADE_DTEST_METHODS: null + - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 + - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 parallelism: 1 + resource_class: xlarge + shell: /bin/bash -eo pipefail -l steps: - attach_workspace: at: /home/cassandra - run: - name: Build Cassandra DTest jars - command: | - export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - mkdir ~/dtest_jars - git remote add apache git://github.com/apache/cassandra.git - for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 trunk; do - # check out the correct cassandra version: - git remote set-branches --add apache '$branch' - git fetch --depth 1 apache $branch - git checkout $branch - git clean -fd - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - cp build/dtest*.jar ~/dtest_jars - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - done - # and build the dtest-jar for the branch under test - ${ANT_HOME}/bin/ant realclean - git checkout origin/$CIRCLE_BRANCH - git clean -fd - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - cp build/dtest*.jar ~/dtest_jars - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - ls -l ~/dtest_jars + command: "export PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif + [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\nant + stress-test\n" + name: Run Unit Tests (stress-test) no_output_timeout: 15m - - persist_to_workspace: - root: /home/cassandra - paths: - - dtest_jars + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + destination: junitxml + path: /tmp/cassandra/build/test/output + - store_artifacts: + destination: logs + path: /tmp/cassandra/build/test/logs + working_directory: ~/ + utests_system_keyspace_directory: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304 environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@ -3860,202 +4344,151 @@ jobs: - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + parallelism: 20 + resource_class: xlarge + shell: /bin/bash -eo pipefail -l + steps: + - attach_workspace: + at: /home/cassandra + - run: + command: '# reminder: this code (along with all the steps) is independently + executed on every circle container + + # so the goal here is to get the circleci script to return the tests *this* + container will run + + # which we do via the `circleci` cli tool. + + + rm -fr ~/cassandra-dtest/upgrade_tests + + echo "***java tests***" + + + # get all of our unit test filenames + + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" + > /tmp/all_java_unit_tests.txt + + + # split up the unit tests into groups based on the number of containers + we have + + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename + --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt + > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" + | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + + ' + name: Determine unit Tests to Run + no_output_timeout: 15m + - run: + command: 'echo ''*** id ***'' + + id + + echo ''*** cat /proc/cpuinfo ***'' + + cat /proc/cpuinfo + + echo ''*** free -m ***'' + + free -m + + echo ''*** df -m ***'' + + df -m + + echo ''*** ifconfig -a ***'' + + ifconfig -a + + echo ''*** uname -a ***'' + + uname -a + + echo ''*** mount ***'' + + mount + + echo ''*** env ***'' + + env + + echo ''*** java ***'' + + which java + + java -version + + ' + name: Log Environment Information + - run: + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd + /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\ntest_timeout=$(grep + 'name=\"test.unit.timeout\"' build.xml | awk -F'\"' '{print $4}' || true)\nif + [ -z \"$test_timeout\" ]; then\n test_timeout=$(grep 'name=\"test.timeout\"' + build.xml | awk -F'\"' '{print $4}')\nfi\nant testclasslist-system-keyspace-directory + -Dtest.timeout=\"$test_timeout\" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + \ -Dtest.classlistprefix=unit\n" + name: Run Unit Tests (testclasslist-system-keyspace-directory) + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + destination: junitxml + path: /tmp/cassandra/build/test/output + - store_artifacts: + destination: logs + path: /tmp/cassandra/build/test/logs + working_directory: ~/ +version: 2 workflows: - version: 2 - java8_separate_tests: + java11_tests: jobs: - - start_j8_build: - type: approval - - j8_build: - requires: - - start_j8_build - - start_j8_unit_tests: - type: approval - - j8_unit_tests: - requires: - - start_j8_unit_tests - - j8_build - - start_j8_jvm_dtests: - type: approval - - j8_jvm_dtests: - requires: - - start_j8_jvm_dtests - - j8_build - - start_j8_cqlshlib_tests: - type: approval - - j8_cqlshlib_tests: - requires: - - start_j8_cqlshlib_tests - - j8_build - - start_j11_unit_tests: - type: approval + - j11_build: {} - j11_unit_tests: requires: - - start_j11_unit_tests - - j8_build - - start_utests_long: - type: approval - - utests_long: - requires: - - start_utests_long - - j8_build - - start_utests_compression: - type: approval - - utests_compression: - requires: - - start_utests_compression - - j8_build - - start_utests_stress: - type: approval - - utests_stress: - requires: - - start_utests_stress - - j8_build - - start_utests_fqltool: - type: approval - - utests_fqltool: - requires: - - start_utests_fqltool - - j8_build - - start_utests_system_keyspace_directory: - type: approval - - utests_system_keyspace_directory: - requires: - - start_utests_system_keyspace_directory - - j8_build - - start_j8_dtest_jars_build: - type: approval - - j8_dtest_jars_build: - requires: - - j8_build - - start_j8_dtest_jars_build - - start_jvm_upgrade_dtest: - type: approval - - j8_jvm_upgrade_dtests: - requires: - - start_jvm_upgrade_dtest - - j8_dtest_jars_build - - start_j8_dtests: - type: approval - - j8_dtests-with-vnodes: + - j11_build + - j11_jvm_dtests: requires: - - start_j8_dtests - - j8_build - - j8_dtests-no-vnodes: + - j11_build + - j11_cqlshlib_tests: requires: - - start_j8_dtests - - j8_build - - start_j11_dtests: - type: approval + - j11_build - j11_dtests-with-vnodes: requires: - - start_j11_dtests - - j8_build + - j11_build - j11_dtests-no-vnodes: requires: - - start_j11_dtests - - j8_build - - start_upgrade_tests: - type: approval - - j8_upgradetests-no-vnodes: - requires: - - start_upgrade_tests - - j8_build - - start_j8_cqlsh_tests: - type: approval - - j8_cqlsh-dtests-py2-with-vnodes: - requires: - - start_j8_cqlsh_tests - - j8_build - - j8_cqlsh-dtests-py3-with-vnodes: - requires: - - start_j8_cqlsh_tests - - j8_build - - j8_cqlsh-dtests-py38-with-vnodes: - requires: - - start_j8_cqlsh_tests - - j8_build - - j8_cqlsh-dtests-py2-no-vnodes: - requires: - - start_j8_cqlsh_tests - - j8_build - - j8_cqlsh-dtests-py3-no-vnodes: - requires: - - start_j8_cqlsh_tests - - j8_build - - j8_cqlsh-dtests-py38-no-vnodes: - requires: - - start_j8_cqlsh_tests - - j8_build - - start_j11_cqlsh_tests: - type: approval + - j11_build - j11_cqlsh-dtests-py2-with-vnodes: requires: - - start_j11_cqlsh_tests - - j8_build + - j11_build - j11_cqlsh-dtests-py3-with-vnodes: requires: - - start_j11_cqlsh_tests - - j8_build + - j11_build - j11_cqlsh-dtests-py38-with-vnodes: requires: - - start_j11_cqlsh_tests - - j8_build + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - - start_j11_cqlsh_tests - - j8_build + - j11_build - j11_cqlsh-dtests-py3-no-vnodes: requires: - - start_j11_cqlsh_tests - - j8_build + - j11_build - j11_cqlsh-dtests-py38-no-vnodes: requires: - - start_j11_cqlsh_tests - - j8_build - - start_j8_repeated_utest: - type: approval - - j8_repeated_utest: - requires: - - start_j8_repeated_utest - - j8_build - - start_j11_repeated_utest: - type: approval - - j11_repeated_utest: - requires: - - start_j11_repeated_utest - - j8_build - - start_j8_repeated_dtest: - type: approval - - j8_repeated_dtest: - requires: - - start_j8_repeated_dtest - - j8_build - - start_j11_repeated_dtest: - type: approval - - j11_repeated_dtest: - requires: - - start_j11_repeated_dtest - - j8_build - - start_repeated_upgrade_dtest: - type: approval - - repeated_upgrade_dtest: - requires: - - start_repeated_upgrade_dtest - - j8_build - - start_repeated_jvm_upgrade_dtest: - type: approval - - repeated_jvm_upgrade_dtest: - requires: - - start_repeated_jvm_upgrade_dtest - - j8_dtest_jars_build - java8_pre-commit_tests: + - j11_build + java8_tests: jobs: - - start_pre-commit_tests: - type: approval - - j8_build: - requires: - - start_pre-commit_tests + - j8_build: {} - j8_unit_tests: requires: - j8_build @@ -4065,9 +4498,6 @@ workflows: - j8_cqlshlib_tests: requires: - j8_build - - j11_unit_tests: - requires: - - j8_build - start_utests_long: type: approval - utests_long: @@ -4086,11 +4516,8 @@ workflows: requires: - start_utests_stress - j8_build - - start_utests_fqltool: - type: approval - utests_fqltool: requires: - - start_utests_fqltool - j8_build - start_utests_system_keyspace_directory: type: approval @@ -4098,17 +4525,11 @@ workflows: requires: - start_utests_system_keyspace_directory - j8_build - - start_j8_dtest_jars_build: - type: approval - j8_dtest_jars_build: requires: - j8_build - - start_j8_dtest_jars_build - - start_jvm_upgrade_dtest: - type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest - j8_dtest_jars_build - j8_dtests-with-vnodes: requires: @@ -4116,18 +4537,9 @@ workflows: - j8_dtests-no-vnodes: requires: - j8_build - - j11_dtests-with-vnodes: - requires: - - j8_build - - j11_dtests-no-vnodes: - requires: - - j8_build - - start_upgrade_tests: - type: approval - j8_upgradetests-no-vnodes: requires: - j8_build - - start_upgrade_tests - j8_cqlsh-dtests-py2-with-vnodes: requires: - j8_build @@ -4146,188 +4558,4 @@ workflows: - j8_cqlsh-dtests-py38-no-vnodes: requires: - j8_build - - j11_cqlsh-dtests-py2-with-vnodes: - requires: - - j8_build - - j11_cqlsh-dtests-py3-with-vnodes: - requires: - - j8_build - - j11_cqlsh-dtests-py38-with-vnodes: - requires: - - j8_build - - j11_cqlsh-dtests-py2-no-vnodes: - requires: - - j8_build - - j11_cqlsh-dtests-py3-no-vnodes: - requires: - - j8_build - - j11_cqlsh-dtests-py38-no-vnodes: - requires: - - j8_build - - start_j8_repeated_utest: - type: approval - - j8_repeated_utest: - requires: - - start_j8_repeated_utest - - j8_build - - start_j11_repeated_utest: - type: approval - - j11_repeated_utest: - requires: - - start_j11_repeated_utest - - j8_build - - start_j8_repeated_dtest: - type: approval - - j8_repeated_dtest: - requires: - - start_j8_repeated_dtest - - j8_build - - start_j11_repeated_dtest: - type: approval - - j11_repeated_dtest: - requires: - - start_j11_repeated_dtest - - j8_build - - start_repeated_upgrade_dtest: - type: approval - - repeated_upgrade_dtest: - requires: - - start_repeated_upgrade_dtest - - j8_build - - start_repeated_jvm_upgrade_dtest: - type: approval - - repeated_jvm_upgrade_dtest: - requires: - - start_repeated_jvm_upgrade_dtest - - j8_dtest_jars_build - java11_separate_tests: - jobs: - - start_j11_build: - type: approval - - j11_build: - requires: - - start_j11_build - - start_j11_unit_tests: - type: approval - - j11_unit_tests: - requires: - - start_j11_unit_tests - - j11_build - - start_j11_jvm_dtests: - type: approval - - j11_jvm_dtests: - requires: - - start_j11_jvm_dtests - - j11_build - - start_j11_cqlshlib_tests: - type: approval - - j11_cqlshlib_tests: - requires: - - start_j11_cqlshlib_tests - - j11_build - - start_j11_dtests: - type: approval - - j11_dtests-with-vnodes: - requires: - - start_j11_dtests - - j11_build - - j11_dtests-no-vnodes: - requires: - - start_j11_dtests - - j11_build - - start_j11_cqlsh_tests: - type: approval - - j11_cqlsh-dtests-py2-with-vnodes: - requires: - - start_j11_cqlsh_tests - - j11_build - - j11_cqlsh-dtests-py3-with-vnodes: - requires: - - start_j11_cqlsh_tests - - j11_build - - j11_cqlsh-dtests-py38-with-vnodes: - requires: - - start_j11_cqlsh_tests - - j11_build - - j11_cqlsh-dtests-py2-no-vnodes: - requires: - - start_j11_cqlsh_tests - - j11_build - - j11_cqlsh-dtests-py3-no-vnodes: - requires: - - start_j11_cqlsh_tests - - j11_build - - j11_cqlsh-dtests-py38-no-vnodes: - requires: - - start_j11_cqlsh_tests - - j11_build - - start_j11_repeated_utest: - type: approval - - j11_repeated_utest: - requires: - - start_j11_repeated_utest - - j11_build - - start_j11_repeated_dtest: - type: approval - - j11_repeated_dtest: - requires: - - start_j11_repeated_dtest - - j11_build - java11_pre-commit_tests: - jobs: - - start_pre-commit_tests: - type: approval - - j11_build: - requires: - - start_pre-commit_tests - - j11_unit_tests: - requires: - - j11_build - - j11_jvm_dtests: - requires: - - j11_build - - j11_cqlshlib_tests: - requires: - - j11_build - - j11_jvm_dtests: - requires: - - j11_build - - j11_cqlshlib_tests: - requires: - - j11_build - - j11_dtests-with-vnodes: - requires: - - j11_build - - j11_dtests-no-vnodes: - requires: - - j11_build - - j11_cqlsh-dtests-py2-with-vnodes: - requires: - - j11_build - - j11_cqlsh-dtests-py3-with-vnodes: - requires: - - j11_build - - j11_cqlsh-dtests-py38-with-vnodes: - requires: - - j11_build - - j11_cqlsh-dtests-py2-no-vnodes: - requires: - - j11_build - - j11_cqlsh-dtests-py3-no-vnodes: - requires: - - j11_build - - j11_cqlsh-dtests-py38-no-vnodes: - requires: - - j11_build - - start_j11_repeated_utest: - type: approval - - j11_repeated_utest: - requires: - - start_j11_repeated_utest - - j11_build - - start_j11_repeated_dtest: - type: approval - - j11_repeated_dtest: - requires: - - start_j11_repeated_dtest - - j11_build + version: 2 diff --git a/src/java/org/apache/cassandra/net/InboundSockets.java b/src/java/org/apache/cassandra/net/InboundSockets.java index d1f76d3ea854..d0659eee928c 100644 --- a/src/java/org/apache/cassandra/net/InboundSockets.java +++ b/src/java/org/apache/cassandra/net/InboundSockets.java @@ -99,15 +99,28 @@ private Future open(Consumer pipelineInjector) throw new IllegalStateException(); binding = InboundConnectionInitiator.bind(settings, connections, pipelineInjector); } - - return binding.addListener(ignore -> { + // isOpen is defined as "listen.isOpen", but this is set AFTER the binding future is set + // to make sure the future returned does not complete until listen is set, need a new + // future to replicate "Future.map" behavior. + AsyncChannelPromise promise = new AsyncChannelPromise(binding.channel()); + binding.addListener(f -> { + if (!f.isSuccess()) + { + synchronized (this) + { + binding = null; + } + promise.setFailure(f.cause()); + return; + } synchronized (this) { - if (binding.isSuccess()) - listen = binding.channel(); + listen = binding.channel(); binding = null; } + promise.setSuccess(null); }); + return promise; } /** diff --git a/test/unit/org/apache/cassandra/net/MessagingServiceTest.java b/test/unit/org/apache/cassandra/net/MessagingServiceTest.java index a82315b48b5f..871f5924abab 100644 --- a/test/unit/org/apache/cassandra/net/MessagingServiceTest.java +++ b/test/unit/org/apache/cassandra/net/MessagingServiceTest.java @@ -310,9 +310,12 @@ public void listenRequiredSecureConnectionWithBroadcastAddrAndLegacyPort() throw @Test public void listenOptionalSecureConnection() throws InterruptedException { - ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions() - .withOptional(true); - listen(serverEncryptionOptions, false); + for (int i = 0; i < 500; i++) // test used to be flaky, so run in a loop to make sure stable (see CASSANDRA-17033) + { + ServerEncryptionOptions serverEncryptionOptions = new ServerEncryptionOptions() + .withOptional(true); + listen(serverEncryptionOptions, false); + } } @Test @@ -339,8 +342,8 @@ private void listen(ServerEncryptionOptions serverEncryptionOptions, boolean lis InboundSockets connections = new InboundSockets(settings); try { - connections.open().await(); - Assert.assertTrue(connections.isListening()); + connections.open().sync(); + Assert.assertTrue("connections is not listening", connections.isListening()); Set expect = new HashSet<>(); expect.add(InetAddressAndPort.getByAddressOverrideDefaults(listenAddress, DatabaseDescriptor.getStoragePort()));