From d970146620834490f735fc4a1eee498e4b64f68d Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Tue, 7 Apr 2020 18:01:34 +0300 Subject: [PATCH 01/78] ignite-ducktape: Initial commit. --- tests/MANIFEST.in | 16 + tests/docker/Dockerfile | 63 +++ tests/docker/build/cluster.json | 163 +++++++ tests/docker/build/node_hosts | 14 + tests/docker/ducker-ignite | 580 +++++++++++++++++++++++ tests/docker/run_tests.sh | 30 ++ tests/docker/ssh-config | 21 + tests/docker/ssh/authorized_keys | 15 + tests/docker/ssh/config | 21 + tests/docker/ssh/id_rsa | 27 ++ tests/docker/ssh/id_rsa.pub | 1 + tests/ignitetest/__init__.py | 25 + tests/ignitetest/utils/__init__.py | 16 + tests/ignitetest/utils/remote_account.py | 41 ++ tests/ignitetest/utils/util.py | 138 ++++++ tests/ignitetest/version.py | 140 ++++++ tests/setup.cfg | 30 ++ tests/setup.py | 57 +++ 18 files changed, 1398 insertions(+) create mode 100644 tests/MANIFEST.in create mode 100644 tests/docker/Dockerfile create mode 100644 tests/docker/build/cluster.json create mode 100644 tests/docker/build/node_hosts create mode 100755 tests/docker/ducker-ignite create mode 100755 tests/docker/run_tests.sh create mode 100644 tests/docker/ssh-config create mode 100644 tests/docker/ssh/authorized_keys create mode 100644 tests/docker/ssh/config create mode 100644 tests/docker/ssh/id_rsa create mode 100644 tests/docker/ssh/id_rsa.pub create mode 100644 tests/ignitetest/__init__.py create mode 100644 tests/ignitetest/utils/__init__.py create mode 100644 tests/ignitetest/utils/remote_account.py create mode 100644 tests/ignitetest/utils/util.py create mode 100644 tests/ignitetest/version.py create mode 100644 tests/setup.cfg create mode 100644 tests/setup.py diff --git a/tests/MANIFEST.in b/tests/MANIFEST.in new file mode 100644 index 0000000000000..edef4ab7becbe --- /dev/null +++ b/tests/MANIFEST.in @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +recursive-include ignitetest */templates/* diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile new file mode 100644 index 0000000000000..0ee09b40132bf --- /dev/null +++ b/tests/docker/Dockerfile @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG jdk_version=openjdk:8 +FROM $jdk_version + +MAINTAINER Apache Ignite dev@ignite.apache.org +VOLUME ["/opt/ignite-dev"] + +# Set the timezone. +ENV TZ="/usr/share/zoneinfo/America/Los_Angeles" + +# Do not ask for confirmations when running apt-get, etc. +ENV DEBIAN_FRONTEND noninteractive + +# Set the ducker.creator label so that we know that this is a ducker image. This will make it +# visible to 'ducker purge'. The ducker.creator label also lets us know what UNIX user built this +# image. +ARG ducker_creator=default +LABEL ducker.creator=$ducker_creator + +# Update Linux and install necessary utilities. +RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute && apt-get -y clean +RUN python -m pip install -U pip==9.0.3; +RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.6 + +# Set up ssh +COPY ./ssh-config /root/.ssh/config +# NOTE: The paramiko library supports the PEM-format private key, but does not support the RFC4716 format. +RUN ssh-keygen -m PEM -q -t rsa -N '' -f /root/.ssh/id_rsa && cp -f /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys +RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config + +# Install binary test dependencies. +# we use the same versions as in vagrant/base.sh +ARG IGNITE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" +ARG ARCHIVE_NAME="apache-ignite-2.8.0-bin.zip" +RUN mkdir -p "/opt/ignite-2.8.0" && chmod a+rw /opt/ignite-2.8.0 && cd /opt/ignite-2.8.0 && curl -s "$IGNITE_MIRROR/ignite/2.8.0/$ARCHIVE_NAME" > /opt/ignite-2.8.0/$ARCHIVE_NAME && unzip /opt/ignite-2.8.0/$ARCHIVE_NAME -d /opt/ignite-2.8.0/ + +# The version of Kibosh to use for testing. +# If you update this, also update vagrant/base.sh +ARG KIBOSH_VERSION="8841dd392e6fbf02986e2fb1f1ebf04df344b65a" + +# Install Kibosh +RUN apt-get install fuse +RUN cd /opt && git clone -q https://github.com/confluentinc/kibosh.git && cd "/opt/kibosh" && git reset --hard $KIBOSH_VERSION && mkdir "/opt/kibosh/build" && cd "/opt/kibosh/build" && ../configure && make -j 2 + +# Set up the ducker user. +RUN useradd -ms /bin/bash ducker && mkdir -p /home/ducker/ && rsync -aiq /root/.ssh/ /home/ducker/.ssh && chown -R ducker /home/ducker/ /mnt/ /var/log/ && echo "PATH=$(runuser -l ducker -c 'echo $PATH'):$JAVA_HOME/bin" >> /home/ducker/.ssh/environment && echo 'PATH=$PATH:'"$JAVA_HOME/bin" >> /home/ducker/.profile && echo 'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers +USER ducker + +CMD sudo service ssh start && tail -f /dev/null diff --git a/tests/docker/build/cluster.json b/tests/docker/build/cluster.json new file mode 100644 index 0000000000000..3597cac944a2f --- /dev/null +++ b/tests/docker/build/cluster.json @@ -0,0 +1,163 @@ +{ + "_comment": [ + "Licensed to the Apache Software Foundation (ASF) under one or more", + "contributor license agreements. See the NOTICE file distributed with", + "this work for additional information regarding copyright ownership.", + "The ASF licenses this file to You under the Apache License, Version 2.0", + "(the \"License\"); you may not use this file except in compliance with", + "the License. You may obtain a copy of the License at", + "", + "http://www.apache.org/licenses/LICENSE-2.0", + "", + "Unless required by applicable law or agreed to in writing, software", + "distributed under the License is distributed on an \"AS IS\" BASIS,", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", + "See the License for the specific language governing permissions and", + "limitations under the License." + ], + "nodes": [ + { + "externally_routable_ip": "ducker02", + "ssh_config": { + "host": "ducker02", + "hostname": "ducker02", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker03", + "ssh_config": { + "host": "ducker03", + "hostname": "ducker03", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker04", + "ssh_config": { + "host": "ducker04", + "hostname": "ducker04", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker05", + "ssh_config": { + "host": "ducker05", + "hostname": "ducker05", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker06", + "ssh_config": { + "host": "ducker06", + "hostname": "ducker06", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker07", + "ssh_config": { + "host": "ducker07", + "hostname": "ducker07", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker08", + "ssh_config": { + "host": "ducker08", + "hostname": "ducker08", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker09", + "ssh_config": { + "host": "ducker09", + "hostname": "ducker09", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker10", + "ssh_config": { + "host": "ducker10", + "hostname": "ducker10", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker11", + "ssh_config": { + "host": "ducker11", + "hostname": "ducker11", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker12", + "ssh_config": { + "host": "ducker12", + "hostname": "ducker12", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker13", + "ssh_config": { + "host": "ducker13", + "hostname": "ducker13", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker14", + "ssh_config": { + "host": "ducker14", + "hostname": "ducker14", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + } + ] +} diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts new file mode 100644 index 0000000000000..e0aba2434ae2f --- /dev/null +++ b/tests/docker/build/node_hosts @@ -0,0 +1,14 @@ +172.19.0.2 ducker01 +172.19.0.3 ducker02 +172.19.0.4 ducker03 +172.19.0.5 ducker04 +172.19.0.6 ducker05 +172.19.0.7 ducker06 +172.19.0.8 ducker07 +172.19.0.9 ducker08 +172.19.0.10 ducker09 +172.19.0.11 ducker10 +172.19.0.12 ducker11 +172.19.0.13 ducker12 +172.19.0.14 ducker13 +172.19.0.15 ducker14 diff --git a/tests/docker/ducker-ignite b/tests/docker/ducker-ignite new file mode 100755 index 0000000000000..6ea13a0a8188c --- /dev/null +++ b/tests/docker/ducker-ignite @@ -0,0 +1,580 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Ducker-Ignite: a tool for running Apache Ignite system tests inside Docker images. +# +# Note: this should be compatible with the version of bash that ships on most +# Macs, bash 3.2.57. +# + +script_path="${0}" + +# The absolute path to the directory which this script is in. This will also be the directory +# which we run docker build from. +ducker_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# The absolute path to the root Ignite directory +ignite_dir="$( cd "${ducker_dir}/../.." && pwd )" + +# The memory consumption to allow during the docker build. +# This does not include swap. +docker_build_memory_limit="3200m" + +# The maximum mmemory consumption to allow in containers. +docker_run_memory_limit="2000m" + +# The default number of cluster nodes to bring up if a number is not specified. +default_num_nodes=14 + +# The default OpenJDK base image. +default_jdk="openjdk:8" + +# The default ducker-ignite image name. +default_image_name="ducker-ak" + +# Display a usage message on the terminal and exit. +# +# $1: The exit status to use +usage() { + local exit_status="${1}" + cat < /dev/null || die "You must install ${cmd} to run this script." + done +} + +# Set a global variable to a value. +# +# $1: The variable name to set. This function will die if the variable already has a value. The +# variable will be made readonly to prevent any future modifications. +# $2: The value to set the variable to. This function will die if the value is empty or starts +# with a dash. +# $3: A human-readable description of the variable. +set_once() { + local key="${1}" + local value="${2}" + local what="${3}" + [[ -n "${!key}" ]] && die "Error: more than one value specified for ${what}." + verify_command_line_argument "${value}" "${what}" + # It would be better to use declare -g, but older bash versions don't support it. + export ${key}="${value}" +} + +# Verify that a command-line argument is present and does not start with a slash. +# +# $1: The command-line argument to verify. +# $2: A human-readable description of the variable. +verify_command_line_argument() { + local value="${1}" + local what="${2}" + [[ -n "${value}" ]] || die "Error: no value specified for ${what}" + [[ ${value} == -* ]] && die "Error: invalid value ${value} specified for ${what}" +} + +# Echo a message if a flag is set. +# +# $1: If this is 1, the message will be echoed. +# $@: The message +maybe_echo() { + local verbose="${1}" + shift + [[ "${verbose}" -eq 1 ]] && echo "${@}" +} + +# Counts the number of elements passed to this subroutine. +count() { + echo $# +} + +# Push a new directory on to the bash directory stack, or exit with a failure message. +# +# $1: The directory push on to the directory stack. +must_pushd() { + local target_dir="${1}" + pushd -- "${target_dir}" &> /dev/null || die "failed to change directory to ${target_dir}" +} + +# Pop a directory from the bash directory stack, or exit with a failure message. +must_popd() { + popd &> /dev/null || die "failed to popd" +} + +# Run a command and die if it fails. +# +# Optional flags: +# -v: print the command before running it. +# -o: display the command output. +# $@: The command to run. +must_do() { + local verbose=0 + local output="/dev/null" + while true; do + case ${1} in + -v) verbose=1; shift;; + -o) output="/dev/stdout"; shift;; + *) break;; + esac + done + local cmd="${@}" + [[ "${verbose}" -eq 1 ]] && echo "${cmd}" + ${cmd} >${output} || die "${1} failed" +} + +# Ask the user a yes/no question. +# +# $1: The prompt to use +# $_return: 0 if the user answered no; 1 if the user anМинус - создаст дополнительную нагрузку на core team.swered yes. +ask_yes_no() { + local prompt="${1}" + while true; do + read -r -p "${prompt} " response + case "${response}" in + [yY]|[yY][eE][sS]) _return=1; return;; + [nN]|[nN][oO]) _return=0; return;; + *);; + esac + echo "Please respond 'yes' or 'no'." + echo + done +} + +# Build a docker image. +# +# $1: The name of the image to build. +ducker_build() { + local image_name="${1}" + + # Use SECONDS, a builtin bash variable that gets incremented each second, to measure the docker + # build duration. + SECONDS=0 + + must_pushd "${ducker_dir}" + # Tip: if you are scratching your head for some dependency problems that are referring to an old code version + # (for example java.lang.NoClassDefFoundError), add --no-cache flag to the build shall give you a clean start. + must_do -v -o docker build --memory="${docker_build_memory_limit}" \ + --build-arg "ducker_creator=${user_name}" --build-arg "jdk_version=${jdk_version}" -t "${image_name}" \ + -f "${ducker_dir}/Dockerfile" ${docker_args} -- . + docker_status=$? + must_popd + duration="${SECONDS}" + if [[ ${docker_status} -ne 0 ]]; then + die "** ERROR: Failed to build ${what} image after $((${duration} / 60))m \ +$((${duration} % 60))s. See ${build_log} for details." + fi + echo "** Successfully built ${what} image in $((${duration} / 60))m \ +$((${duration} % 60))s. See ${build_log} for details." +} + +docker_run() { + local node=${1} + local image_name=${2} + local ports_option=${3} + + local expose_ports="" + if [[ -n ${ports_option} ]]; then + expose_ports="-P" + for expose_port in ${ports_option//,/ }; do + expose_ports="${expose_ports} --expose ${expose_port}" + done + fi + + # Invoke docker-run. We need privileged mode to be able to run iptables + # and mount FUSE filesystems inside the container. We also need it to + # run iptables inside the container. + must_do -v docker run --privileged \ + -d -t -h "${node}" --network ducknet "${expose_ports}" \ + --memory=${docker_run_memory_limit} --memory-swappiness=1 \ + -v "${ignite_dir}:/opt/ignite-dev" --name "${node}" -- "${image_name}" +} + +setup_custom_ducktape() { + local custom_ducktape="${1}" + local image_name="${2}" + + [[ -f "${custom_ducktape}/ducktape/__init__.py" ]] || \ + die "You must supply a valid ducktape directory to --custom-ducktape" + docker_run ducker01 "${image_name}" + local running_container="$(docker ps -f=network=ducknet -q)" + must_do -v -o docker cp "${custom_ducktape}" "${running_container}:/opt/ducktape" + docker exec --user=root ducker01 bash -c 'set -x && cd /opt/ignite-dev/tests && sudo python ./setup.py develop install && cd /opt/ducktape && sudo python ./setup.py develop install' + [[ $? -ne 0 ]] && die "failed to install the new ducktape." + must_do -v -o docker commit ducker01 "${image_name}" + must_do -v docker kill "${running_container}" + must_do -v docker rm ducker01 +} + +ducker_up() { + require_commands docker + while [[ $# -ge 1 ]]; do + case "${1}" in + -C|--custom-ducktape) set_once custom_ducktape "${2}" "the custom ducktape directory"; shift 2;; + -f|--force) force=1; shift;; + -n|--num-nodes) set_once num_nodes "${2}" "number of nodes"; shift 2;; + -j|--jdk) set_once jdk_version "${2}" "the OpenJDK base image"; shift 2;; + -e|--expose-ports) set_once expose_ports "${2}" "the ports to expose"; shift 2;; + *) set_once image_name "${1}" "docker image name"; shift;; + esac + done + [[ -n "${num_nodes}" ]] || num_nodes="${default_num_nodes}" + [[ -n "${jdk_version}" ]] || jdk_version="${default_jdk}" + [[ -n "${image_name}" ]] || image_name="${default_image_name}-${jdk_version/:/-}" + [[ "${num_nodes}" =~ ^-?[0-9]+$ ]] || \ + die "ducker_up: the number of nodes must be an integer." + [[ "${num_nodes}" -gt 0 ]] || die "ducker_up: the number of nodes must be greater than 0." + if [[ "${num_nodes}" -lt 2 ]]; then + if [[ "${force}" -ne 1 ]]; then + echo "ducker_up: It is recommended to run at least 2 nodes, since ducker01 is only \ +used to run ducktape itself. If you want to do it anyway, you can use --force to attempt to \ +use only ${num_nodes}." + exit 1 + fi + fi + + docker ps >/dev/null || die "ducker_up: failed to run docker. Please check that the daemon is started." + + ducker_build "${image_name}" + + docker inspect --format='{{.Config.Labels}}' --type=image "${image_name}" | grep -q 'ducker.type' + local docker_status=${PIPESTATUS[0]} + local grep_status=${PIPESTATUS[1]} + [[ "${docker_status}" -eq 0 ]] || die "ducker_up: failed to inspect image ${image_name}. \ +Please check that it exists." + if [[ "${grep_status}" -ne 0 ]]; then + if [[ "${force}" -ne 1 ]]; then + echo "ducker_up: ${image_name} does not appear to be a ducker image. It lacks the \ +ducker.type label. If you think this is a mistake, you can use --force to attempt to bring \ +it up anyway." + exit 1 + fi + fi + local running_containers="$(docker ps -f=network=ducknet -q)" + local num_running_containers=$(count ${running_containers}) + if [[ ${num_running_containers} -gt 0 ]]; then + die "ducker_up: there are ${num_running_containers} ducker containers \ +running already. Use ducker down to bring down these containers before \ +attempting to start new ones." + fi + + echo "ducker_up: Bringing up ${image_name} with ${num_nodes} nodes..." + if docker network inspect ducknet &>/dev/null; then + must_do -v docker network rm ducknet + fi + must_do -v docker network create ducknet + if [[ -n "${custom_ducktape}" ]]; then + setup_custom_ducktape "${custom_ducktape}" "${image_name}" + fi + for n in $(seq -f %02g 1 ${num_nodes}); do + local node="ducker${n}" + docker_run "${node}" "${image_name}" "${expose_ports}" + done + mkdir -p "${ducker_dir}/build" + exec 3<> "${ducker_dir}/build/node_hosts" + for n in $(seq -f %02g 1 ${num_nodes}); do + local node="ducker${n}" + docker exec --user=root "${node}" grep "${node}" /etc/hosts >&3 + [[ $? -ne 0 ]] && die "failed to find the /etc/hosts entry for ${node}" + done + exec 3>&- + for n in $(seq -f %02g 1 ${num_nodes}); do + local node="ducker${n}" + docker exec --user=root "${node}" \ + bash -c "grep -v ${node} /opt/ignite-dev/tests/docker/build/node_hosts >> /etc/hosts" + [[ $? -ne 0 ]] && die "failed to append to the /etc/hosts file on ${node}" + done + + echo "ducker_up: added the latest entries to /etc/hosts on each node." + generate_cluster_json_file "${num_nodes}" "${ducker_dir}/build/cluster.json" + echo "ducker_up: successfully wrote ${ducker_dir}/build/cluster.json" + echo "** ducker_up: successfully brought up ${num_nodes} nodes." +} + +# Generate the cluster.json file used by ducktape to identify cluster nodes. +# +# $1: The number of cluster nodes. +# $2: The path to write the cluster.json file to. +generate_cluster_json_file() { + local num_nodes="${1}" + local path="${2}" + exec 3<> "${path}" +cat<&3 +{ + "_comment": [ + "Licensed to the Apache Software Foundation (ASF) under one or more", + "contributor license agreements. See the NOTICE file distributed with", + "this work for additional information regarding copyright ownership.", + "The ASF licenses this file to You under the Apache License, Version 2.0", + "(the \"License\"); you may not use this file except in compliance with", + "the License. You may obtain a copy of the License at", + "", + "http://www.apache.org/licenses/LICENSE-2.0", + "", + "Unless required by applicable law or agreed to in writing, software", + "distributed under the License is distributed on an \"AS IS\" BASIS,", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", + "See the License for the specific language governing permissions and", + "limitations under the License." + ], + "nodes": [ +EOF + for n in $(seq 2 ${num_nodes}); do + if [[ ${n} -eq ${num_nodes} ]]; then + suffix="" + else + suffix="," + fi + local node=$(printf ducker%02d ${n}) +cat<&3 + { + "externally_routable_ip": "${node}", + "ssh_config": { + "host": "${node}", + "hostname": "${node}", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }${suffix} +EOF + done +cat<&3 + ] +} +EOF + exec 3>&- +} + +ducker_test() { + require_commands docker + docker inspect ducker01 &>/dev/null || \ + die "ducker_test: the ducker01 instance appears to be down. Did you run 'ducker up'?" + [[ $# -lt 1 ]] && \ + die "ducker_test: you must supply at least one system test to run. Type --help for help." + local args="" + local ignite_test=0 + for arg in "${@}"; do + local regex=".*\/ignitetest\/(.*)" + if [[ $arg =~ $regex ]]; then + local kpath=${BASH_REMATCH[1]} + args="${args} ./tests/ignitetest/${kpath}" + else + args="${args} ${arg}" + fi + done + must_pushd "${ignite_dir}" + (test -f ./gradlew || gradle) && ./gradlew systemTestLibs + must_popd + cmd="cd /opt/ignite-dev && ducktape --cluster-file /opt/ignite-dev/tests/docker/build/cluster.json $args" + echo "docker exec ducker01 bash -c \"${cmd}\"" + exec docker exec --user=ducker ducker01 bash -c "${cmd}" +} + +ducker_ssh() { + require_commands docker + [[ $# -eq 0 ]] && die "ducker_ssh: Please specify a container name to log into. \ +Currently active containers: $(echo_running_container_names)" + local node_info="${1}" + shift + local guest_command="$*" + local user_name="ducker" + if [[ "${node_info}" =~ @ ]]; then + user_name="${node_info%%@*}" + local node_name="${node_info##*@}" + else + local node_name="${node_info}" + fi + local docker_flags="" + if [[ -z "${guest_command}" ]]; then + local docker_flags="${docker_flags} -t" + local guest_command_prefix="" + guest_command=bash + else + local guest_command_prefix="bash -c" + fi + if [[ "${node_name}" == "all" ]]; then + local nodes=$(echo_running_container_names) + [[ "${nodes}" == "(none)" ]] && die "ducker_ssh: can't locate any running ducker nodes." + for node in ${nodes}; do + docker exec --user=${user_name} -i ${docker_flags} "${node}" \ + ${guest_command_prefix} "${guest_command}" || die "docker exec ${node} failed" + done + else + docker inspect --type=container -- "${node_name}" &>/dev/null || \ + die "ducker_ssh: can't locate node ${node_name}. Currently running nodes: \ +$(echo_running_container_names)" + exec docker exec --user=${user_name} -i ${docker_flags} "${node_name}" \ + ${guest_command_prefix} "${guest_command}" + fi +} + +# Echo all the running Ducker container names, or (none) if there are no running Ducker containers. +echo_running_container_names() { + node_names="$(docker ps -f=network=ducknet -q --format '{{.Names}}' | sort)" + if [[ -z "${node_names}" ]]; then + echo "(none)" + else + echo ${node_names//$'\n'/ } + fi +} + +ducker_down() { + require_commands docker + local verbose=1 + local force_str="" + while [[ $# -ge 1 ]]; do + case "${1}" in + -q|--quiet) verbose=0; shift;; + -f|--force) force_str="-f"; shift;; + *) die "ducker_down: unexpected command-line argument ${1}";; + esac + done + local running_containers + running_containers="$(docker ps -f=network=ducknet -q)" + [[ $? -eq 0 ]] || die "ducker_down: docker command failed. Is the docker daemon running?" + running_containers=${running_containers//$'\n'/ } + local all_containers="$(docker ps -a -f=network=ducknet -q)" + all_containers=${all_containers//$'\n'/ } + if [[ -z "${all_containers}" ]]; then + maybe_echo "${verbose}" "No ducker containers found." + return + fi + verbose_flag="" + if [[ ${verbose} == 1 ]]; then + verbose_flag="-v" + fi + if [[ -n "${running_containers}" ]]; then + must_do ${verbose_flag} docker kill "${running_containers}" + fi + must_do ${verbose_flag} docker rm ${force_str} "${all_containers}" + must_do ${verbose_flag} -o rm -f -- "${ducker_dir}/build/node_hosts" "${ducker_dir}/build/cluster.json" + if docker network inspect ducknet &>/dev/null; then + must_do -v docker network rm ducknet + fi + maybe_echo "${verbose}" "ducker_down: removed $(count ${all_containers}) containers." +} + +ducker_purge() { + require_commands docker + local force_str="" + while [[ $# -ge 1 ]]; do + case "${1}" in + -f|--force) force_str="-f"; shift;; + *) die "ducker_purge: unknown argument ${1}";; + esac + done + echo "** ducker_purge: attempting to locate ducker images to purge" + local images + images=$(docker images -q -a -f label=ducker.creator) + [[ $? -ne 0 ]] && die "docker images command failed" + images=${images//$'\n'/ } + declare -a purge_images=() + if [[ -z "${images}" ]]; then + echo "** ducker_purge: no images found to purge." + exit 0 + fi + echo "** ducker_purge: images to delete:" + for image in ${images}; do + echo -n "${image} " + docker inspect --format='{{.Config.Labels}} {{.Created}}' --type=image "${image}" + [[ $? -ne 0 ]] && die "docker inspect ${image} failed" + done + ask_yes_no "Delete these docker images? [y/n]" + [[ "${_return}" -eq 0 ]] && exit 0 + must_do -v -o docker rmi ${force_str} ${images} +} + +# Parse command-line arguments +[[ $# -lt 1 ]] && usage 0 +# Display the help text if -h or --help appears in the command line +for arg in ${@}; do + case "${arg}" in + -h|--help) usage 0;; + --) break;; + *);; + esac +done +action="${1}" +shift +case "${action}" in + help) usage 0;; + + up|test|ssh|down|purge) + ducker_${action} "${@}"; exit 0;; + + *) echo "Unknown command '${action}'. Type '${script_path} --help' for usage information." + exit 1;; +esac diff --git a/tests/docker/run_tests.sh b/tests/docker/run_tests.sh new file mode 100755 index 0000000000000..09c67976e9896 --- /dev/null +++ b/tests/docker/run_tests.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-14} +TC_PATHS=${TC_PATHS:-./ignitetest/} + +die() { + echo $@ + exit 1 +} + +if ${SCRIPT_DIR}/ducker-ignite ssh | grep -q '(none)'; then + ${SCRIPT_DIR}/ducker-ignite up -n "${IGNITE_NUM_CONTAINERS}" || die "ducker-ak up failed" +fi +${SCRIPT_DIR}/ducker-ignite test ${TC_PATHS} ${_DUCKTAPE_OPTIONS} || die "ducker-ak test failed" diff --git a/tests/docker/ssh-config b/tests/docker/ssh-config new file mode 100644 index 0000000000000..1f874178074ed --- /dev/null +++ b/tests/docker/ssh-config @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Host * + ControlMaster auto + ControlPath ~/.ssh/master-%r@%h:%p + StrictHostKeyChecking no + ConnectTimeout=10 + IdentityFile ~/.ssh/id_rsa diff --git a/tests/docker/ssh/authorized_keys b/tests/docker/ssh/authorized_keys new file mode 100644 index 0000000000000..9f9da1f7bfe9d --- /dev/null +++ b/tests/docker/ssh/authorized_keys @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC0qDT9kEPWc8JQ53b4KnT/ZJOLwb+3c//jpLW/2ofjDyIsPW4FohLpicfouch/zsRpN4G38lua+2BsGls9sMIZc6PXY2L+NIGCkqEMdCoU1Ym8SMtyJklfzp3m/0PeK9s2dLlR3PFRYvyFA4btQK5hkbYDNZPzf4airvzdRzLkrFf81+RemaMI2EtONwJRcbLViPaTXVKJdbFwJTJ1u7yu9wDYWHKBMA92mHTQeP6bhVYCqxJn3to/RfZYd+sHw6mfxVg5OrAlUOYpSV4pDNCAsIHdtZ56V8NQlJL6NJ2vzzSSYUwLMqe88fhrC8yYHoxC07QPy1EdkSTHdohAicyT root@knode01.knw diff --git a/tests/docker/ssh/config b/tests/docker/ssh/config new file mode 100644 index 0000000000000..1f874178074ed --- /dev/null +++ b/tests/docker/ssh/config @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Host * + ControlMaster auto + ControlPath ~/.ssh/master-%r@%h:%p + StrictHostKeyChecking no + ConnectTimeout=10 + IdentityFile ~/.ssh/id_rsa diff --git a/tests/docker/ssh/id_rsa b/tests/docker/ssh/id_rsa new file mode 100644 index 0000000000000..276e07b9eb0b5 --- /dev/null +++ b/tests/docker/ssh/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAtKg0/ZBD1nPCUOd2+Cp0/2STi8G/t3P/46S1v9qH4w8iLD1u +BaIS6YnH6LnIf87EaTeBt/JbmvtgbBpbPbDCGXOj12Ni/jSBgpKhDHQqFNWJvEjL +ciZJX86d5v9D3ivbNnS5UdzxUWL8hQOG7UCuYZG2AzWT83+Goq783Ucy5KxX/Nfk +XpmjCNhLTjcCUXGy1Yj2k11SiXWxcCUydbu8rvcA2FhygTAPdph00Hj+m4VWAqsS +Z97aP0X2WHfrB8Opn8VYOTqwJVDmKUleKQzQgLCB3bWeelfDUJSS+jSdr880kmFM +CzKnvPH4awvMmB6MQtO0D8tRHZEkx3aIQInMkwIDAQABAoIBAQCz6EMFNNLp0NP1 +X9yRXS6wW4e4CRWUazesiw3YZpcmnp6IchCMGZA99FEZyVILPW1J3tYWyotBdw7Z ++RFeCRXy5L+IMtiVkNJcpwss7M4ve0w0LkY0gj5V49xJ+3Gp4gDnZSxcguvrAem5 +yP5obR572fDpl0SknB4HCr6U2l+rauzrLyevy5eeDT/vmXbuM1cdHpNIXmmElz4L +t31n+exQRn6tP1h516iXbcYbopxDgdv2qKGAqzWKE6TyWpzF5x7kjOEYt0bZ5QO3 +Lwh7AAqE/3mwxlYwng1L4WAT7RtcP19W+9JDIc7ENInMGxq6q46p1S3IPZsf1cj/ +aAJ9q3LBAoGBAOVJr0+WkR786n3BuswpGQWBgVxfai4y9Lf90vuGKawdQUzXv0/c +EB/CFqP/dIsquukA8PfzjNMyTNmEHXi4Sf16H8Rg4EGhIYMEqIQojx1t/yLLm0aU +YPEvW/02Umtlg3pJw9fQAAzFVqCasw2E2lUdAUkydGRwDUJZmv2/b3NzAoGBAMm0 +Jo7Et7ochH8Vku6uA+hG+RdwlKFm5JA7/Ci3DOdQ1zmJNrvBBFQLo7AjA4iSCoBd +s9+y0nrSPcF4pM3l6ghLheaqbnIi2HqIMH9mjDbrOZiWvbnjvjpOketgNX8vV3Ye +GUkSjoNcmvRmdsICmUjeML8bGOmq4zF9W/GIfTphAoGBAKGRo8R8f/SLGh3VtvCI +gUY89NAHuEWnyIQii1qMNq8+yjYAzaHTm1UVqmiT6SbrzFvGOwcuCu0Dw91+2Fmp +2xGPzfTOoxf8GCY/0ROXlQmS6jc1rEw24Hzz92ldrwRYuyYf9q4Ltw1IvXtcp5F+ +LW/OiYpv0E66Gs3HYI0wKbP7AoGBAJMZWeFW37LQJ2TTJAQDToAwemq4xPxsoJX7 +2SsMTFHKKBwi0JLe8jwk/OxwrJwF/bieHZcvv8ao2zbkuDQcz6/a/D074C5G8V9z +QQM4k1td8vQwQw91Yv782/gvgvRNX1iaHNCowtxURgGlVEirQoTc3eoRZfrLkMM/ +7DTa2JEhAoGACEu3zHJ1sgyeOEgLArUJXlQM30A/ulMrnCd4MEyIE+ReyWAUevUQ +0lYdVNva0/W4C5e2lUOJL41jjIPLqI7tcFR2PZE6n0xTTkxNH5W2u1WpFeKjx+O3 +czv7Bt6wYyLHIMy1JEqAQ7pw1mtJ5s76UDvXUhciF+DU2pWYc6APKR0= +-----END RSA PRIVATE KEY----- diff --git a/tests/docker/ssh/id_rsa.pub b/tests/docker/ssh/id_rsa.pub new file mode 100644 index 0000000000000..76e8f5fdf64df --- /dev/null +++ b/tests/docker/ssh/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC0qDT9kEPWc8JQ53b4KnT/ZJOLwb+3c//jpLW/2ofjDyIsPW4FohLpicfouch/zsRpN4G38lua+2BsGls9sMIZc6PXY2L+NIGCkqEMdCoU1Ym8SMtyJklfzp3m/0PeK9s2dLlR3PFRYvyFA4btQK5hkbYDNZPzf4airvzdRzLkrFf81+RemaMI2EtONwJRcbLViPaTXVKJdbFwJTJ1u7yu9wDYWHKBMA92mHTQeP6bhVYCqxJn3to/RfZYd+sHw6mfxVg5OrAlUOYpSV4pDNCAsIHdtZ56V8NQlJL6NJ2vzzSSYUwLMqe88fhrC8yYHoxC07QPy1EdkSTHdohAicyT root@knode01.knw diff --git a/tests/ignitetest/__init__.py b/tests/ignitetest/__init__.py new file mode 100644 index 0000000000000..c5862fe4ccf08 --- /dev/null +++ b/tests/ignitetest/__init__.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This determines the version of kafkatest that can be published to PyPi and installed with pip +# +# Note that in development, this version name can't follow Kafka's convention of having a trailing "-SNAPSHOT" +# due to python version naming restrictions, which are enforced by python packaging tools +# (see https://www.python.org/dev/peps/pep-0440/) +# +# Instead, in development branches, the version should have a suffix of the form ".devN" +# +# For example, when Ignite is at version 2.9.0-SNAPSHOT, this should be something like "2.9.0.dev0" +__version__ = '2.9.0.dev0' diff --git a/tests/ignitetest/utils/__init__.py b/tests/ignitetest/utils/__init__.py new file mode 100644 index 0000000000000..d753d8ef5ed3d --- /dev/null +++ b/tests/ignitetest/utils/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from util import ignitetest_version, is_int, is_int_with_prefix, node_is_reachable, validate_delivery diff --git a/tests/ignitetest/utils/remote_account.py b/tests/ignitetest/utils/remote_account.py new file mode 100644 index 0000000000000..e838a96090c82 --- /dev/null +++ b/tests/ignitetest/utils/remote_account.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def file_exists(node, file): + """Quick and dirty check for existence of remote file.""" + try: + node.account.ssh("cat " + file, allow_fail=False) + return True + except: + return False + + +def path_exists(node, path): + """Quick and dirty check for existence of remote path.""" + try: + node.account.ssh("ls " + path, allow_fail=False) + return True + except: + return False + + +def line_count(node, file): + """Return the line count of file on node""" + out = [line for line in node.account.ssh_capture("wc -l %s" % file)] + if len(out) != 1: + raise Exception("Expected single line of output from wc -l") + + return int(out[0].strip().split(" ")[0]) diff --git a/tests/ignitetest/utils/util.py b/tests/ignitetest/utils/util.py new file mode 100644 index 0000000000000..0bb22eba2161b --- /dev/null +++ b/tests/ignitetest/utils/util.py @@ -0,0 +1,138 @@ +# Copyright 2015 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ignitetest import __version__ as __ignitetest_version__ + +import math +import re +import time + + +def ignitetest_version(): + """Return string representation of current ducktape version.""" + return __ignitetest_version__ + + +def is_int(msg): + """Method used to check whether the given message is an integer + + return int or raises an exception if message is not an integer + """ + try: + return int(msg) + except ValueError: + raise Exception("Unexpected message format (expected an integer). Message: %s" % (msg)) + + +def is_int_with_prefix(msg): + """ + Method used check whether the given message is of format 'integer_prefix'.'integer_value' + + :param msg: message to validate + :return: msg or raises an exception is a message is of wrong format + """ + try: + parts = msg.split(".") + if len(parts) != 2: + raise Exception("Unexpected message format. Message should be of format: integer " + "prefix dot integer value. Message: %s" % (msg)) + int(parts[0]) + int(parts[1]) + return msg + except ValueError: + raise Exception("Unexpected message format. Message should be of format: integer " + "prefix dot integer value, but one of the two parts (before or after dot) " + "are not integers. Message: %s" % (msg)) + + +def node_is_reachable(src_node, dst_node): + """ + Returns true if a node is unreachable from another node. + + :param src_node: The source node to check from reachability from. + :param dst_node: The destination node to check for reachability to. + :return: True only if dst is reachable from src. + """ + return 0 == src_node.account.ssh("nc -w 3 -z %s 22" % dst_node.account.hostname, allow_fail=True) + + +def annotate_missing_msgs(missing, acked, consumed, msg): + missing_list = list(missing) + msg += "%s acked message did not make it to the Consumer. They are: " %\ + len(missing_list) + if len(missing_list) < 20: + msg += str(missing_list) + ". " + else: + msg += ", ".join(str(m) for m in missing_list[:20]) + msg += "...plus %s more. Total Acked: %s, Total Consumed: %s. " \ + % (len(missing_list) - 20, len(set(acked)), len(set(consumed))) + return msg + + +def annotate_data_lost(data_lost, msg, number_validated): + print_limit = 10 + if len(data_lost) > 0: + msg += "The first %s missing messages were validated to ensure they are in Kafka's data files. " \ + "%s were missing. This suggests data loss. Here are some of the messages not found in the data files: %s\n" \ + % (number_validated, len(data_lost), str(data_lost[0:print_limit]) if len(data_lost) > print_limit else str(data_lost)) + else: + msg += "We validated that the first %s of these missing messages correctly made it into Kafka's data files. " \ + "This suggests they were lost on their way to the consumer." % number_validated + return msg + + +def validate_delivery(acked, consumed, idempotence_enabled=False, check_lost_data=None, may_truncate_acked_records=False): + """Check that each acked message was consumed.""" + success = True + msg = "" + + # Correctness of the set difference operation depends on using equivalent + # message_validators in producer and consumer + missing = set(acked) - set(consumed) + + # Were all acked messages consumed? + if len(missing) > 0: + msg = annotate_missing_msgs(missing, acked, consumed, msg) + + # Did we miss anything due to data loss? + if check_lost_data: + max_truncate_count = 100 if may_truncate_acked_records else 0 + max_validate_count = max(1000, max_truncate_count) + + to_validate = list(missing)[0:min(len(missing), max_validate_count)] + data_lost = check_lost_data(to_validate) + + # With older versions of message format before KIP-101, data loss could occur due to truncation. + # These records won't be in the data logs. Tolerate limited data loss for this case. + if len(missing) < max_truncate_count and len(data_lost) == len(missing): + msg += "The %s missing messages were not present in Kafka's data files. This suggests data loss " \ + "due to truncation, which is possible with older message formats and hence are ignored " \ + "by this test. The messages lost: %s\n" % (len(data_lost), str(data_lost)) + else: + msg = annotate_data_lost(data_lost, msg, len(to_validate)) + success = False + else: + success = False + + # Are there duplicates? + if len(set(consumed)) != len(consumed): + num_duplicates = abs(len(set(consumed)) - len(consumed)) + + if idempotence_enabled: + success = False + msg += "Detected %d duplicates even though idempotence was enabled.\n" % num_duplicates + else: + msg += "(There are also %d duplicate messages in the log - but that is an acceptable outcome)\n" % num_duplicates + + return success, msg diff --git a/tests/ignitetest/version.py b/tests/ignitetest/version.py new file mode 100644 index 0000000000000..9509909206c6a --- /dev/null +++ b/tests/ignitetest/version.py @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from distutils.version import LooseVersion +from ignitetest.utils import ignitetest_version + + +class KafkaVersion(LooseVersion): + """Container for kafka versions which makes versions simple to compare. + + distutils.version.LooseVersion (and StrictVersion) has robust comparison and ordering logic. + + Example: + + v10 = KafkaVersion("0.10.0") + v9 = KafkaVersion("0.9.0.1") + assert v10 > v9 # assertion passes! + """ + def __init__(self, version_string): + self.is_dev = (version_string.lower() == "dev") + if self.is_dev: + version_string = kafkatest_version() + + # Drop dev suffix if present + dev_suffix_index = version_string.find(".dev") + if dev_suffix_index >= 0: + version_string = version_string[:dev_suffix_index] + + # Don't use the form super.(...).__init__(...) because + # LooseVersion is an "old style" python class + LooseVersion.__init__(self, version_string) + + def __str__(self): + if self.is_dev: + return "dev" + else: + return LooseVersion.__str__(self) + + def supports_named_listeners(self): + return self >= V_0_10_2_0 + + def topic_command_supports_bootstrap_server(self): + return self >= V_2_3_0 + +def get_version(node=None): + """Return the version attached to the given node. + Default to DEV_BRANCH if node or node.version is undefined (aka None) + """ + if node is not None and hasattr(node, "version") and node.version is not None: + return node.version + else: + return DEV_BRANCH + +DEV_BRANCH = KafkaVersion("dev") +DEV_VERSION = KafkaVersion("2.6.0-SNAPSHOT") + +# 0.8.2.x versions +V_0_8_2_1 = KafkaVersion("0.8.2.1") +V_0_8_2_2 = KafkaVersion("0.8.2.2") +LATEST_0_8_2 = V_0_8_2_2 + +# 0.9.0.x versions +V_0_9_0_0 = KafkaVersion("0.9.0.0") +V_0_9_0_1 = KafkaVersion("0.9.0.1") +LATEST_0_9 = V_0_9_0_1 + +# 0.10.0.x versions +V_0_10_0_0 = KafkaVersion("0.10.0.0") +V_0_10_0_1 = KafkaVersion("0.10.0.1") +LATEST_0_10_0 = V_0_10_0_1 + +# 0.10.1.x versions +V_0_10_1_0 = KafkaVersion("0.10.1.0") +V_0_10_1_1 = KafkaVersion("0.10.1.1") +LATEST_0_10_1 = V_0_10_1_1 + +# 0.10.2.x versions +V_0_10_2_0 = KafkaVersion("0.10.2.0") +V_0_10_2_1 = KafkaVersion("0.10.2.1") +V_0_10_2_2 = KafkaVersion("0.10.2.2") +LATEST_0_10_2 = V_0_10_2_2 + +LATEST_0_10 = LATEST_0_10_2 + +# 0.11.0.x versions +V_0_11_0_0 = KafkaVersion("0.11.0.0") +V_0_11_0_1 = KafkaVersion("0.11.0.1") +V_0_11_0_2 = KafkaVersion("0.11.0.2") +V_0_11_0_3 = KafkaVersion("0.11.0.3") +LATEST_0_11_0 = V_0_11_0_3 +LATEST_0_11 = LATEST_0_11_0 + +# 1.0.x versions +V_1_0_0 = KafkaVersion("1.0.0") +V_1_0_1 = KafkaVersion("1.0.1") +V_1_0_2 = KafkaVersion("1.0.2") +LATEST_1_0 = V_1_0_2 + +# 1.1.x versions +V_1_1_0 = KafkaVersion("1.1.0") +V_1_1_1 = KafkaVersion("1.1.1") +LATEST_1_1 = V_1_1_1 + +# 2.0.x versions +V_2_0_0 = KafkaVersion("2.0.0") +V_2_0_1 = KafkaVersion("2.0.1") +LATEST_2_0 = V_2_0_1 + +# 2.1.x versions +V_2_1_0 = KafkaVersion("2.1.0") +V_2_1_1 = KafkaVersion("2.1.1") +LATEST_2_1 = V_2_1_1 + +# 2.2.x versions +V_2_2_0 = KafkaVersion("2.2.0") +V_2_2_1 = KafkaVersion("2.2.1") +V_2_2_2 = KafkaVersion("2.2.2") +LATEST_2_2 = V_2_2_2 + +# 2.3.x versions +V_2_3_0 = KafkaVersion("2.3.0") +V_2_3_1 = KafkaVersion("2.3.1") +LATEST_2_3 = V_2_3_1 + +# 2.4.x versions +V_2_4_0 = KafkaVersion("2.4.0") +LATEST_2_4 = V_2_4_0 diff --git a/tests/setup.cfg b/tests/setup.cfg new file mode 100644 index 0000000000000..974d5bb9a972f --- /dev/null +++ b/tests/setup.cfg @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pytest configuration (can also be defined in tox.ini or pytest.ini file) +# +# This file defines naming convention and root search directory for autodiscovery of +# pytest unit tests for the system test service classes. +# +# To ease possible confusion, 'check' instead of 'test' as a prefix for unit tests, since +# many system test files, classes, and methods have 'test' somewhere in the name +[pytest] +testpaths=unit +python_files=check_*.py +python_classes=Check +python_functions=check_* + +# don't search inside any resources directory for unit tests +norecursedirs = resources diff --git a/tests/setup.py b/tests/setup.py new file mode 100644 index 0000000000000..3e477167e1c7e --- /dev/null +++ b/tests/setup.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import sys +from setuptools import find_packages, setup +from setuptools.command.test import test as TestCommand + +version = '' +with open('ignitetest/__init__.py', 'r') as fd: + version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) + + +class PyTest(TestCommand): + user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = [] + + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + # import here, cause outside the eggs aren't loaded + import pytest + print(self.pytest_args) + errno = pytest.main(self.pytest_args) + sys.exit(errno) + +# Note: when changing the version of ducktape, also revise tests/docker/Dockerfile +setup(name="ignitetest", + version=version, + description="Apache Ignite System Tests", + author="Apache Ignite", + platforms=["any"], + license="apache2.0", + packages=find_packages(), + include_package_data=True, + install_requires=["ducktape==0.7.6", "requests==2.20.0"], + tests_require=["pytest", "mock"], + cmdclass={'test': PyTest} +) From c56b8b86df36bba672531ebd6c929dd1a24957cd Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 8 Apr 2020 11:25:32 +0300 Subject: [PATCH 02/78] ignite-ducktape: Initial commit. --- tests/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 0ee09b40132bf..7d639d284d207 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -46,7 +46,7 @@ RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config # we use the same versions as in vagrant/base.sh ARG IGNITE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" ARG ARCHIVE_NAME="apache-ignite-2.8.0-bin.zip" -RUN mkdir -p "/opt/ignite-2.8.0" && chmod a+rw /opt/ignite-2.8.0 && cd /opt/ignite-2.8.0 && curl -s "$IGNITE_MIRROR/ignite/2.8.0/$ARCHIVE_NAME" > /opt/ignite-2.8.0/$ARCHIVE_NAME && unzip /opt/ignite-2.8.0/$ARCHIVE_NAME -d /opt/ignite-2.8.0/ +RUN mkdir -p "/opt/ignite-2.8.0" && chmod a+rw /opt/ignite-2.8.0 && cd /opt/ignite-2.8.0 && curl -s "$IGNITE_MIRROR/ignite/2.8.0/$ARCHIVE_NAME" > /opt/$ARCHIVE_NAME && unzip /opt/$ARCHIVE_NAME # The version of Kibosh to use for testing. # If you update this, also update vagrant/base.sh From 200e760877884761fc56966e2aeb8307dbb0bd7e Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 8 Apr 2020 18:02:21 +0300 Subject: [PATCH 03/78] ignite-ducktape: Initial commit. --- tests/docker/Dockerfile | 5 ++++- tests/docker/build/node_hosts | 28 ++++++++++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 7d639d284d207..d0135e4b11a2c 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -46,7 +46,7 @@ RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config # we use the same versions as in vagrant/base.sh ARG IGNITE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" ARG ARCHIVE_NAME="apache-ignite-2.8.0-bin.zip" -RUN mkdir -p "/opt/ignite-2.8.0" && chmod a+rw /opt/ignite-2.8.0 && cd /opt/ignite-2.8.0 && curl -s "$IGNITE_MIRROR/ignite/2.8.0/$ARCHIVE_NAME" > /opt/$ARCHIVE_NAME && unzip /opt/$ARCHIVE_NAME +RUN mkdir -p "/opt/ignite-2.8.0" && chmod a+rw /opt/ignite-2.8.0 && cd /opt && curl -s "$IGNITE_MIRROR/ignite/2.8.0/$ARCHIVE_NAME" > /opt/$ARCHIVE_NAME && unzip /opt/ # The version of Kibosh to use for testing. # If you update this, also update vagrant/base.sh @@ -61,3 +61,6 @@ RUN useradd -ms /bin/bash ducker && mkdir -p /home/ducker/ && rsync -aiq /root/. USER ducker CMD sudo service ssh start && tail -f /dev/null + +# Container port exposure +EXPOSE 11211 47100 47500 49112 10800 8080 \ No newline at end of file diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts index e0aba2434ae2f..73a0745b64190 100644 --- a/tests/docker/build/node_hosts +++ b/tests/docker/build/node_hosts @@ -1,14 +1,14 @@ -172.19.0.2 ducker01 -172.19.0.3 ducker02 -172.19.0.4 ducker03 -172.19.0.5 ducker04 -172.19.0.6 ducker05 -172.19.0.7 ducker06 -172.19.0.8 ducker07 -172.19.0.9 ducker08 -172.19.0.10 ducker09 -172.19.0.11 ducker10 -172.19.0.12 ducker11 -172.19.0.13 ducker12 -172.19.0.14 ducker13 -172.19.0.15 ducker14 +172.21.0.2 ducker01 +172.21.0.3 ducker02 +172.21.0.4 ducker03 +172.21.0.5 ducker04 +172.21.0.6 ducker05 +172.21.0.7 ducker06 +172.21.0.8 ducker07 +172.21.0.9 ducker08 +172.21.0.10 ducker09 +172.21.0.11 ducker10 +172.21.0.12 ducker11 +172.21.0.13 ducker12 +172.21.0.14 ducker13 +172.21.0.15 ducker14 From d42ff0239c11c097991aace1a70fd22ff6e6829f Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Thu, 9 Apr 2020 14:23:08 +0300 Subject: [PATCH 04/78] ignite-ducktape: WIP. --- tests/docker/Dockerfile | 7 +- tests/docker/build/cluster.json | 6 +- tests/docker/build/node_hosts | 28 ++--- tests/docker/ducker-ignite | 18 +-- tests/docker/run_tests.sh | 6 +- tests/ignitetest/__init__.py | 4 +- .../rebalance/add_node_rebalance_test.py | 38 ++++++ tests/ignitetest/services/__init__.py | 15 +++ tests/ignitetest/services/ignite/__init__.py | 16 +++ tests/ignitetest/services/ignite/ignite.py | 113 ++++++++++++++++++ tests/ignitetest/services/ignite/util.py | 37 ++++++ tests/ignitetest/tests/__init__.py | 14 +++ tests/ignitetest/tests/ignite_test.py | 35 ++++++ tests/ignitetest/version.py | 106 +++------------- 14 files changed, 324 insertions(+), 119 deletions(-) create mode 100644 tests/ignitetest/rebalance/add_node_rebalance_test.py create mode 100644 tests/ignitetest/services/__init__.py create mode 100644 tests/ignitetest/services/ignite/__init__.py create mode 100644 tests/ignitetest/services/ignite/ignite.py create mode 100644 tests/ignitetest/services/ignite/util.py create mode 100644 tests/ignitetest/tests/__init__.py create mode 100644 tests/ignitetest/tests/ignite_test.py diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index d0135e4b11a2c..eb53d8c8149b0 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -45,8 +45,11 @@ RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config # Install binary test dependencies. # we use the same versions as in vagrant/base.sh ARG IGNITE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" -ARG ARCHIVE_NAME="apache-ignite-2.8.0-bin.zip" -RUN mkdir -p "/opt/ignite-2.8.0" && chmod a+rw /opt/ignite-2.8.0 && cd /opt && curl -s "$IGNITE_MIRROR/ignite/2.8.0/$ARCHIVE_NAME" > /opt/$ARCHIVE_NAME && unzip /opt/ +ARG RELEASE_NAME="apache-ignite-2.8.0-bin" +RUN chmod a+wr /opt +RUN curl -s "$IGNITE_MIRROR/ignite/2.8.0/$RELEASE_NAME.zip" > /opt/$RELEASE_NAME.zip +RUN cd /opt && unzip $RELEASE_NAME.zip +RUN chmod a+wr /opt/$RELEASE_NAME -R # The version of Kibosh to use for testing. # If you update this, also update vagrant/base.sh diff --git a/tests/docker/build/cluster.json b/tests/docker/build/cluster.json index 3597cac944a2f..e0de33005298b 100644 --- a/tests/docker/build/cluster.json +++ b/tests/docker/build/cluster.json @@ -37,8 +37,10 @@ "port": 22, "user": "ducker" } - }, - { + } + ] +} + "externally_routable_ip": "ducker04", "ssh_config": { "host": "ducker04", diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts index 73a0745b64190..291afdf1a9e68 100644 --- a/tests/docker/build/node_hosts +++ b/tests/docker/build/node_hosts @@ -1,14 +1,14 @@ -172.21.0.2 ducker01 -172.21.0.3 ducker02 -172.21.0.4 ducker03 -172.21.0.5 ducker04 -172.21.0.6 ducker05 -172.21.0.7 ducker06 -172.21.0.8 ducker07 -172.21.0.9 ducker08 -172.21.0.10 ducker09 -172.21.0.11 ducker10 -172.21.0.12 ducker11 -172.21.0.13 ducker12 -172.21.0.14 ducker13 -172.21.0.15 ducker14 +172.18.0.2 ducker01 +172.18.0.3 ducker02 +172.18.0.4 ducker03 +172.26.0.5 ducker04 +172.26.0.6 ducker05 +172.26.0.7 ducker06 +172.26.0.8 ducker07 +172.26.0.9 ducker08 +172.26.0.10 ducker09 +172.26.0.11 ducker10 +172.26.0.12 ducker11 +172.26.0.13 ducker12 +172.26.0.14 ducker13 +172.26.0.15 ducker14 diff --git a/tests/docker/ducker-ignite b/tests/docker/ducker-ignite index 6ea13a0a8188c..2a90c04d48263 100755 --- a/tests/docker/ducker-ignite +++ b/tests/docker/ducker-ignite @@ -45,7 +45,7 @@ default_num_nodes=14 default_jdk="openjdk:8" # The default ducker-ignite image name. -default_image_name="ducker-ak" +default_image_name="ducker-ignite" # Display a usage message on the terminal and exit. # @@ -53,7 +53,7 @@ default_image_name="ducker-ak" usage() { local exit_status="${1}" cat < 0 + + def start(self): + Service.start(self) + + self.logger.info("Waiting for Ignite to start...") + + def start_cmd(self, node): + cmd = "%s %s 1>> %s 2>> %s &" % \ + (self.path.script("ignite.sh", node), + IgniteService.CONFIG_FILE, + IgniteService.STDOUT_STDERR_CAPTURE, + IgniteService.STDOUT_STDERR_CAPTURE) + return cmd + + def start_node(self, node, timeout_sec=60): + node.account.mkdirs(IgniteService.PERSISTENT_ROOT) + + cmd = self.start_cmd(node) + self.logger.debug("Attempting to start IgniteService on %s with command: %s" % (str(node.account), cmd)) + with node.account.monitor_log(IgniteService.STDOUT_STDERR_CAPTURE) as monitor: + node.account.ssh(cmd) + monitor.wait_until("Ignite\s*Server.*started", timeout_sec=timeout_sec, backoff_sec=.25, + err_msg="Ignite server didn't finish startup in %d seconds" % timeout_sec) + + if len(self.pids(node)) == 0: + raise Exception("No process ids recorded on node %s" % node.account.hostname) + + def pids(self, node): + """Return process ids associated with running processes on the given node.""" + try: + cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.java_class_name() + pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] + return pid_arr + except (RemoteCommandError, ValueError) as e: + return [] + + def stop_node(self, node, clean_shutdown=True, timeout_sec=60): + pids = self.pids(node) + sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL + + for pid in pids: + node.account.signal(pid, sig, allow_fail=False) + + try: + wait_until(lambda: len(self.pids(node)) == 0, timeout_sec=timeout_sec, + err_msg="Ignite node failed to stop in %d seconds" % timeout_sec) + except Exception: + self.thread_dump(node) + raise + + def thread_dump(self, node): + for pid in self.pids(node): + try: + node.account.signal(pid, signal.SIGQUIT, allow_fail=True) + except: + self.logger.warn("Could not dump threads on node") + + def clean_node(self, node): + node.account.kill_java_processes(self.java_class_name(), + clean_shutdown=False, allow_fail=True) + node.account.ssh("sudo rm -rf -- %s" % IgniteService.PERSISTENT_ROOT, allow_fail=False) diff --git a/tests/ignitetest/services/ignite/util.py b/tests/ignitetest/services/ignite/util.py new file mode 100644 index 0000000000000..bbd63e78df726 --- /dev/null +++ b/tests/ignitetest/services/ignite/util.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path + +from collections import namedtuple + +def java_version(node): + # Determine java version on the node + version = -1 + for line in node.account.ssh_capture("java -version"): + if line.find("version") != -1: + version = parse_version_str(line) + return version + +def parse_version_str(line): + # Parse java version string. Examples: + #`openjdk version "11.0.5" 2019-10-15` will return 11. + #`java version "1.5.0"` will return 5. + line = line[line.find('version \"') + 9:] + dot_pos = line.find(".") + if line[:dot_pos] == "1": + return int(line[dot_pos+1:line.find(".", dot_pos+1)]) + else: + return int(line[:dot_pos]) diff --git a/tests/ignitetest/tests/__init__.py b/tests/ignitetest/tests/__init__.py new file mode 100644 index 0000000000000..ec2014340d78f --- /dev/null +++ b/tests/ignitetest/tests/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/ignitetest/tests/ignite_test.py b/tests/ignitetest/tests/ignite_test.py new file mode 100644 index 0000000000000..c30a8fd82d611 --- /dev/null +++ b/tests/ignitetest/tests/ignite_test.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ducktape.tests.test import Test + + +from ignitetest.services.ignite import IgniteService + + +class IgniteTest(Test): + """ + Helper class that manages setting up a Ignite cluster. Use this if the + default settings for Ignite are sufficient for your test; any customization + needs to be done manually. Your run() method should call tearDown and + setUp. The Ignite service are available as the fields IgniteTest.ignite. + """ + def __init__(self, test_context): + super(IgniteTest, self).__init__(test_context) + + self.ignite = IgniteService(test_context, self.num_brokers) + + def setUp(self): + self.ignite.start() \ No newline at end of file diff --git a/tests/ignitetest/version.py b/tests/ignitetest/version.py index 9509909206c6a..31ad73da2124d 100644 --- a/tests/ignitetest/version.py +++ b/tests/ignitetest/version.py @@ -15,24 +15,25 @@ from distutils.version import LooseVersion -from ignitetest.utils import ignitetest_version +from ignitetest import __version__ -class KafkaVersion(LooseVersion): - """Container for kafka versions which makes versions simple to compare. +class IgniteVersion(LooseVersion): + """ + Container for Ignite versions which makes versions simple to compare. distutils.version.LooseVersion (and StrictVersion) has robust comparison and ordering logic. Example: - v10 = KafkaVersion("0.10.0") - v9 = KafkaVersion("0.9.0.1") - assert v10 > v9 # assertion passes! + v27 = IgniteVersion("2.7.0") + v28 = IgniteVersion("2.8.1") + assert v28 > v27 # assertion passes! """ def __init__(self, version_string): self.is_dev = (version_string.lower() == "dev") if self.is_dev: - version_string = kafkatest_version() + version_string = __version__ # Drop dev suffix if present dev_suffix_index = version_string.find(".dev") @@ -49,14 +50,10 @@ def __str__(self): else: return LooseVersion.__str__(self) - def supports_named_listeners(self): - return self >= V_0_10_2_0 - - def topic_command_supports_bootstrap_server(self): - return self >= V_2_3_0 def get_version(node=None): - """Return the version attached to the given node. + """ + Return the version attached to the given node. Default to DEV_BRANCH if node or node.version is undefined (aka None) """ if node is not None and hasattr(node, "version") and node.version is not None: @@ -64,77 +61,12 @@ def get_version(node=None): else: return DEV_BRANCH -DEV_BRANCH = KafkaVersion("dev") -DEV_VERSION = KafkaVersion("2.6.0-SNAPSHOT") - -# 0.8.2.x versions -V_0_8_2_1 = KafkaVersion("0.8.2.1") -V_0_8_2_2 = KafkaVersion("0.8.2.2") -LATEST_0_8_2 = V_0_8_2_2 - -# 0.9.0.x versions -V_0_9_0_0 = KafkaVersion("0.9.0.0") -V_0_9_0_1 = KafkaVersion("0.9.0.1") -LATEST_0_9 = V_0_9_0_1 - -# 0.10.0.x versions -V_0_10_0_0 = KafkaVersion("0.10.0.0") -V_0_10_0_1 = KafkaVersion("0.10.0.1") -LATEST_0_10_0 = V_0_10_0_1 - -# 0.10.1.x versions -V_0_10_1_0 = KafkaVersion("0.10.1.0") -V_0_10_1_1 = KafkaVersion("0.10.1.1") -LATEST_0_10_1 = V_0_10_1_1 - -# 0.10.2.x versions -V_0_10_2_0 = KafkaVersion("0.10.2.0") -V_0_10_2_1 = KafkaVersion("0.10.2.1") -V_0_10_2_2 = KafkaVersion("0.10.2.2") -LATEST_0_10_2 = V_0_10_2_2 - -LATEST_0_10 = LATEST_0_10_2 - -# 0.11.0.x versions -V_0_11_0_0 = KafkaVersion("0.11.0.0") -V_0_11_0_1 = KafkaVersion("0.11.0.1") -V_0_11_0_2 = KafkaVersion("0.11.0.2") -V_0_11_0_3 = KafkaVersion("0.11.0.3") -LATEST_0_11_0 = V_0_11_0_3 -LATEST_0_11 = LATEST_0_11_0 - -# 1.0.x versions -V_1_0_0 = KafkaVersion("1.0.0") -V_1_0_1 = KafkaVersion("1.0.1") -V_1_0_2 = KafkaVersion("1.0.2") -LATEST_1_0 = V_1_0_2 - -# 1.1.x versions -V_1_1_0 = KafkaVersion("1.1.0") -V_1_1_1 = KafkaVersion("1.1.1") -LATEST_1_1 = V_1_1_1 - -# 2.0.x versions -V_2_0_0 = KafkaVersion("2.0.0") -V_2_0_1 = KafkaVersion("2.0.1") -LATEST_2_0 = V_2_0_1 - -# 2.1.x versions -V_2_1_0 = KafkaVersion("2.1.0") -V_2_1_1 = KafkaVersion("2.1.1") -LATEST_2_1 = V_2_1_1 - -# 2.2.x versions -V_2_2_0 = KafkaVersion("2.2.0") -V_2_2_1 = KafkaVersion("2.2.1") -V_2_2_2 = KafkaVersion("2.2.2") -LATEST_2_2 = V_2_2_2 - -# 2.3.x versions -V_2_3_0 = KafkaVersion("2.3.0") -V_2_3_1 = KafkaVersion("2.3.1") -LATEST_2_3 = V_2_3_1 - -# 2.4.x versions -V_2_4_0 = KafkaVersion("2.4.0") -LATEST_2_4 = V_2_4_0 + +DEV_BRANCH = IgniteVersion("dev") +DEV_VERSION = IgniteVersion("2.9.0-SNAPSHOT") + +# 2.7.x versions +V_2_7_6 = IgniteVersion("2.7.6") + +# 2.8.0 versions +V_2_8_0 = IgniteVersion("2.8.0") From f43ac878c357c1540d19302c5ceef47b7cdc7d85 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 10 Apr 2020 10:17:08 +0300 Subject: [PATCH 05/78] ignite-ducktape: WIP. --- tests/docker/Dockerfile | 6 +- tests/docker/build/cluster.json | 116 +----------------- tests/docker/build/node_hosts | 6 +- tests/docker/ducker-ignite | 4 +- tests/docker/run_tests.sh | 2 +- .../{tests => ignite_utils}/__init__.py | 0 .../ignitetest/ignite_utils/ignite_config.py | 36 ++++++ tests/ignitetest/ignite_utils/ignite_path.py | 109 ++++++++++++++++ tests/ignitetest/services/ignite/ignite.py | 21 ++-- .../ignite_test.py => suites/__init__.py} | 21 ---- .../add_node_rebalance_test.py | 14 ++- 11 files changed, 182 insertions(+), 153 deletions(-) rename tests/ignitetest/{tests => ignite_utils}/__init__.py (100%) create mode 100644 tests/ignitetest/ignite_utils/ignite_config.py create mode 100644 tests/ignitetest/ignite_utils/ignite_path.py rename tests/ignitetest/{tests/ignite_test.py => suites/__init__.py} (54%) rename tests/ignitetest/{rebalance => suites}/add_node_rebalance_test.py (78%) diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index eb53d8c8149b0..390d84955b6f6 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -45,11 +45,13 @@ RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config # Install binary test dependencies. # we use the same versions as in vagrant/base.sh ARG IGNITE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" +ARG IGNITE_NAME="ignite-2.8.0" ARG RELEASE_NAME="apache-ignite-2.8.0-bin" RUN chmod a+wr /opt RUN curl -s "$IGNITE_MIRROR/ignite/2.8.0/$RELEASE_NAME.zip" > /opt/$RELEASE_NAME.zip -RUN cd /opt && unzip $RELEASE_NAME.zip -RUN chmod a+wr /opt/$RELEASE_NAME -R +RUN cd /opt && unzip $RELEASE_NAME.zip && rm $RELEASE_NAME.zip +RUN mv /opt/$RELEASE_NAME /opt/$IGNITE_NAME +RUN chmod a+wr /opt/$IGNITE_NAME -R # The version of Kibosh to use for testing. # If you update this, also update vagrant/base.sh diff --git a/tests/docker/build/cluster.json b/tests/docker/build/cluster.json index e0de33005298b..6e5e46ea4850d 100644 --- a/tests/docker/build/cluster.json +++ b/tests/docker/build/cluster.json @@ -37,10 +37,8 @@ "port": 22, "user": "ducker" } - } - ] -} - + }, + { "externally_routable_ip": "ducker04", "ssh_config": { "host": "ducker04", @@ -50,116 +48,6 @@ "port": 22, "user": "ducker" } - }, - { - "externally_routable_ip": "ducker05", - "ssh_config": { - "host": "ducker05", - "hostname": "ducker05", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker06", - "ssh_config": { - "host": "ducker06", - "hostname": "ducker06", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker07", - "ssh_config": { - "host": "ducker07", - "hostname": "ducker07", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker08", - "ssh_config": { - "host": "ducker08", - "hostname": "ducker08", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker09", - "ssh_config": { - "host": "ducker09", - "hostname": "ducker09", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker10", - "ssh_config": { - "host": "ducker10", - "hostname": "ducker10", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker11", - "ssh_config": { - "host": "ducker11", - "hostname": "ducker11", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker12", - "ssh_config": { - "host": "ducker12", - "hostname": "ducker12", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker13", - "ssh_config": { - "host": "ducker13", - "hostname": "ducker13", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker14", - "ssh_config": { - "host": "ducker14", - "hostname": "ducker14", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } } ] } diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts index 291afdf1a9e68..9da72b8fb98bb 100644 --- a/tests/docker/build/node_hosts +++ b/tests/docker/build/node_hosts @@ -1,6 +1,6 @@ -172.18.0.2 ducker01 -172.18.0.3 ducker02 -172.18.0.4 ducker03 +172.26.0.2 ducker01 +172.26.0.3 ducker02 +172.26.0.4 ducker03 172.26.0.5 ducker04 172.26.0.6 ducker05 172.26.0.7 ducker06 diff --git a/tests/docker/ducker-ignite b/tests/docker/ducker-ignite index 2a90c04d48263..c2f2cf8f279c5 100755 --- a/tests/docker/ducker-ignite +++ b/tests/docker/ducker-ignite @@ -39,7 +39,7 @@ docker_build_memory_limit="3200m" docker_run_memory_limit="2000m" # The default number of cluster nodes to bring up if a number is not specified. -default_num_nodes=14 +default_num_nodes=4 # The default OpenJDK base image. default_jdk="openjdk:8" @@ -369,6 +369,8 @@ attempting to start new ones." generate_cluster_json_file() { local num_nodes="${1}" local path="${2}" + rm ${path} + touch ${path} exec 3<> "${path}" cat<&3 { diff --git a/tests/docker/run_tests.sh b/tests/docker/run_tests.sh index 96064bfcd86e8..653b46a6911d2 100755 --- a/tests/docker/run_tests.sh +++ b/tests/docker/run_tests.sh @@ -16,7 +16,7 @@ # limitations under the License. SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-3} +IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-4} TC_PATHS=${TC_PATHS:-./ignitetest/} die() { diff --git a/tests/ignitetest/tests/__init__.py b/tests/ignitetest/ignite_utils/__init__.py similarity index 100% rename from tests/ignitetest/tests/__init__.py rename to tests/ignitetest/ignite_utils/__init__.py diff --git a/tests/ignitetest/ignite_utils/ignite_config.py b/tests/ignitetest/ignite_utils/ignite_config.py new file mode 100644 index 0000000000000..416391ac04611 --- /dev/null +++ b/tests/ignitetest/ignite_utils/ignite_config.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module renders Ignite config and all related artifacts +""" + + +class IgniteConfig: + def __init__(self, project="ignite"): + self.project = project + + def render(self, work_dir): + return """ + + + + + + + """.format(work_dir) diff --git a/tests/ignitetest/ignite_utils/ignite_path.py b/tests/ignitetest/ignite_utils/ignite_path.py new file mode 100644 index 0000000000000..6b733d62a3921 --- /dev/null +++ b/tests/ignitetest/ignite_utils/ignite_path.py @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import os + +from ignitetest.version import get_version, IgniteVersion, DEV_BRANCH + + +"""This module serves a few purposes: + +First, it gathers information about path layout in a single place, and second, it +makes the layout of the Ignite installation pluggable, so that users are not forced +to use the layout assumed in the IgnitePathResolver class. +""" + +SCRATCH_ROOT = "/mnt" +IGNITE_INSTALL_ROOT = "/opt" + +def create_path_resolver(context, project="ignite"): + """Factory for generating a path resolver class + + This will first check for a fully qualified path resolver classname in context.globals. + + If present, construct a new instance, else default to IgniteSystemTestPathResolver + """ + assert project is not None + + resolver_fully_qualified_classname = "ignitetest.ignite_utils.ignite_path.IgniteSystemTestPathResolver" + # Using the fully qualified classname, import the resolver class + (module_name, resolver_class_name) = resolver_fully_qualified_classname.rsplit('.', 1) + cluster_mod = importlib.import_module(module_name) + path_resolver_class = getattr(cluster_mod, resolver_class_name) + path_resolver = path_resolver_class(context, project) + + return path_resolver + + +class IgnitePathResolverMixin(object): + """Mixin to automatically provide pluggable path resolution functionality to any class using it. + + Keep life simple, and don't add a constructor to this class: + Since use of a mixin entails multiple inheritence, it is *much* simpler to reason about the interaction of this + class with subclasses if we don't have to worry about method resolution order, constructor signatures etc. + """ + + @property + def path(self): + if not hasattr(self, "_path"): + setattr(self, "_path", create_path_resolver(self.context, "ignite")) + if hasattr(self.context, "logger") and self.context.logger is not None: + self.context.logger.debug("Using path resolver %s" % self._path.__class__.__name__) + + return self._path + + +class IgniteSystemTestPathResolver(object): + """Path resolver for Ignite system tests which assumes the following layout: + + /opt/ignite-dev # Current version of Ignite under test + /opt/ignite-2.7.6 # Example of an older version of Ignite installed from tarball + /opt/ignite- # Other previous versions of Ignite + ... + """ + def __init__(self, context, project="ignite"): + self.context = context + self.project = project + + def home(self, node_or_version=DEV_BRANCH, project=None): + version = self._version(node_or_version) + home_dir = project or self.project + if version is not None: + home_dir += "-%s" % str(version) + + return os.path.join(IGNITE_INSTALL_ROOT, home_dir) + + def bin(self, node_or_version=DEV_BRANCH, project=None): + version = self._version(node_or_version) + return os.path.join(self.home(version, project=project), "bin") + + def script(self, script_name, node_or_version=DEV_BRANCH, project=None): + version = self._version(node_or_version) + return os.path.join(self.bin(version, project=project), script_name) + + def jar(self, jar_name, node_or_version=DEV_BRANCH, project=None): + version = self._version(node_or_version) + return os.path.join(self.home(version, project=project), JARS[str(version)][jar_name]) + + def scratch_space(self, service_instance): + return os.path.join(SCRATCH_ROOT, service_instance.service_id) + + def _version(self, node_or_version): + if isinstance(node_or_version, IgniteVersion): + return node_or_version + else: + return get_version(node_or_version) + diff --git a/tests/ignitetest/services/ignite/ignite.py b/tests/ignitetest/services/ignite/ignite.py index 7db90ec65d45c..af60986e6af35 100644 --- a/tests/ignitetest/services/ignite/ignite.py +++ b/tests/ignitetest/services/ignite/ignite.py @@ -20,19 +20,19 @@ from ducktape.utils.util import wait_until from ducktape.cluster.remoteaccount import RemoteCommandError -from ignitetest.version import DEV_VERSION +from ignitetest.ignite_utils.ignite_config import IgniteConfig +from ignitetest.ignite_utils.ignite_path import IgnitePathResolverMixin +from ignitetest.version import DEV_BRANCH -class IgniteService(Service): +class IgniteService(IgnitePathResolverMixin, Service): PERSISTENT_ROOT = "/mnt/ignite" + WORK_DIR = os.path.join(PERSISTENT_ROOT, "work") CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-config.xml") HEAP_DUMP_FILE = os.path.join(PERSISTENT_ROOT, "ignite-heap.bin") STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") - def __init__(self, - context, - num_nodes, - version=DEV_VERSION): + def __init__(self, context, num_nodes=3, version=DEV_BRANCH): """ :param context: test context :param num_nodes: number of Ignite nodes. @@ -40,6 +40,7 @@ def __init__(self, Service.__init__(self, context, num_nodes) self.log_level = "DEBUG" + self.config = IgniteConfig() for node in self.nodes: node.version = version @@ -64,14 +65,15 @@ def start_cmd(self, node): IgniteService.STDOUT_STDERR_CAPTURE) return cmd - def start_node(self, node, timeout_sec=60): + def start_node(self, node, timeout_sec=180): node.account.mkdirs(IgniteService.PERSISTENT_ROOT) + node.account.create_file(IgniteService.CONFIG_FILE, self.config.render(IgniteService.WORK_DIR)) cmd = self.start_cmd(node) self.logger.debug("Attempting to start IgniteService on %s with command: %s" % (str(node.account), cmd)) with node.account.monitor_log(IgniteService.STDOUT_STDERR_CAPTURE) as monitor: node.account.ssh(cmd) - monitor.wait_until("Ignite\s*Server.*started", timeout_sec=timeout_sec, backoff_sec=.25, + monitor.wait_until("Topology snapshot", timeout_sec=timeout_sec, backoff_sec=.25, err_msg="Ignite server didn't finish startup in %d seconds" % timeout_sec) if len(self.pids(node)) == 0: @@ -111,3 +113,6 @@ def clean_node(self, node): node.account.kill_java_processes(self.java_class_name(), clean_shutdown=False, allow_fail=True) node.account.ssh("sudo rm -rf -- %s" % IgniteService.PERSISTENT_ROOT, allow_fail=False) + + def java_class_name(self): + return "org.apache.ignite.startup.cmdline.CommandLineStartup" diff --git a/tests/ignitetest/tests/ignite_test.py b/tests/ignitetest/suites/__init__.py similarity index 54% rename from tests/ignitetest/tests/ignite_test.py rename to tests/ignitetest/suites/__init__.py index c30a8fd82d611..ec2014340d78f 100644 --- a/tests/ignitetest/tests/ignite_test.py +++ b/tests/ignitetest/suites/__init__.py @@ -12,24 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from ducktape.tests.test import Test - - -from ignitetest.services.ignite import IgniteService - - -class IgniteTest(Test): - """ - Helper class that manages setting up a Ignite cluster. Use this if the - default settings for Ignite are sufficient for your test; any customization - needs to be done manually. Your run() method should call tearDown and - setUp. The Ignite service are available as the fields IgniteTest.ignite. - """ - def __init__(self, test_context): - super(IgniteTest, self).__init__(test_context) - - self.ignite = IgniteService(test_context, self.num_brokers) - - def setUp(self): - self.ignite.start() \ No newline at end of file diff --git a/tests/ignitetest/rebalance/add_node_rebalance_test.py b/tests/ignitetest/suites/add_node_rebalance_test.py similarity index 78% rename from tests/ignitetest/rebalance/add_node_rebalance_test.py rename to tests/ignitetest/suites/add_node_rebalance_test.py index 501bf42fbc4d9..6b354ce708393 100644 --- a/tests/ignitetest/rebalance/add_node_rebalance_test.py +++ b/tests/ignitetest/suites/add_node_rebalance_test.py @@ -13,15 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ignitetest.tests.ignite_test import IgniteTest +from ducktape.tests.test import Test +from ignitetest.services.ignite.ignite import IgniteService -class AddNodeRebalanceTest(IgniteTest): + +class AddNodeRebalanceTest(Test): """ Test performs rebalance tests. """ def __init__(self, test_context): - super(IgniteTest, self).__init__(test_context) + super(AddNodeRebalanceTest, self).__init__(test_context=test_context) + self.ignite = IgniteService(test_context) + + def setUp(self): + self.ignite.start() def teardown(self): self.ignite.stop() @@ -34,5 +40,7 @@ def test_add_node(self): * Start one more node. * Await for rebalance to finish. """ + self.logger.info("Start add node rebalance test.") + for node in self.ignite.nodes: node.account.ssh("touch /opt/hello-from-test.txt") From 84c302b811c9b1587ba6371364cc90d9755eef5b Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Mon, 13 Apr 2020 12:29:28 +0300 Subject: [PATCH 06/78] ignite-ducktape: WIP. --- .../apache/ignite/test/IgniteApplication.java | 27 ++++ .../ignitetest/services/ignite_client_app.py | 146 ++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 modules/spring/src/main/java/org/apache/ignite/test/IgniteApplication.java create mode 100644 tests/ignitetest/services/ignite_client_app.py diff --git a/modules/spring/src/main/java/org/apache/ignite/test/IgniteApplication.java b/modules/spring/src/main/java/org/apache/ignite/test/IgniteApplication.java new file mode 100644 index 0000000000000..d3446fc0dd90e --- /dev/null +++ b/modules/spring/src/main/java/org/apache/ignite/test/IgniteApplication.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.test; + +import java.io.File; + +public class IgniteApplication { + public static void main(String[] args) throws Exception { + System.out.println("IgniteApplication.main"); + new File("/opt/hello-from-app.txt").createNewFile(); + } +} diff --git a/tests/ignitetest/services/ignite_client_app.py b/tests/ignitetest/services/ignite_client_app.py new file mode 100644 index 0000000000000..4462e7182ca50 --- /dev/null +++ b/tests/ignitetest/services/ignite_client_app.py @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from ducktape.cluster.remoteaccount import RemoteCommandError +from ducktape.services.background_thread import BackgroundThreadService + +from ignitetest.directory_layout.ignite_path import IgnitePathResolverMixin + +from ignitetest.version import DEV_BRANCH + +""" +The console consumer is a tool that reads data from Kafka and outputs it to standard output. +""" + + +class IgniteClientApp(IgnitePathResolverMixin, BackgroundThreadService): + # Root directory for persistent output + PERSISTENT_ROOT = "/mnt/client_app" + STDOUT_CAPTURE = os.path.join(PERSISTENT_ROOT, "client_app.stdout") + STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "client_app.stderr") + LOG_DIR = os.path.join(PERSISTENT_ROOT, "logs") + LOG_FILE = os.path.join(LOG_DIR, "client_app.log") + + logs = { + "consumer_stdout": { + "path": STDOUT_CAPTURE, + "collect_default": False}, + "consumer_stderr": { + "path": STDERR_CAPTURE, + "collect_default": False}, + "consumer_log": { + "path": LOG_FILE, + "collect_default": True} + } + + def __init__(self, context, version=DEV_BRANCH, num_nodes=1): + """ + Args: + num_nodes: number of nodes to use (this should be 1) + """ + BackgroundThreadService.__init__(self, context, num_nodes) + + for node in self.nodes: + node.version = version + + def start_cmd(self, node): + """Return the start command appropriate for the given node.""" + args = self.args.copy() + args['stdout'] = IgniteClientApp.STDOUT_CAPTURE + args['stderr'] = IgniteClientApp.STDERR_CAPTURE + args['log_dir'] = IgniteClientApp.LOG_DIR + args['config_file'] = IgniteClientApp.CONFIG_FILE + args['stdout'] = IgniteClientApp.STDOUT_CAPTURE + args['console_consumer'] = self.path.script("kafka-console-consumer.sh", node) + + cmd = "%(console_consumer)s " \ + "--topic %(topic)s " \ + "--consumer.config %(config_file)s " % args + + cmd += " 2>> %(stderr)s | tee -a %(stdout)s &" % args + return cmd + + def pids(self, node): + return node.account.java_pids(self.java_class_name()) + + def alive(self, node): + return len(self.pids(node)) > 0 + + def _worker(self, idx, node): + node.account.ssh("mkdir -p %s" % IgniteClientApp.PERSISTENT_ROOT, allow_fail=False) + + self.security_config = self.kafka.security_config.client_config(node=node, + jaas_override_variables=self.jaas_override_variables) + self.security_config.setup_node(node) + + if self.client_prop_file_override: + prop_file = self.client_prop_file_override + else: + prop_file = self.prop_file(node) + + # TODO: create ignite config: node.account.create_file(IgniteClientApp.CONFIG_FILE, prop_file) + + # Run and capture output + cmd = self.start_cmd(node) + self.logger.debug("Console consumer %d command: %s", idx, cmd) + + consumer_output = node.account.ssh_capture(cmd, allow_fail=False) + + # TODO: create ignite version of waiting for a app finish + for line in consumer_output: + msg = line.strip() + if msg == "shutdown_complete": + # Note that we can only rely on shutdown_complete message if running 0.10.0 or greater + if node in self.clean_shutdown_nodes: + raise Exception( + "Unexpected shutdown event from consumer, already shutdown. Consumer index: %d" % idx) + self.clean_shutdown_nodes.add(node) + else: + if self.message_validator is not None: + msg = self.message_validator(msg) + if msg is not None: + self.messages_consumed[idx].append(msg) + + def start_node(self, node): + BackgroundThreadService.start_node(self, node) + + def stop_node(self, node): + self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) + node.account.kill_java_processes(self.java_class_name(), + clean_shutdown=True, allow_fail=True) + + stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec) + assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ + (str(node.account), str(self.stop_timeout_sec)) + + def clean_node(self, node): + if self.alive(node): + self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." % + (self.__class__.__name__, node.account)) + node.account.kill_java_processes(self.java_class_name(), clean_shutdown=False, allow_fail=True) + node.account.ssh("rm -rf %s" % IgniteClientApp.PERSISTENT_ROOT, allow_fail=False) + self.security_config.clean_node(node) + + def java_class_name(self): + return "org.apache.ignite.test.IgniteApplication" + + def has_log_message(self, node, message): + try: + node.account.ssh("grep '%s' %s" % (message, IgniteClientApp.LOG_FILE)) + except RemoteCommandError: + return False + return True From 082b1876cd424dc594d574562a6c8a56b0d869e9 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Tue, 5 May 2020 22:59:29 +0300 Subject: [PATCH 07/78] ignite-ducktape: fix logging to the docker image instead of src/ignite/work directory. --- .gitignore | 3 ++ bin/include/build-classpath.sh | 6 ++- tests/docker/Dockerfile | 2 +- tests/docker/build/node_hosts | 8 +-- .../ignitetest/ignite_utils/ignite_config.py | 54 +++++++++++++++++-- tests/ignitetest/services/ignite/ignite.py | 13 ++++- 6 files changed, 74 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index cdc70655df7fe..c5415d47c199b 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,6 @@ packages #NodeJs files /modules/platforms/nodejs/node_modules + +/results +.ducktape diff --git a/bin/include/build-classpath.sh b/bin/include/build-classpath.sh index dbcd81e24e861..edfb2679b6890 100644 --- a/bin/include/build-classpath.sh +++ b/bin/include/build-classpath.sh @@ -52,8 +52,10 @@ includeToClassPath() { IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/classes fi - if [ -d "${file}/target/test-classes" ]; then - IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/test-classes + if [[ -z "${EXCLUDE_TEST_CLASSES}" ]]; then + if [ -d "${file}/target/test-classes" ]; then + IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/test-classes + fi fi if [ -d "${file}/target/libs" ]; then diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 390d84955b6f6..c94e9f300d5f0 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -32,7 +32,7 @@ ARG ducker_creator=default LABEL ducker.creator=$ducker_creator # Update Linux and install necessary utilities. -RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute && apt-get -y clean +RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute mc && apt-get -y clean RUN python -m pip install -U pip==9.0.3; RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.6 diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts index 9da72b8fb98bb..e1183dd4aa70e 100644 --- a/tests/docker/build/node_hosts +++ b/tests/docker/build/node_hosts @@ -1,7 +1,7 @@ -172.26.0.2 ducker01 -172.26.0.3 ducker02 -172.26.0.4 ducker03 -172.26.0.5 ducker04 +172.22.0.2 ducker01 +172.22.0.3 ducker02 +172.22.0.4 ducker03 +172.22.0.5 ducker04 172.26.0.6 ducker05 172.26.0.7 ducker06 172.26.0.8 ducker07 diff --git a/tests/ignitetest/ignite_utils/ignite_config.py b/tests/ignitetest/ignite_utils/ignite_config.py index 416391ac04611..b0008a6af8112 100644 --- a/tests/ignitetest/ignite_utils/ignite_config.py +++ b/tests/ignitetest/ignite_utils/ignite_config.py @@ -22,7 +22,7 @@ class IgniteConfig: def __init__(self, project="ignite"): self.project = project - def render(self, work_dir): + def render(self, config_dir, work_dir): return """ - + + + + + + - """.format(work_dir) + """.format(config_dir=config_dir, work_dir=work_dir) + + def render_log4j(self, work_dir): + return """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + """.format(work_dir=work_dir) diff --git a/tests/ignitetest/services/ignite/ignite.py b/tests/ignitetest/services/ignite/ignite.py index af60986e6af35..e92508d526091 100644 --- a/tests/ignitetest/services/ignite/ignite.py +++ b/tests/ignitetest/services/ignite/ignite.py @@ -29,6 +29,7 @@ class IgniteService(IgnitePathResolverMixin, Service): PERSISTENT_ROOT = "/mnt/ignite" WORK_DIR = os.path.join(PERSISTENT_ROOT, "work") CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-config.xml") + LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-log4j.xml") HEAP_DUMP_FILE = os.path.join(PERSISTENT_ROOT, "ignite-heap.bin") STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") @@ -58,8 +59,14 @@ def start(self): self.logger.info("Waiting for Ignite to start...") def start_cmd(self, node): - cmd = "%s %s 1>> %s 2>> %s &" % \ + jvm_opts = "-J-DIGNITE_SUCCESS_FILE=" + IgniteService.PERSISTENT_ROOT + "/success_file " + jvm_opts += "-J-Dlog4j.configDebug=true" + + cmd = "export EXCLUDE_TEST_CLASSES=true; " + cmd += "export IGNITE_LOG_DIR=" + IgniteService.PERSISTENT_ROOT + "; " + cmd += "%s %s %s 1>> %s 2>> %s &" % \ (self.path.script("ignite.sh", node), + jvm_opts, IgniteService.CONFIG_FILE, IgniteService.STDOUT_STDERR_CAPTURE, IgniteService.STDOUT_STDERR_CAPTURE) @@ -67,7 +74,9 @@ def start_cmd(self, node): def start_node(self, node, timeout_sec=180): node.account.mkdirs(IgniteService.PERSISTENT_ROOT) - node.account.create_file(IgniteService.CONFIG_FILE, self.config.render(IgniteService.WORK_DIR)) + node.account.create_file(IgniteService.CONFIG_FILE, + self.config.render(IgniteService.PERSISTENT_ROOT, IgniteService.WORK_DIR)) + node.account.create_file(IgniteService.LOG4J_CONFIG_FILE, self.config.render_log4j(IgniteService.WORK_DIR)) cmd = self.start_cmd(node) self.logger.debug("Attempting to start IgniteService on %s with command: %s" % (str(node.account), cmd)) From 410e347ddd044562f3ffa91ba3fb81f6f5697870 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 6 May 2020 10:57:50 +0300 Subject: [PATCH 08/78] ignite-ducktape: console.log now captured for each Ignite node. --- .gitignore | 1 + tests/ignitetest/services/ignite/ignite.py | 51 +++++++++++++--------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index c5415d47c199b..18bcd81cbf26a 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,4 @@ packages /results .ducktape +*.pyc diff --git a/tests/ignitetest/services/ignite/ignite.py b/tests/ignitetest/services/ignite/ignite.py index e92508d526091..1dcdbdca460fd 100644 --- a/tests/ignitetest/services/ignite/ignite.py +++ b/tests/ignitetest/services/ignite/ignite.py @@ -33,6 +33,16 @@ class IgniteService(IgnitePathResolverMixin, Service): HEAP_DUMP_FILE = os.path.join(PERSISTENT_ROOT, "ignite-heap.bin") STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") + logs = { + "console_log": { + "path": STDOUT_STDERR_CAPTURE, + "collect_default": True}, + + "heap_dump": { + "path": HEAP_DUMP_FILE, + "collect_default": False} + } + def __init__(self, context, num_nodes=3, version=DEV_BRANCH): """ :param context: test context @@ -46,13 +56,6 @@ def __init__(self, context, num_nodes=3, version=DEV_BRANCH): for node in self.nodes: node.version = version - def set_version(self, version): - for node in self.nodes: - node.version = version - - def alive(self, node): - return len(self.pids(node)) > 0 - def start(self): Service.start(self) @@ -88,15 +91,6 @@ def start_node(self, node, timeout_sec=180): if len(self.pids(node)) == 0: raise Exception("No process ids recorded on node %s" % node.account.hostname) - def pids(self, node): - """Return process ids associated with running processes on the given node.""" - try: - cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.java_class_name() - pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] - return pid_arr - except (RemoteCommandError, ValueError) as e: - return [] - def stop_node(self, node, clean_shutdown=True, timeout_sec=60): pids = self.pids(node) sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL @@ -111,6 +105,11 @@ def stop_node(self, node, clean_shutdown=True, timeout_sec=60): self.thread_dump(node) raise + def clean_node(self, node): + node.account.kill_java_processes(self.java_class_name(), + clean_shutdown=False, allow_fail=True) + node.account.ssh("sudo rm -rf -- %s" % IgniteService.PERSISTENT_ROOT, allow_fail=False) + def thread_dump(self, node): for pid in self.pids(node): try: @@ -118,10 +117,22 @@ def thread_dump(self, node): except: self.logger.warn("Could not dump threads on node") - def clean_node(self, node): - node.account.kill_java_processes(self.java_class_name(), - clean_shutdown=False, allow_fail=True) - node.account.ssh("sudo rm -rf -- %s" % IgniteService.PERSISTENT_ROOT, allow_fail=False) + def pids(self, node): + """Return process ids associated with running processes on the given node.""" + try: + cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.java_class_name() + pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] + return pid_arr + except (RemoteCommandError, ValueError) as e: + return [] def java_class_name(self): return "org.apache.ignite.startup.cmdline.CommandLineStartup" + + def set_version(self, version): + for node in self.nodes: + node.version = version + + def alive(self, node): + return len(self.pids(node)) > 0 + From f8b176a272d1cbdbadb3c6b66ae301b91615f0bc Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 6 May 2020 16:05:52 +0300 Subject: [PATCH 09/78] ignite-ducktape: IgniteClientApplication added and working. --- .../internal/test/IgniteApplication.java | 52 +++++++ tests/docker/build/cluster.json | 11 ++ tests/docker/build/node_hosts | 8 +- tests/docker/run_tests.sh | 2 +- .../ignitetest/ignite_utils/ignite_config.py | 2 +- tests/ignitetest/ignite_utils/ignite_path.py | 71 ++------- tests/ignitetest/services/ignite/ignite.py | 10 +- .../ignitetest/services/ignite_client_app.py | 137 +++++++----------- .../suites/add_node_rebalance_test.py | 6 +- 9 files changed, 143 insertions(+), 156 deletions(-) create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java new file mode 100644 index 0000000000000..dffa891f01afe --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.test; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.Ignition; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgnitionEx; +import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; +import org.apache.ignite.lang.IgniteBiTuple; + +public class IgniteApplication { + public static final String CONFIG_PATH = "/mnt/client_app/ignite-config.xml"; + + public static void main(String[] args) throws IgniteCheckedException { + IgniteBiTuple cfgs = IgnitionEx.loadConfiguration(CONFIG_PATH); + IgniteConfiguration cfg = cfgs.get1(); + + cfg.setClientMode(true); + + System.out.println("Starting Ignite client..."); + + try (Ignite ign = Ignition.start(cfg)) { + System.out.println("Creating cache..."); + + IgniteCache cache = ign.createCache("test-cache"); + + for (int i = 0; i < 1000; i++) { + cache.put(i, i); + } + + System.out.println("Ignite Client Finish."); + } + } +} diff --git a/tests/docker/build/cluster.json b/tests/docker/build/cluster.json index 6e5e46ea4850d..297d9c4a83fd6 100644 --- a/tests/docker/build/cluster.json +++ b/tests/docker/build/cluster.json @@ -48,6 +48,17 @@ "port": 22, "user": "ducker" } + }, + { + "externally_routable_ip": "ducker05", + "ssh_config": { + "host": "ducker05", + "hostname": "ducker05", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } } ] } diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts index e1183dd4aa70e..9da72b8fb98bb 100644 --- a/tests/docker/build/node_hosts +++ b/tests/docker/build/node_hosts @@ -1,7 +1,7 @@ -172.22.0.2 ducker01 -172.22.0.3 ducker02 -172.22.0.4 ducker03 -172.22.0.5 ducker04 +172.26.0.2 ducker01 +172.26.0.3 ducker02 +172.26.0.4 ducker03 +172.26.0.5 ducker04 172.26.0.6 ducker05 172.26.0.7 ducker06 172.26.0.8 ducker07 diff --git a/tests/docker/run_tests.sh b/tests/docker/run_tests.sh index 653b46a6911d2..9de23173a590d 100755 --- a/tests/docker/run_tests.sh +++ b/tests/docker/run_tests.sh @@ -16,7 +16,7 @@ # limitations under the License. SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-4} +IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-5} TC_PATHS=${TC_PATHS:-./ignitetest/} die() { diff --git a/tests/ignitetest/ignite_utils/ignite_config.py b/tests/ignitetest/ignite_utils/ignite_config.py index b0008a6af8112..534df42d55fde 100644 --- a/tests/ignitetest/ignite_utils/ignite_config.py +++ b/tests/ignitetest/ignite_utils/ignite_config.py @@ -49,7 +49,7 @@ def render_log4j(self, work_dir): - + diff --git a/tests/ignitetest/ignite_utils/ignite_path.py b/tests/ignitetest/ignite_utils/ignite_path.py index 6b733d62a3921..63071758a69c9 100644 --- a/tests/ignitetest/ignite_utils/ignite_path.py +++ b/tests/ignitetest/ignite_utils/ignite_path.py @@ -13,60 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import importlib import os from ignitetest.version import get_version, IgniteVersion, DEV_BRANCH - -"""This module serves a few purposes: - -First, it gathers information about path layout in a single place, and second, it -makes the layout of the Ignite installation pluggable, so that users are not forced -to use the layout assumed in the IgnitePathResolver class. +""" +This module provides Ignite path methods """ -SCRATCH_ROOT = "/mnt" -IGNITE_INSTALL_ROOT = "/opt" - -def create_path_resolver(context, project="ignite"): - """Factory for generating a path resolver class - - This will first check for a fully qualified path resolver classname in context.globals. - - If present, construct a new instance, else default to IgniteSystemTestPathResolver - """ - assert project is not None - - resolver_fully_qualified_classname = "ignitetest.ignite_utils.ignite_path.IgniteSystemTestPathResolver" - # Using the fully qualified classname, import the resolver class - (module_name, resolver_class_name) = resolver_fully_qualified_classname.rsplit('.', 1) - cluster_mod = importlib.import_module(module_name) - path_resolver_class = getattr(cluster_mod, resolver_class_name) - path_resolver = path_resolver_class(context, project) - - return path_resolver - - -class IgnitePathResolverMixin(object): - """Mixin to automatically provide pluggable path resolution functionality to any class using it. - - Keep life simple, and don't add a constructor to this class: - Since use of a mixin entails multiple inheritence, it is *much* simpler to reason about the interaction of this - class with subclasses if we don't have to worry about method resolution order, constructor signatures etc. - """ - - @property - def path(self): - if not hasattr(self, "_path"): - setattr(self, "_path", create_path_resolver(self.context, "ignite")) - if hasattr(self.context, "logger") and self.context.logger is not None: - self.context.logger.debug("Using path resolver %s" % self._path.__class__.__name__) - - return self._path +class IgnitePath: + SCRATCH_ROOT = "/mnt" + IGNITE_INSTALL_ROOT = "/opt" -class IgniteSystemTestPathResolver(object): """Path resolver for Ignite system tests which assumes the following layout: /opt/ignite-dev # Current version of Ignite under test @@ -74,8 +33,8 @@ class IgniteSystemTestPathResolver(object): /opt/ignite- # Other previous versions of Ignite ... """ - def __init__(self, context, project="ignite"): - self.context = context + + def __init__(self, project="ignite"): self.project = project def home(self, node_or_version=DEV_BRANCH, project=None): @@ -84,26 +43,14 @@ def home(self, node_or_version=DEV_BRANCH, project=None): if version is not None: home_dir += "-%s" % str(version) - return os.path.join(IGNITE_INSTALL_ROOT, home_dir) - - def bin(self, node_or_version=DEV_BRANCH, project=None): - version = self._version(node_or_version) - return os.path.join(self.home(version, project=project), "bin") + return os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir) def script(self, script_name, node_or_version=DEV_BRANCH, project=None): version = self._version(node_or_version) - return os.path.join(self.bin(version, project=project), script_name) - - def jar(self, jar_name, node_or_version=DEV_BRANCH, project=None): - version = self._version(node_or_version) - return os.path.join(self.home(version, project=project), JARS[str(version)][jar_name]) - - def scratch_space(self, service_instance): - return os.path.join(SCRATCH_ROOT, service_instance.service_id) + return os.path.join(self.home(version, project=project), "bin", script_name) def _version(self, node_or_version): if isinstance(node_or_version, IgniteVersion): return node_or_version else: return get_version(node_or_version) - diff --git a/tests/ignitetest/services/ignite/ignite.py b/tests/ignitetest/services/ignite/ignite.py index 1dcdbdca460fd..f8f208a6e065c 100644 --- a/tests/ignitetest/services/ignite/ignite.py +++ b/tests/ignitetest/services/ignite/ignite.py @@ -18,14 +18,13 @@ from ducktape.services.service import Service from ducktape.utils.util import wait_until -from ducktape.cluster.remoteaccount import RemoteCommandError from ignitetest.ignite_utils.ignite_config import IgniteConfig -from ignitetest.ignite_utils.ignite_path import IgnitePathResolverMixin +from ignitetest.ignite_utils.ignite_path import IgnitePath from ignitetest.version import DEV_BRANCH -class IgniteService(IgnitePathResolverMixin, Service): +class IgniteService(Service): PERSISTENT_ROOT = "/mnt/ignite" WORK_DIR = os.path.join(PERSISTENT_ROOT, "work") CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-config.xml") @@ -43,7 +42,7 @@ class IgniteService(IgnitePathResolverMixin, Service): "collect_default": False} } - def __init__(self, context, num_nodes=3, version=DEV_BRANCH): + def __init__(self, context, num_nodes=1, version=DEV_BRANCH): """ :param context: test context :param num_nodes: number of Ignite nodes. @@ -52,6 +51,7 @@ def __init__(self, context, num_nodes=3, version=DEV_BRANCH): self.log_level = "DEBUG" self.config = IgniteConfig() + self.path = IgnitePath() for node in self.nodes: node.version = version @@ -85,7 +85,7 @@ def start_node(self, node, timeout_sec=180): self.logger.debug("Attempting to start IgniteService on %s with command: %s" % (str(node.account), cmd)) with node.account.monitor_log(IgniteService.STDOUT_STDERR_CAPTURE) as monitor: node.account.ssh(cmd) - monitor.wait_until("Topology snapshot", timeout_sec=timeout_sec, backoff_sec=.25, + monitor.wait_until("Topology snapshot", timeout_sec=timeout_sec, backoff_sec=5, err_msg="Ignite server didn't finish startup in %d seconds" % timeout_sec) if len(self.pids(node)) == 0: diff --git a/tests/ignitetest/services/ignite_client_app.py b/tests/ignitetest/services/ignite_client_app.py index 4462e7182ca50..82a661ae96e01 100644 --- a/tests/ignitetest/services/ignite_client_app.py +++ b/tests/ignitetest/services/ignite_client_app.py @@ -15,35 +15,29 @@ import os -from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.services.background_thread import BackgroundThreadService -from ignitetest.directory_layout.ignite_path import IgnitePathResolverMixin - +from ignitetest.ignite_utils.ignite_config import IgniteConfig +from ignitetest.ignite_utils.ignite_path import IgnitePath from ignitetest.version import DEV_BRANCH """ -The console consumer is a tool that reads data from Kafka and outputs it to standard output. +The Ignite client application is a main class that implements custom logic. +First CMD param is an absolute path to the Ignite config file. """ -class IgniteClientApp(IgnitePathResolverMixin, BackgroundThreadService): +class IgniteClientApp(BackgroundThreadService): # Root directory for persistent output PERSISTENT_ROOT = "/mnt/client_app" - STDOUT_CAPTURE = os.path.join(PERSISTENT_ROOT, "client_app.stdout") - STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "client_app.stderr") - LOG_DIR = os.path.join(PERSISTENT_ROOT, "logs") - LOG_FILE = os.path.join(LOG_DIR, "client_app.log") + STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") + WORK_DIR = os.path.join(PERSISTENT_ROOT, "work") + CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-config.xml") + LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-log4j.xml") logs = { - "consumer_stdout": { - "path": STDOUT_CAPTURE, - "collect_default": False}, - "consumer_stderr": { - "path": STDERR_CAPTURE, - "collect_default": False}, - "consumer_log": { - "path": LOG_FILE, + "console_log": { + "path": STDOUT_STDERR_CAPTURE, "collect_default": True} } @@ -54,66 +48,30 @@ def __init__(self, context, version=DEV_BRANCH, num_nodes=1): """ BackgroundThreadService.__init__(self, context, num_nodes) + self.stop_timeout_sec = 10 + self.log_level = "DEBUG" + self.config = IgniteConfig() + self.path = IgnitePath() + for node in self.nodes: node.version = version def start_cmd(self, node): """Return the start command appropriate for the given node.""" - args = self.args.copy() - args['stdout'] = IgniteClientApp.STDOUT_CAPTURE - args['stderr'] = IgniteClientApp.STDERR_CAPTURE - args['log_dir'] = IgniteClientApp.LOG_DIR - args['config_file'] = IgniteClientApp.CONFIG_FILE - args['stdout'] = IgniteClientApp.STDOUT_CAPTURE - args['console_consumer'] = self.path.script("kafka-console-consumer.sh", node) - - cmd = "%(console_consumer)s " \ - "--topic %(topic)s " \ - "--consumer.config %(config_file)s " % args - - cmd += " 2>> %(stderr)s | tee -a %(stdout)s &" % args - return cmd - - def pids(self, node): - return node.account.java_pids(self.java_class_name()) - def alive(self, node): - return len(self.pids(node)) > 0 - - def _worker(self, idx, node): - node.account.ssh("mkdir -p %s" % IgniteClientApp.PERSISTENT_ROOT, allow_fail=False) - - self.security_config = self.kafka.security_config.client_config(node=node, - jaas_override_variables=self.jaas_override_variables) - self.security_config.setup_node(node) - - if self.client_prop_file_override: - prop_file = self.client_prop_file_override - else: - prop_file = self.prop_file(node) - - # TODO: create ignite config: node.account.create_file(IgniteClientApp.CONFIG_FILE, prop_file) - - # Run and capture output - cmd = self.start_cmd(node) - self.logger.debug("Console consumer %d command: %s", idx, cmd) - - consumer_output = node.account.ssh_capture(cmd, allow_fail=False) - - # TODO: create ignite version of waiting for a app finish - for line in consumer_output: - msg = line.strip() - if msg == "shutdown_complete": - # Note that we can only rely on shutdown_complete message if running 0.10.0 or greater - if node in self.clean_shutdown_nodes: - raise Exception( - "Unexpected shutdown event from consumer, already shutdown. Consumer index: %d" % idx) - self.clean_shutdown_nodes.add(node) - else: - if self.message_validator is not None: - msg = self.message_validator(msg) - if msg is not None: - self.messages_consumed[idx].append(msg) + jvm_opts = "-J-DIGNITE_SUCCESS_FILE=" + IgniteClientApp.PERSISTENT_ROOT + "/success_file " + jvm_opts += "-J-Dlog4j.configDebug=true" + + cmd = "export MAIN_CLASS={main_class}; ".format(main_class=self.java_class_name()) + cmd += "export EXCLUDE_TEST_CLASSES=true; " + cmd += "export IGNITE_LOG_DIR=" + IgniteClientApp.PERSISTENT_ROOT + "; " + cmd += "%s %s %s 1>> %s 2>> %s " % \ + (self.path.script("ignite.sh", node), + jvm_opts, + IgniteClientApp.CONFIG_FILE, + IgniteClientApp.STDOUT_STDERR_CAPTURE, + IgniteClientApp.STDOUT_STDERR_CAPTURE) + return cmd def start_node(self, node): BackgroundThreadService.start_node(self, node) @@ -121,7 +79,8 @@ def start_node(self, node): def stop_node(self, node): self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) node.account.kill_java_processes(self.java_class_name(), - clean_shutdown=True, allow_fail=True) + clean_shutdown=True, + allow_fail=True) stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec) assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ @@ -131,16 +90,30 @@ def clean_node(self, node): if self.alive(node): self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." % (self.__class__.__name__, node.account)) - node.account.kill_java_processes(self.java_class_name(), clean_shutdown=False, allow_fail=True) + + node.account.kill_java_processes(self.java_class_name(), + clean_shutdown=False, + allow_fail=True) + node.account.ssh("rm -rf %s" % IgniteClientApp.PERSISTENT_ROOT, allow_fail=False) - self.security_config.clean_node(node) + + def pids(self, node): + return node.account.java_pids(self.java_class_name()) + + def alive(self, node): + return len(self.pids(node)) > 0 + + def _worker(self, idx, node): + node.account.mkdirs(IgniteClientApp.PERSISTENT_ROOT) + node.account.create_file(IgniteClientApp.CONFIG_FILE, + self.config.render(IgniteClientApp.PERSISTENT_ROOT, IgniteClientApp.WORK_DIR)) + node.account.create_file(IgniteClientApp.LOG4J_CONFIG_FILE, + self.config.render_log4j(IgniteClientApp.WORK_DIR)) + + # Just run application. + cmd = self.start_cmd(node) + self.logger.info("Ignite client application command: %s", cmd) + node.account.ssh(cmd, allow_fail=False) def java_class_name(self): - return "org.apache.ignite.test.IgniteApplication" - - def has_log_message(self, node, message): - try: - node.account.ssh("grep '%s' %s" % (message, IgniteClientApp.LOG_FILE)) - except RemoteCommandError: - return False - return True + return "org.apache.ignite.internal.test.IgniteApplication" diff --git a/tests/ignitetest/suites/add_node_rebalance_test.py b/tests/ignitetest/suites/add_node_rebalance_test.py index 6b354ce708393..24ca1512e31ee 100644 --- a/tests/ignitetest/suites/add_node_rebalance_test.py +++ b/tests/ignitetest/suites/add_node_rebalance_test.py @@ -16,6 +16,7 @@ from ducktape.tests.test import Test from ignitetest.services.ignite.ignite import IgniteService +from ignitetest.services.ignite_client_app import IgniteClientApp class AddNodeRebalanceTest(Test): @@ -41,6 +42,9 @@ def test_add_node(self): * Await for rebalance to finish. """ self.logger.info("Start add node rebalance test.") + self.client = IgniteClientApp(self.test_context) + self.client.start() + self.client.wait() for node in self.ignite.nodes: - node.account.ssh("touch /opt/hello-from-test.txt") + node.account.ssh("touch /opt/hello-from-test-after-client.txt") From f05dfb3e95b75bb3e4c3066268fd396632b19818 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 6 May 2020 17:12:49 +0300 Subject: [PATCH 10/78] ignite-ducktape: starting last node after client and waiting for rebalance. --- tests/docker/build/node_hosts | 10 +++++----- tests/ignitetest/services/ignite/ignite.py | 11 +++++++--- .../suites/add_node_rebalance_test.py | 20 ++++++++++++------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts index 9da72b8fb98bb..e675c022bc8e8 100644 --- a/tests/docker/build/node_hosts +++ b/tests/docker/build/node_hosts @@ -1,8 +1,8 @@ -172.26.0.2 ducker01 -172.26.0.3 ducker02 -172.26.0.4 ducker03 -172.26.0.5 ducker04 -172.26.0.6 ducker05 +172.27.0.2 ducker01 +172.27.0.3 ducker02 +172.27.0.4 ducker03 +172.27.0.5 ducker04 +172.27.0.6 ducker05 172.26.0.7 ducker06 172.26.0.8 ducker07 172.26.0.9 ducker08 diff --git a/tests/ignitetest/services/ignite/ignite.py b/tests/ignitetest/services/ignite/ignite.py index f8f208a6e065c..16b87fe08a6c3 100644 --- a/tests/ignitetest/services/ignite/ignite.py +++ b/tests/ignitetest/services/ignite/ignite.py @@ -42,7 +42,7 @@ class IgniteService(Service): "collect_default": False} } - def __init__(self, context, num_nodes=1, version=DEV_BRANCH): + def __init__(self, context, num_nodes=3, version=DEV_BRANCH): """ :param context: test context :param num_nodes: number of Ignite nodes. @@ -75,7 +75,7 @@ def start_cmd(self, node): IgniteService.STDOUT_STDERR_CAPTURE) return cmd - def start_node(self, node, timeout_sec=180): + def start_node(self, node, timeout_sec=180, wait_for_rebalance=False): node.account.mkdirs(IgniteService.PERSISTENT_ROOT) node.account.create_file(IgniteService.CONFIG_FILE, self.config.render(IgniteService.PERSISTENT_ROOT, IgniteService.WORK_DIR)) @@ -83,9 +83,14 @@ def start_node(self, node, timeout_sec=180): cmd = self.start_cmd(node) self.logger.debug("Attempting to start IgniteService on %s with command: %s" % (str(node.account), cmd)) + + wait_for_message = "Topology snapshot" + if wait_for_rebalance: + wait_for_message = "Completed (final) rebalancing [grp=test-cache" + with node.account.monitor_log(IgniteService.STDOUT_STDERR_CAPTURE) as monitor: node.account.ssh(cmd) - monitor.wait_until("Topology snapshot", timeout_sec=timeout_sec, backoff_sec=5, + monitor.wait_until(wait_for_message, timeout_sec=timeout_sec, backoff_sec=5, err_msg="Ignite server didn't finish startup in %d seconds" % timeout_sec) if len(self.pids(node)) == 0: diff --git a/tests/ignitetest/suites/add_node_rebalance_test.py b/tests/ignitetest/suites/add_node_rebalance_test.py index 24ca1512e31ee..fad6bd0010b1d 100644 --- a/tests/ignitetest/suites/add_node_rebalance_test.py +++ b/tests/ignitetest/suites/add_node_rebalance_test.py @@ -20,15 +20,20 @@ class AddNodeRebalanceTest(Test): + NUM_NODES = 3 + REBALANCE_TIMEOUT = 600 + """ Test performs rebalance tests. """ def __init__(self, test_context): super(AddNodeRebalanceTest, self).__init__(test_context=test_context) - self.ignite = IgniteService(test_context) + self.ignite = IgniteService(test_context, num_nodes=AddNodeRebalanceTest.NUM_NODES) def setUp(self): - self.ignite.start() + # starting all nodes except last. + for i in range(AddNodeRebalanceTest.NUM_NODES-1): + self.ignite.start_node(self.ignite.nodes[i]) def teardown(self): self.ignite.stop() @@ -42,9 +47,10 @@ def test_add_node(self): * Await for rebalance to finish. """ self.logger.info("Start add node rebalance test.") - self.client = IgniteClientApp(self.test_context) - self.client.start() - self.client.wait() - for node in self.ignite.nodes: - node.account.ssh("touch /opt/hello-from-test-after-client.txt") + # This client just put some data to the cache. + IgniteClientApp(self.test_context).run() + + self.ignite.start_node(self.ignite.nodes[AddNodeRebalanceTest.NUM_NODES-1], + timeout_sec=AddNodeRebalanceTest.REBALANCE_TIMEOUT, + wait_for_rebalance = True) From c59ec2810aa5d8bd68b066de1da2a2687a1d65b6 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 6 May 2020 17:13:20 +0300 Subject: [PATCH 11/78] ignite-ducktape: starting last node after client and waiting for rebalance. --- tests/ignitetest/suites/add_node_rebalance_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/ignitetest/suites/add_node_rebalance_test.py b/tests/ignitetest/suites/add_node_rebalance_test.py index fad6bd0010b1d..569ed7425415c 100644 --- a/tests/ignitetest/suites/add_node_rebalance_test.py +++ b/tests/ignitetest/suites/add_node_rebalance_test.py @@ -42,9 +42,8 @@ def test_add_node(self): """ Test performs add node rebalance test which consists of following steps: * Start cluster. - * Put data to it via CacheDataProducer. - * Start one more node. - * Await for rebalance to finish. + * Put data to it via IgniteClientApp. + * Start one more node and awaits for rebalance to finish. """ self.logger.info("Start add node rebalance test.") From f5eeb043ff7b59339bc91dba5d4d55815e315b09 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 6 May 2020 17:17:03 +0300 Subject: [PATCH 12/78] ignite-ducktape: cleaning garbage. --- .../apache/ignite/test/IgniteApplication.java | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 modules/spring/src/main/java/org/apache/ignite/test/IgniteApplication.java diff --git a/modules/spring/src/main/java/org/apache/ignite/test/IgniteApplication.java b/modules/spring/src/main/java/org/apache/ignite/test/IgniteApplication.java deleted file mode 100644 index d3446fc0dd90e..0000000000000 --- a/modules/spring/src/main/java/org/apache/ignite/test/IgniteApplication.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.test; - -import java.io.File; - -public class IgniteApplication { - public static void main(String[] args) throws Exception { - System.out.println("IgniteApplication.main"); - new File("/opt/hello-from-app.txt").createNewFile(); - } -} From 7a2a226ec168c62a32ab8d305eebfd0466a83733 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 6 May 2020 17:18:35 +0300 Subject: [PATCH 13/78] ignite-ducktape: cleaning garbage. --- .gitignore | 1 + tests/docker/build/cluster.json | 64 --------------------------------- tests/docker/build/node_hosts | 14 -------- 3 files changed, 1 insertion(+), 78 deletions(-) delete mode 100644 tests/docker/build/cluster.json delete mode 100644 tests/docker/build/node_hosts diff --git a/.gitignore b/.gitignore index 18bcd81cbf26a..fee7d47ee5293 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,4 @@ packages /results .ducktape *.pyc +tests/docker/build/* diff --git a/tests/docker/build/cluster.json b/tests/docker/build/cluster.json deleted file mode 100644 index 297d9c4a83fd6..0000000000000 --- a/tests/docker/build/cluster.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "_comment": [ - "Licensed to the Apache Software Foundation (ASF) under one or more", - "contributor license agreements. See the NOTICE file distributed with", - "this work for additional information regarding copyright ownership.", - "The ASF licenses this file to You under the Apache License, Version 2.0", - "(the \"License\"); you may not use this file except in compliance with", - "the License. You may obtain a copy of the License at", - "", - "http://www.apache.org/licenses/LICENSE-2.0", - "", - "Unless required by applicable law or agreed to in writing, software", - "distributed under the License is distributed on an \"AS IS\" BASIS,", - "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", - "See the License for the specific language governing permissions and", - "limitations under the License." - ], - "nodes": [ - { - "externally_routable_ip": "ducker02", - "ssh_config": { - "host": "ducker02", - "hostname": "ducker02", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker03", - "ssh_config": { - "host": "ducker03", - "hostname": "ducker03", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker04", - "ssh_config": { - "host": "ducker04", - "hostname": "ducker04", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker05", - "ssh_config": { - "host": "ducker05", - "hostname": "ducker05", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - } - ] -} diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts deleted file mode 100644 index e675c022bc8e8..0000000000000 --- a/tests/docker/build/node_hosts +++ /dev/null @@ -1,14 +0,0 @@ -172.27.0.2 ducker01 -172.27.0.3 ducker02 -172.27.0.4 ducker03 -172.27.0.5 ducker04 -172.27.0.6 ducker05 -172.26.0.7 ducker06 -172.26.0.8 ducker07 -172.26.0.9 ducker08 -172.26.0.10 ducker09 -172.26.0.11 ducker10 -172.26.0.12 ducker11 -172.26.0.13 ducker12 -172.26.0.14 ducker13 -172.26.0.15 ducker14 From 2d57713cf88ec3d287e9a5fe10ee7d8f3dd580df Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 6 May 2020 17:42:41 +0300 Subject: [PATCH 14/78] ignite-ducktape: cleaning garbage. --- tests/ignitetest/services/ignite/__init__.py | 2 - tests/ignitetest/services/ignite/ignite.py | 2 +- tests/ignitetest/services/ignite/util.py | 37 ----- tests/ignitetest/utils/__init__.py | 16 --- tests/ignitetest/utils/remote_account.py | 41 ------ tests/ignitetest/utils/util.py | 138 ------------------- 6 files changed, 1 insertion(+), 235 deletions(-) delete mode 100644 tests/ignitetest/services/ignite/util.py delete mode 100644 tests/ignitetest/utils/__init__.py delete mode 100644 tests/ignitetest/utils/remote_account.py delete mode 100644 tests/ignitetest/utils/util.py diff --git a/tests/ignitetest/services/ignite/__init__.py b/tests/ignitetest/services/ignite/__init__.py index 6377f9ebe1fc8..ec2014340d78f 100644 --- a/tests/ignitetest/services/ignite/__init__.py +++ b/tests/ignitetest/services/ignite/__init__.py @@ -12,5 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from ignite import IgniteService \ No newline at end of file diff --git a/tests/ignitetest/services/ignite/ignite.py b/tests/ignitetest/services/ignite/ignite.py index 16b87fe08a6c3..1107ffe49d453 100644 --- a/tests/ignitetest/services/ignite/ignite.py +++ b/tests/ignitetest/services/ignite/ignite.py @@ -86,7 +86,7 @@ def start_node(self, node, timeout_sec=180, wait_for_rebalance=False): wait_for_message = "Topology snapshot" if wait_for_rebalance: - wait_for_message = "Completed (final) rebalancing [grp=test-cache" + wait_for_message = "Completed (final) rebalancing \[grp=test-cache" with node.account.monitor_log(IgniteService.STDOUT_STDERR_CAPTURE) as monitor: node.account.ssh(cmd) diff --git a/tests/ignitetest/services/ignite/util.py b/tests/ignitetest/services/ignite/util.py deleted file mode 100644 index bbd63e78df726..0000000000000 --- a/tests/ignitetest/services/ignite/util.py +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os.path - -from collections import namedtuple - -def java_version(node): - # Determine java version on the node - version = -1 - for line in node.account.ssh_capture("java -version"): - if line.find("version") != -1: - version = parse_version_str(line) - return version - -def parse_version_str(line): - # Parse java version string. Examples: - #`openjdk version "11.0.5" 2019-10-15` will return 11. - #`java version "1.5.0"` will return 5. - line = line[line.find('version \"') + 9:] - dot_pos = line.find(".") - if line[:dot_pos] == "1": - return int(line[dot_pos+1:line.find(".", dot_pos+1)]) - else: - return int(line[:dot_pos]) diff --git a/tests/ignitetest/utils/__init__.py b/tests/ignitetest/utils/__init__.py deleted file mode 100644 index d753d8ef5ed3d..0000000000000 --- a/tests/ignitetest/utils/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from util import ignitetest_version, is_int, is_int_with_prefix, node_is_reachable, validate_delivery diff --git a/tests/ignitetest/utils/remote_account.py b/tests/ignitetest/utils/remote_account.py deleted file mode 100644 index e838a96090c82..0000000000000 --- a/tests/ignitetest/utils/remote_account.py +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def file_exists(node, file): - """Quick and dirty check for existence of remote file.""" - try: - node.account.ssh("cat " + file, allow_fail=False) - return True - except: - return False - - -def path_exists(node, path): - """Quick and dirty check for existence of remote path.""" - try: - node.account.ssh("ls " + path, allow_fail=False) - return True - except: - return False - - -def line_count(node, file): - """Return the line count of file on node""" - out = [line for line in node.account.ssh_capture("wc -l %s" % file)] - if len(out) != 1: - raise Exception("Expected single line of output from wc -l") - - return int(out[0].strip().split(" ")[0]) diff --git a/tests/ignitetest/utils/util.py b/tests/ignitetest/utils/util.py deleted file mode 100644 index 0bb22eba2161b..0000000000000 --- a/tests/ignitetest/utils/util.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2015 Confluent Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ignitetest import __version__ as __ignitetest_version__ - -import math -import re -import time - - -def ignitetest_version(): - """Return string representation of current ducktape version.""" - return __ignitetest_version__ - - -def is_int(msg): - """Method used to check whether the given message is an integer - - return int or raises an exception if message is not an integer - """ - try: - return int(msg) - except ValueError: - raise Exception("Unexpected message format (expected an integer). Message: %s" % (msg)) - - -def is_int_with_prefix(msg): - """ - Method used check whether the given message is of format 'integer_prefix'.'integer_value' - - :param msg: message to validate - :return: msg or raises an exception is a message is of wrong format - """ - try: - parts = msg.split(".") - if len(parts) != 2: - raise Exception("Unexpected message format. Message should be of format: integer " - "prefix dot integer value. Message: %s" % (msg)) - int(parts[0]) - int(parts[1]) - return msg - except ValueError: - raise Exception("Unexpected message format. Message should be of format: integer " - "prefix dot integer value, but one of the two parts (before or after dot) " - "are not integers. Message: %s" % (msg)) - - -def node_is_reachable(src_node, dst_node): - """ - Returns true if a node is unreachable from another node. - - :param src_node: The source node to check from reachability from. - :param dst_node: The destination node to check for reachability to. - :return: True only if dst is reachable from src. - """ - return 0 == src_node.account.ssh("nc -w 3 -z %s 22" % dst_node.account.hostname, allow_fail=True) - - -def annotate_missing_msgs(missing, acked, consumed, msg): - missing_list = list(missing) - msg += "%s acked message did not make it to the Consumer. They are: " %\ - len(missing_list) - if len(missing_list) < 20: - msg += str(missing_list) + ". " - else: - msg += ", ".join(str(m) for m in missing_list[:20]) - msg += "...plus %s more. Total Acked: %s, Total Consumed: %s. " \ - % (len(missing_list) - 20, len(set(acked)), len(set(consumed))) - return msg - - -def annotate_data_lost(data_lost, msg, number_validated): - print_limit = 10 - if len(data_lost) > 0: - msg += "The first %s missing messages were validated to ensure they are in Kafka's data files. " \ - "%s were missing. This suggests data loss. Here are some of the messages not found in the data files: %s\n" \ - % (number_validated, len(data_lost), str(data_lost[0:print_limit]) if len(data_lost) > print_limit else str(data_lost)) - else: - msg += "We validated that the first %s of these missing messages correctly made it into Kafka's data files. " \ - "This suggests they were lost on their way to the consumer." % number_validated - return msg - - -def validate_delivery(acked, consumed, idempotence_enabled=False, check_lost_data=None, may_truncate_acked_records=False): - """Check that each acked message was consumed.""" - success = True - msg = "" - - # Correctness of the set difference operation depends on using equivalent - # message_validators in producer and consumer - missing = set(acked) - set(consumed) - - # Were all acked messages consumed? - if len(missing) > 0: - msg = annotate_missing_msgs(missing, acked, consumed, msg) - - # Did we miss anything due to data loss? - if check_lost_data: - max_truncate_count = 100 if may_truncate_acked_records else 0 - max_validate_count = max(1000, max_truncate_count) - - to_validate = list(missing)[0:min(len(missing), max_validate_count)] - data_lost = check_lost_data(to_validate) - - # With older versions of message format before KIP-101, data loss could occur due to truncation. - # These records won't be in the data logs. Tolerate limited data loss for this case. - if len(missing) < max_truncate_count and len(data_lost) == len(missing): - msg += "The %s missing messages were not present in Kafka's data files. This suggests data loss " \ - "due to truncation, which is possible with older message formats and hence are ignored " \ - "by this test. The messages lost: %s\n" % (len(data_lost), str(data_lost)) - else: - msg = annotate_data_lost(data_lost, msg, len(to_validate)) - success = False - else: - success = False - - # Are there duplicates? - if len(set(consumed)) != len(consumed): - num_duplicates = abs(len(set(consumed)) - len(consumed)) - - if idempotence_enabled: - success = False - msg += "Detected %d duplicates even though idempotence was enabled.\n" % num_duplicates - else: - msg += "(There are also %d duplicate messages in the log - but that is an acceptable outcome)\n" % num_duplicates - - return success, msg From c7611c11bca87cfe1600fa59192c3941c472cd90 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Sat, 9 May 2020 17:20:18 +0300 Subject: [PATCH 15/78] ignite-ducktape: yandex cluster file. --- .gitignore | 1 - tests/docker/build/cluster.json | 64 +++++++++++++++++++++++++++++++++ tests/docker/build/node_hosts | 5 +++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 tests/docker/build/cluster.json create mode 100644 tests/docker/build/node_hosts diff --git a/.gitignore b/.gitignore index fee7d47ee5293..18bcd81cbf26a 100644 --- a/.gitignore +++ b/.gitignore @@ -111,4 +111,3 @@ packages /results .ducktape *.pyc -tests/docker/build/* diff --git a/tests/docker/build/cluster.json b/tests/docker/build/cluster.json new file mode 100644 index 0000000000000..4c15a52ac5b50 --- /dev/null +++ b/tests/docker/build/cluster.json @@ -0,0 +1,64 @@ +{ + "_comment": [ + "Licensed to the Apache Software Foundation (ASF) under one or more", + "contributor license agreements. See the NOTICE file distributed with", + "this work for additional information regarding copyright ownership.", + "The ASF licenses this file to You under the Apache License, Version 2.0", + "(the \"License\"); you may not use this file except in compliance with", + "the License. You may obtain a copy of the License at", + "", + "http://www.apache.org/licenses/LICENSE-2.0", + "", + "Unless required by applicable law or agreed to in writing, software", + "distributed under the License is distributed on an \"AS IS\" BASIS,", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", + "See the License for the specific language governing permissions and", + "limitations under the License." + ], + "nodes": [ + { + "externally_routable_ip": "130.193.62.38", + "ssh_config": { + "host": "130.193.62.38", + "hostname": "ignite-1", + "identityfile": "/home/ducker/.ssh/id_rsa_yandex", + "password": "", + "port": 22, + "user": "nizhikov" + } + }, + { + "externally_routable_ip": "84.201.189.216", + "ssh_config": { + "host": "84.201.189.216", + "hostname": "ignite-2", + "identityfile": "/home/ducker/.ssh/id_rsa_yandex", + "password": "", + "port": 22, + "user": "nizhikov" + } + }, + { + "externally_routable_ip": "130.193.62.6", + "ssh_config": { + "host": "130.193.62.6", + "hostname": "ignite-3", + "identityfile": "/home/ducker/.ssh/id_rsa_yandex", + "password": "", + "port": 22, + "user": "nizhikov" + } + }, + { + "externally_routable_ip": "130.193.62.60", + "ssh_config": { + "host": "130.193.62.60", + "hostname": "ignite-4", + "identityfile": "/home/ducker/.ssh/id_rsa_yandex", + "password": "", + "port": 22, + "user": "nizhikov" + } + } + ] +} diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts new file mode 100644 index 0000000000000..805226f197108 --- /dev/null +++ b/tests/docker/build/node_hosts @@ -0,0 +1,5 @@ +130.193.62.16 ignite-coordinator +130.193.62.38 ignite-1 +84.201.189.216 ignite-2 +130.193.62.6 ignite-3 +130.193.62.60 ignite-4 \ No newline at end of file From 53755e9d3e004fa565fd75fb338bc03745e9d259 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Tue, 12 May 2020 18:33:54 +0300 Subject: [PATCH 16/78] ignite-ducktape: Spark integration test added. --- .gitignore | 1 + bin/include/build-classpath.sh | 27 ++-- modules/integration-tests/pom.xml | 141 ++++++++++++++++++ .../internal/test/IgniteApplication.java | 12 +- .../internal/test/SparkApplication.java | 111 ++++++++++++++ parent/pom.xml | 2 +- pom.xml | 1 + tests/docker/Dockerfile | 29 +++- tests/docker/build/cluster.json | 51 ++++--- tests/docker/build/node_hosts | 12 +- tests/docker/run_tests.sh | 2 +- .../ignitetest/ignite_utils/ignite_config.py | 7 +- .../services/{ignite => }/ignite.py | 1 + tests/ignitetest/services/ignite/__init__.py | 14 -- .../ignitetest/services/ignite_client_app.py | 76 +++++++--- tests/ignitetest/services/spark.py | 119 +++++++++++++++ .../suites/add_node_rebalance_test.py | 5 +- .../suites/spark_integration_test.py | 36 +++++ 18 files changed, 560 insertions(+), 87 deletions(-) create mode 100644 modules/integration-tests/pom.xml rename modules/{benchmarks => integration-tests}/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java (76%) create mode 100644 modules/integration-tests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java rename tests/ignitetest/services/{ignite => }/ignite.py (98%) delete mode 100644 tests/ignitetest/services/ignite/__init__.py create mode 100644 tests/ignitetest/services/spark.py create mode 100644 tests/ignitetest/suites/spark_integration_test.py diff --git a/.gitignore b/.gitignore index 18bcd81cbf26a..b0cc5b790e3db 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,4 @@ packages /results .ducktape *.pyc +/tests/venv diff --git a/bin/include/build-classpath.sh b/bin/include/build-classpath.sh index edfb2679b6890..9a43683c36f97 100644 --- a/bin/include/build-classpath.sh +++ b/bin/include/build-classpath.sh @@ -47,20 +47,25 @@ includeToClassPath() { for file in $1/* do - if [ -d ${file} ] && [ -d "${file}/target" ]; then - if [ -d "${file}/target/classes" ]; then - IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/classes - fi + if [[ -z "${EXCLUDE_MODULES}" ]] || [[ ${EXCLUDE_MODULES} != *"`basename $file`"* ]]; then + echo "$file included" + if [ -d ${file} ] && [ -d "${file}/target" ]; then + if [ -d "${file}/target/classes" ]; then + IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/classes + fi - if [[ -z "${EXCLUDE_TEST_CLASSES}" ]]; then - if [ -d "${file}/target/test-classes" ]; then - IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/test-classes - fi - fi + if [[ -z "${EXCLUDE_TEST_CLASSES}" ]]; then + if [ -d "${file}/target/test-classes" ]; then + IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/test-classes + fi + fi - if [ -d "${file}/target/libs" ]; then - IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/libs/* + if [ -d "${file}/target/libs" ]; then + IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/libs/* + fi fi + else + echo "$file excluded by EXCLUDE_MODULES settings" fi done diff --git a/modules/integration-tests/pom.xml b/modules/integration-tests/pom.xml new file mode 100644 index 0000000000000..47fd0803830ad --- /dev/null +++ b/modules/integration-tests/pom.xml @@ -0,0 +1,141 @@ + + + + + + + 4.0.0 + + + org.apache.ignite + ignite-parent + 1 + ../../parent + + + ignite-integration-tests + 2.9.0-SNAPSHOT + http://ignite.apache.org + + + + org.apache.ignite + ignite-core + ${project.version} + + + + org.apache.ignite + ignite-indexing + ${project.version} + + + + org.apache.ignite + ignite-spark + ${project.version} + + + + org.apache.spark + spark-core_2.11 + ${spark.version} + + + + org.apache.spark + spark-sql_2.11 + ${spark.version} + + + + org.apache.spark + spark-tags_2.11 + ${spark.version} + + + + org.apache.spark + spark-catalyst_2.11 + ${spark.version} + + + + org.apache.spark + spark-network-shuffle_2.11 + ${spark.version} + + + + org.apache.spark + spark-network-common_2.11 + ${spark.version} + + + + com.fasterxml.woodstox + woodstox-core + 5.0.3 + + + + org.codehaus.woodstox + stax2-api + 3.1.4 + + + + org.apache.htrace + htrace-core4 + 4.1.0-incubating + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + maven-dependency-plugin + + + copy-libs + test-compile + + copy-dependencies + + + org.apache.ignite + target/libs + compile + false + + + + + + + diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java b/modules/integration-tests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java similarity index 76% rename from modules/benchmarks/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java rename to modules/integration-tests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java index dffa891f01afe..1a480bbc69e5d 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java +++ b/modules/integration-tests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java @@ -21,13 +21,14 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.Ignition; +import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgnitionEx; import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; import org.apache.ignite.lang.IgniteBiTuple; public class IgniteApplication { - public static final String CONFIG_PATH = "/mnt/client_app/ignite-config.xml"; + public static final String CONFIG_PATH = "/mnt/client_app/ignite-client-config.xml"; public static void main(String[] args) throws IgniteCheckedException { IgniteBiTuple cfgs = IgnitionEx.loadConfiguration(CONFIG_PATH); @@ -46,7 +47,16 @@ public static void main(String[] args) throws IgniteCheckedException { cache.put(i, i); } + executeSql(cache, "CREATE TABLE person(id INT, fio VARCHAR, PRIMARY KEY(id))"); + executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 1, "Ivanov Ivan"); + executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 2, "Petrov Petr"); + executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 3, "Sidorov Sidr"); + System.out.println("Ignite Client Finish."); } } + + private static void executeSql(IgniteCache cache, String query, Object...args) { + cache.query(new SqlFieldsQuery(query).setArgs(args)).getAll(); + } } diff --git a/modules/integration-tests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java b/modules/integration-tests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java new file mode 100644 index 0000000000000..8a88079857921 --- /dev/null +++ b/modules/integration-tests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.test; + +import org.apache.ignite.spark.IgniteDataFrameSettings; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.ignite.IgniteSparkSession; + +import static org.apache.ignite.internal.test.IgniteApplication.CONFIG_PATH; + +public class SparkApplication { + public static final String HOME = "/opt/ignite-dev"; + + public static final String VER = "2.9.0-SNAPSHOT"; + + public static final String SPRING_VER = "4.3.26.RELEASE"; + + public static void main(String[] args) { + System.out.println("SparkApplication.main - args"); + for (String arg : args) + System.out.println("SparkApplication.main - " + arg); + + sparkSession(args[0]); + + igniteSession(args[0]); + + System.out.println("Ignite Client Finish."); + } + + private static void sparkSession(String masterUrl) { + //Creating spark session. + try (SparkSession spark = SparkSession.builder() + .appName("SparkApplication") + .master(masterUrl) + .getOrCreate()) { + spark.sparkContext().addJar(HOME + "/modules/core/target/ignite-core-" + VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spring/target/ignite-spring-" + VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/log4j/target/ignite-log4j-" + VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spark/target/ignite-spark-" + VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/indexing/target/ignite-indexing-" + VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-beans-" + SPRING_VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-core-" + SPRING_VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-context-" + SPRING_VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-expression-" + SPRING_VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/core/target/libs/cache-api-1.0.0.jar"); + spark.sparkContext().addJar(HOME + "/modules/indexing/target/libs/h2-1.4.197.jar"); + + sparkDSLExample(spark); + } + } + + private static void igniteSession(String masterUrl) { + //Creating spark session. + try (IgniteSparkSession spark = IgniteSparkSession.builder() + .appName("SparkApplication") + .igniteConfig(CONFIG_PATH) + .master(masterUrl) + .getOrCreate()) { + spark.sparkContext().addJar(HOME + "/modules/core/target/ignite-core-" + VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spring/target/ignite-spring-" + VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/log4j/target/ignite-log4j-" + VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spark/target/ignite-spark-" + VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/indexing/target/ignite-indexing-" + VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-beans-" + SPRING_VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-core-" + SPRING_VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-context-" + SPRING_VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-expression-" + SPRING_VER + ".jar"); + spark.sparkContext().addJar(HOME + "/modules/core/target/libs/cache-api-1.0.0.jar"); + spark.sparkContext().addJar(HOME + "/modules/indexing/target/libs/h2-1.4.197.jar"); + + spark.catalog().listTables().show(); + + sparkDSLExample(spark); + } + } + + private static void sparkDSLExample(SparkSession spark) { + System.out.println("Querying using Spark DSL."); + + Dataset igniteDF = spark.read() + .format(IgniteDataFrameSettings.FORMAT_IGNITE()) //Data source type. + .option(IgniteDataFrameSettings.OPTION_TABLE(), "person") //Table to read. + .option(IgniteDataFrameSettings.OPTION_CONFIG_FILE(), CONFIG_PATH) //Ignite config. + .load(); + + System.out.println("Data frame schema:"); + + igniteDF.printSchema(); //Printing query schema to console. + + System.out.println("Data frame content:"); + + igniteDF.show(); //Printing query results to console. + } +} diff --git a/parent/pom.xml b/parent/pom.xml index 929e7f78bd023..1fdccb4d31b81 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -123,7 +123,7 @@ 1.6.4 1.1.7.2 2.6.5 - 2.3.0 + 2.3.4 1.13.23.RELEASE 4.3.26.RELEASE 2.0.13.RELEASE diff --git a/pom.xml b/pom.xml index 325f383843744..e56fd2f7679c5 100644 --- a/pom.xml +++ b/pom.xml @@ -617,6 +617,7 @@ benchmarks modules/benchmarks + modules/integration-tests diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index c94e9f300d5f0..593806bc7dec6 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -42,17 +42,30 @@ COPY ./ssh-config /root/.ssh/config RUN ssh-keygen -m PEM -q -t rsa -N '' -f /root/.ssh/id_rsa && cp -f /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config -# Install binary test dependencies. -# we use the same versions as in vagrant/base.sh -ARG IGNITE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" -ARG IGNITE_NAME="ignite-2.8.0" -ARG RELEASE_NAME="apache-ignite-2.8.0-bin" RUN chmod a+wr /opt -RUN curl -s "$IGNITE_MIRROR/ignite/2.8.0/$RELEASE_NAME.zip" > /opt/$RELEASE_NAME.zip -RUN cd /opt && unzip $RELEASE_NAME.zip && rm $RELEASE_NAME.zip -RUN mv /opt/$RELEASE_NAME /opt/$IGNITE_NAME + +ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" + +# Install binary test dependencies. +ARG IGNITE_VERSION="2.8.0" +ARG IGNITE_NAME="ignite-$IGNITE_VERSION" +ARG IGNITE_RELEASE_NAME="apache-ignite-$IGNITE_VERSION-bin" + +ADD $APACHE_MIRROR/ignite/$IGNITE_VERSION/$IGNITE_RELEASE_NAME.zip /opt/ +RUN cd /opt && unzip $IGNITE_RELEASE_NAME.zip && rm $IGNITE_RELEASE_NAME.zip +RUN mv /opt/$IGNITE_RELEASE_NAME /opt/$IGNITE_NAME RUN chmod a+wr /opt/$IGNITE_NAME -R +# Install spark +ARG SPARK_VERSION="2.3.4" +ARG SPARK_NAME="spark-$SPARK_VERSION" +ARG SPARK_RELEASE_NAME="spark-$SPARK_VERSION-bin-hadoop2.7" + +ADD $APACHE_MIRROR/spark/$SPARK_NAME/$SPARK_RELEASE_NAME.tgz /opt/ +RUN cd /opt && tar xvf $SPARK_RELEASE_NAME.tgz && rm $SPARK_RELEASE_NAME.tgz +RUN mv /opt/$SPARK_RELEASE_NAME /opt/$SPARK_NAME +RUN chmod a+wr /opt/$SPARK_NAME -R + # The version of Kibosh to use for testing. # If you update this, also update vagrant/base.sh ARG KIBOSH_VERSION="8841dd392e6fbf02986e2fb1f1ebf04df344b65a" diff --git a/tests/docker/build/cluster.json b/tests/docker/build/cluster.json index 4c15a52ac5b50..4332532b80485 100644 --- a/tests/docker/build/cluster.json +++ b/tests/docker/build/cluster.json @@ -17,47 +17,58 @@ ], "nodes": [ { - "externally_routable_ip": "130.193.62.38", + "externally_routable_ip": "ducker02", "ssh_config": { - "host": "130.193.62.38", - "hostname": "ignite-1", - "identityfile": "/home/ducker/.ssh/id_rsa_yandex", + "host": "ducker02", + "hostname": "ducker02", + "identityfile": "/home/ducker/.ssh/id_rsa", "password": "", "port": 22, - "user": "nizhikov" + "user": "ducker" } }, { - "externally_routable_ip": "84.201.189.216", + "externally_routable_ip": "ducker03", "ssh_config": { - "host": "84.201.189.216", - "hostname": "ignite-2", - "identityfile": "/home/ducker/.ssh/id_rsa_yandex", + "host": "ducker03", + "hostname": "ducker03", + "identityfile": "/home/ducker/.ssh/id_rsa", "password": "", "port": 22, - "user": "nizhikov" + "user": "ducker" } }, { - "externally_routable_ip": "130.193.62.6", + "externally_routable_ip": "ducker04", "ssh_config": { - "host": "130.193.62.6", - "hostname": "ignite-3", - "identityfile": "/home/ducker/.ssh/id_rsa_yandex", + "host": "ducker04", + "hostname": "ducker04", + "identityfile": "/home/ducker/.ssh/id_rsa", "password": "", "port": 22, - "user": "nizhikov" + "user": "ducker" } }, { - "externally_routable_ip": "130.193.62.60", + "externally_routable_ip": "ducker05", "ssh_config": { - "host": "130.193.62.60", - "hostname": "ignite-4", - "identityfile": "/home/ducker/.ssh/id_rsa_yandex", + "host": "ducker05", + "hostname": "ducker05", + "identityfile": "/home/ducker/.ssh/id_rsa", "password": "", "port": 22, - "user": "nizhikov" + "user": "ducker" + } + }, + { + "externally_routable_ip": "ducker06", + "ssh_config": { + "host": "ducker06", + "hostname": "ducker06", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" } } ] diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts index 805226f197108..173c478cbfe11 100644 --- a/tests/docker/build/node_hosts +++ b/tests/docker/build/node_hosts @@ -1,5 +1,7 @@ -130.193.62.16 ignite-coordinator -130.193.62.38 ignite-1 -84.201.189.216 ignite-2 -130.193.62.6 ignite-3 -130.193.62.60 ignite-4 \ No newline at end of file +172.18.0.2 ducker01 +172.18.0.3 ducker02 +172.18.0.4 ducker03 +172.18.0.5 ducker04 +172.18.0.6 ducker05 +172.18.0.7 ducker06 +.60 ignite-4 \ No newline at end of file diff --git a/tests/docker/run_tests.sh b/tests/docker/run_tests.sh index 9de23173a590d..477ec8224703a 100755 --- a/tests/docker/run_tests.sh +++ b/tests/docker/run_tests.sh @@ -16,7 +16,7 @@ # limitations under the License. SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-5} +IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-6} TC_PATHS=${TC_PATHS:-./ignitetest/} die() { diff --git a/tests/ignitetest/ignite_utils/ignite_config.py b/tests/ignitetest/ignite_utils/ignite_config.py index 534df42d55fde..f8bd92b6c4def 100644 --- a/tests/ignitetest/ignite_utils/ignite_config.py +++ b/tests/ignitetest/ignite_utils/ignite_config.py @@ -22,7 +22,7 @@ class IgniteConfig: def __init__(self, project="ignite"): self.project = project - def render(self, config_dir, work_dir): + def render(self, config_dir, work_dir, client_mode="false"): return """ + - """.format(config_dir=config_dir, work_dir=work_dir) + """.format(config_dir=config_dir, + work_dir=work_dir, + client_mode=client_mode) def render_log4j(self, work_dir): return """ diff --git a/tests/ignitetest/services/ignite/ignite.py b/tests/ignitetest/services/ignite.py similarity index 98% rename from tests/ignitetest/services/ignite/ignite.py rename to tests/ignitetest/services/ignite.py index 1107ffe49d453..f864f88c438d6 100644 --- a/tests/ignitetest/services/ignite/ignite.py +++ b/tests/ignitetest/services/ignite.py @@ -16,6 +16,7 @@ import os.path import signal +from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.services.service import Service from ducktape.utils.util import wait_until diff --git a/tests/ignitetest/services/ignite/__init__.py b/tests/ignitetest/services/ignite/__init__.py deleted file mode 100644 index ec2014340d78f..0000000000000 --- a/tests/ignitetest/services/ignite/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/ignitetest/services/ignite_client_app.py b/tests/ignitetest/services/ignite_client_app.py index 82a661ae96e01..228e76b9203d4 100644 --- a/tests/ignitetest/services/ignite_client_app.py +++ b/tests/ignitetest/services/ignite_client_app.py @@ -32,7 +32,7 @@ class IgniteClientApp(BackgroundThreadService): PERSISTENT_ROOT = "/mnt/client_app" STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") WORK_DIR = os.path.join(PERSISTENT_ROOT, "work") - CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-config.xml") + CLIENT_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-client-config.xml") LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-log4j.xml") logs = { @@ -41,17 +41,19 @@ class IgniteClientApp(BackgroundThreadService): "collect_default": True} } - def __init__(self, context, version=DEV_BRANCH, num_nodes=1): + def __init__(self, context, java_class_name, version=DEV_BRANCH, num_nodes=1): """ Args: num_nodes: number of nodes to use (this should be 1) """ BackgroundThreadService.__init__(self, context, num_nodes) - self.stop_timeout_sec = 10 self.log_level = "DEBUG" self.config = IgniteConfig() self.path = IgnitePath() + self.java_class_name = java_class_name + self.timeout_sec = 60 + self.stop_timeout_sec = 10 for node in self.nodes: node.version = version @@ -59,16 +61,11 @@ def __init__(self, context, version=DEV_BRANCH, num_nodes=1): def start_cmd(self, node): """Return the start command appropriate for the given node.""" - jvm_opts = "-J-DIGNITE_SUCCESS_FILE=" + IgniteClientApp.PERSISTENT_ROOT + "/success_file " - jvm_opts += "-J-Dlog4j.configDebug=true" - - cmd = "export MAIN_CLASS={main_class}; ".format(main_class=self.java_class_name()) - cmd += "export EXCLUDE_TEST_CLASSES=true; " - cmd += "export IGNITE_LOG_DIR=" + IgniteClientApp.PERSISTENT_ROOT + "; " + cmd = self.env() cmd += "%s %s %s 1>> %s 2>> %s " % \ (self.path.script("ignite.sh", node), - jvm_opts, - IgniteClientApp.CONFIG_FILE, + self.jvm_opts(), + self.app_args(), IgniteClientApp.STDOUT_STDERR_CAPTURE, IgniteClientApp.STDOUT_STDERR_CAPTURE) return cmd @@ -78,7 +75,7 @@ def start_node(self, node): def stop_node(self, node): self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) - node.account.kill_java_processes(self.java_class_name(), + node.account.kill_java_processes(self.java_class_name, clean_shutdown=True, allow_fail=True) @@ -91,29 +88,64 @@ def clean_node(self, node): self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." % (self.__class__.__name__, node.account)) - node.account.kill_java_processes(self.java_class_name(), + node.account.kill_java_processes(self.java_class_name, clean_shutdown=False, allow_fail=True) node.account.ssh("rm -rf %s" % IgniteClientApp.PERSISTENT_ROOT, allow_fail=False) def pids(self, node): - return node.account.java_pids(self.java_class_name()) + return node.account.java_pids(self.java_class_name) def alive(self, node): return len(self.pids(node)) > 0 def _worker(self, idx, node): - node.account.mkdirs(IgniteClientApp.PERSISTENT_ROOT) - node.account.create_file(IgniteClientApp.CONFIG_FILE, - self.config.render(IgniteClientApp.PERSISTENT_ROOT, IgniteClientApp.WORK_DIR)) - node.account.create_file(IgniteClientApp.LOG4J_CONFIG_FILE, - self.config.render_log4j(IgniteClientApp.WORK_DIR)) + create_client_configs(node, self.config) # Just run application. cmd = self.start_cmd(node) self.logger.info("Ignite client application command: %s", cmd) - node.account.ssh(cmd, allow_fail=False) - def java_class_name(self): - return "org.apache.ignite.internal.test.IgniteApplication" + with node.account.monitor_log(IgniteClientApp.STDOUT_STDERR_CAPTURE) as monitor: + node.account.ssh(cmd, allow_fail=False) + monitor.wait_until("Ignite Client Finish.", timeout_sec=self.timeout_sec, backoff_sec=5, + err_msg="Ignite client don't finish before timeout %s" % self.timeout_sec) + + def app_args(self): + return IgniteClientApp.CLIENT_CONFIG_FILE + + def jvm_opts(self): + return "-J-DIGNITE_SUCCESS_FILE=" + IgniteClientApp.PERSISTENT_ROOT + "/success_file " + \ + "-J-Dlog4j.configDebug=true " \ + "-J-Xmx1G" + + def env(self): + return "export MAIN_CLASS={main_class}; ".format(main_class=self.java_class_name) + \ + "export EXCLUDE_TEST_CLASSES=true; " + \ + "export IGNITE_LOG_DIR={log_dir}; ".format(log_dir=IgniteClientApp.PERSISTENT_ROOT) + + +class SparkIgniteClientApp(IgniteClientApp): + def __init__(self, context, master_node): + IgniteClientApp.__init__(self, context, java_class_name="org.apache.ignite.internal.test.SparkApplication") + self.master_node = master_node + self.timeout_sec = 120 + + def app_args(self): + return " spark://" + self.master_node.account.hostname + ":7077" + + def env(self): + return IgniteClientApp.env(self) + \ + "export EXCLUDE_MODULES=\"kubernetes,aws,gce,mesos,rest-http,web-agent,zookeeper,serializers,store," \ + "rocketmq\"; " + + +def create_client_configs(node, config): + node.account.mkdirs(IgniteClientApp.PERSISTENT_ROOT) + node.account.create_file(IgniteClientApp.CLIENT_CONFIG_FILE, + config.render(IgniteClientApp.PERSISTENT_ROOT, + IgniteClientApp.WORK_DIR, + "true")) + node.account.create_file(IgniteClientApp.LOG4J_CONFIG_FILE, + config.render_log4j(IgniteClientApp.WORK_DIR)) diff --git a/tests/ignitetest/services/spark.py b/tests/ignitetest/services/spark.py new file mode 100644 index 0000000000000..58f25a4ef434d --- /dev/null +++ b/tests/ignitetest/services/spark.py @@ -0,0 +1,119 @@ +import os.path + +from ducktape.cluster.remoteaccount import RemoteCommandError +from ducktape.services.service import Service + +from ignitetest.ignite_utils.ignite_config import IgniteConfig +from ignitetest.services.ignite_client_app import IgniteClientApp, create_client_configs + + +class SparkService(Service): + INSTALL_DIR = "/opt/spark-{version}".format(version="2.3.4") + PERSISTENT_ROOT = "/mnt/spark" + + logs = {} + + def __init__(self, context, num_nodes=3): + """ + :param context: test context + :param num_nodes: number of Ignite nodes. + """ + Service.__init__(self, context, num_nodes) + + self.log_level = "DEBUG" + self.ignite_config = IgniteConfig() + + for node in self.nodes: + self.logs["master_logs" + node.account.hostname] = { + "path": self.master_log_path(node), + "collect_default": True + } + self.logs["worker_logs" + node.account.hostname] = { + "path": self.slave_log_path(node), + "collect_default": True + } + + def start(self): + Service.start(self) + + self.logger.info("Waiting for Spark to start...") + + def start_cmd(self, node): + if node == self.nodes[0]: + script = "start-master.sh" + else: + script = "start-slave.sh spark://{spark_master}:7077".format(spark_master=self.nodes[0].account.hostname) + + start_script = os.path.join(SparkService.INSTALL_DIR, "sbin", script) + + cmd = "export SPARK_LOG_DIR={spark_dir}; ".format(spark_dir=SparkService.PERSISTENT_ROOT) + cmd += "export SPARK_WORKER_DIR={spark_dir}; ".format(spark_dir=SparkService.PERSISTENT_ROOT) + cmd += "{start_script} &".format(start_script=start_script) + + return cmd + + def start_node(self, node, timeout_sec=30): + create_client_configs(node, self.ignite_config) + + cmd = self.start_cmd(node) + self.logger.debug("Attempting to start SparkService on %s with command: %s" % (str(node.account), cmd)) + + if node == self.nodes[0]: + log_file = self.master_log_path(node) + log_msg = "Started REST server for submitting applications" + else: + log_file = self.slave_log_path(node) + log_msg = "Successfully registered with master" + + self.logger.debug("Monitoring - %s" % log_file) + + with node.account.monitor_log(log_file) as monitor: + node.account.ssh(cmd) + monitor.wait_until(log_msg, timeout_sec=timeout_sec, backoff_sec=5, + err_msg="Spark doesn't start at %d seconds" % timeout_sec) + + if len(self.pids(node)) == 0: + raise Exception("No process ids recorded on node %s" % node.account.hostname) + + def stop_node(self, node, clean_shutdown=True, timeout_sec=60): + if node == self.nodes[0]: + node.account.ssh(os.path.join(SparkService.INSTALL_DIR, "sbin", "stop-master.sh")) + else: + node.account.ssh(os.path.join(SparkService.INSTALL_DIR, "sbin", "stop-slave.sh")) + + def clean_node(self, node): + node.account.kill_java_processes(self.java_class_name(node), + clean_shutdown=False, allow_fail=True) + node.account.ssh("sudo rm -rf -- %s" % SparkService.PERSISTENT_ROOT, allow_fail=False) + + def pids(self, node): + """Return process ids associated with running processes on the given node.""" + try: + cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.java_class_name(node) + pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] + return pid_arr + except (RemoteCommandError, ValueError) as e: + return [] + + def java_class_name(self, node): + if node == self.nodes[0]: + return "org.apache.spark.deploy.master.Master" + else: + return "org.apache.spark.deploy.worker.Worker" + + def alive(self, node): + return len(self.pids(node)) > 0 + + def master_log_path(self, node): + return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.master.Master-{instance}-{host}.out".format( + SPARK_LOG_DIR=SparkService.PERSISTENT_ROOT, + userID=node.account.user, + instance=1, + host=node.account.hostname) + + def slave_log_path(self, node): + return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.worker.Worker-{instance}-{host}.out".format( + SPARK_LOG_DIR=SparkService.PERSISTENT_ROOT, + userID=node.account.user, + instance=1, + host=node.account.hostname) diff --git a/tests/ignitetest/suites/add_node_rebalance_test.py b/tests/ignitetest/suites/add_node_rebalance_test.py index 569ed7425415c..828ec5a560618 100644 --- a/tests/ignitetest/suites/add_node_rebalance_test.py +++ b/tests/ignitetest/suites/add_node_rebalance_test.py @@ -15,7 +15,7 @@ from ducktape.tests.test import Test -from ignitetest.services.ignite.ignite import IgniteService +from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_client_app import IgniteClientApp @@ -48,7 +48,8 @@ def test_add_node(self): self.logger.info("Start add node rebalance test.") # This client just put some data to the cache. - IgniteClientApp(self.test_context).run() + IgniteClientApp(self.test_context, + java_class_name="org.apache.ignite.internal.test.IgniteApplication").run() self.ignite.start_node(self.ignite.nodes[AddNodeRebalanceTest.NUM_NODES-1], timeout_sec=AddNodeRebalanceTest.REBALANCE_TIMEOUT, diff --git a/tests/ignitetest/suites/spark_integration_test.py b/tests/ignitetest/suites/spark_integration_test.py new file mode 100644 index 0000000000000..57c741cc38ed7 --- /dev/null +++ b/tests/ignitetest/suites/spark_integration_test.py @@ -0,0 +1,36 @@ +from ducktape.tests.test import Test + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_client_app import IgniteClientApp, SparkIgniteClientApp +from ignitetest.services.spark import SparkService + + +class SparkIntegrationTest(Test): + """ + Test performs: + 1. Start of Spark cluster. + 2. Start of Spark client application. + 3. Checks results of client application. + """ + + def __init__(self, test_context): + super(SparkIntegrationTest, self).__init__(test_context=test_context) + self.spark = SparkService(test_context, num_nodes=2) + self.ignite = IgniteService(test_context, num_nodes=1) + + def setUp(self): + # starting all nodes except last. + self.spark.start() + self.ignite.start() + + def teardown(self): + self.spark.stop() + self.ignite.stop() + + def test_spark_client(self): + self.logger.info("Spark integration test.") + + IgniteClientApp(self.test_context, + java_class_name="org.apache.ignite.internal.test.IgniteApplication").run() + + SparkIgniteClientApp(self.test_context, self.spark.nodes[0]).run() From 7d4f0d86e5cd57e8c405496f00977128fa1f4078 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Wed, 27 May 2020 15:07:57 +0300 Subject: [PATCH 17/78] naming -> ducktest (#7857) docker clean-up script codestyle fixes modules fix --- .../{integration-tests => ducktests}/pom.xml | 2 +- .../internal/test/IgniteApplication.java | 17 ++++++++++--- .../internal/test/SparkApplication.java | 24 ++++++++++++++++--- pom.xml | 9 ++++++- tests/docker/clean_up.sh | 19 +++++++++++++++ 5 files changed, 63 insertions(+), 8 deletions(-) rename modules/{integration-tests => ducktests}/pom.xml (98%) rename modules/{integration-tests => ducktests}/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java (90%) rename modules/{integration-tests => ducktests}/src/main/java/org/apache/ignite/internal/test/SparkApplication.java (93%) create mode 100644 tests/docker/clean_up.sh diff --git a/modules/integration-tests/pom.xml b/modules/ducktests/pom.xml similarity index 98% rename from modules/integration-tests/pom.xml rename to modules/ducktests/pom.xml index 47fd0803830ad..8e5d5b3c4ef0c 100644 --- a/modules/integration-tests/pom.xml +++ b/modules/ducktests/pom.xml @@ -30,7 +30,7 @@ ../../parent - ignite-integration-tests + ignite-ducktests 2.9.0-SNAPSHOT http://ignite.apache.org diff --git a/modules/integration-tests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java similarity index 90% rename from modules/integration-tests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java rename to modules/ducktests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java index 1a480bbc69e5d..d6d710fbbf97f 100644 --- a/modules/integration-tests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java @@ -27,9 +27,16 @@ import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; import org.apache.ignite.lang.IgniteBiTuple; +/** + * + */ public class IgniteApplication { + /** Config path. */ public static final String CONFIG_PATH = "/mnt/client_app/ignite-client-config.xml"; + /** + * @param args Args. + */ public static void main(String[] args) throws IgniteCheckedException { IgniteBiTuple cfgs = IgnitionEx.loadConfiguration(CONFIG_PATH); IgniteConfiguration cfg = cfgs.get1(); @@ -43,9 +50,8 @@ public static void main(String[] args) throws IgniteCheckedException { IgniteCache cache = ign.createCache("test-cache"); - for (int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) cache.put(i, i); - } executeSql(cache, "CREATE TABLE person(id INT, fio VARCHAR, PRIMARY KEY(id))"); executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 1, "Ivanov Ivan"); @@ -56,7 +62,12 @@ public static void main(String[] args) throws IgniteCheckedException { } } - private static void executeSql(IgniteCache cache, String query, Object...args) { + /** + * @param cache Cache. + * @param query Query. + * @param args Args. + */ + private static void executeSql(IgniteCache cache, String query, Object... args) { cache.query(new SqlFieldsQuery(query).setArgs(args)).getAll(); } } diff --git a/modules/integration-tests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java similarity index 93% rename from modules/integration-tests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java rename to modules/ducktests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java index 8a88079857921..fc41f7f6ed356 100644 --- a/modules/integration-tests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java @@ -25,13 +25,22 @@ import static org.apache.ignite.internal.test.IgniteApplication.CONFIG_PATH; +/** + * + */ public class SparkApplication { + /** Home. */ public static final String HOME = "/opt/ignite-dev"; + /** Version. */ public static final String VER = "2.9.0-SNAPSHOT"; + /** Spring version. */ public static final String SPRING_VER = "4.3.26.RELEASE"; + /** + * @param args Args. + */ public static void main(String[] args) { System.out.println("SparkApplication.main - args"); for (String arg : args) @@ -44,12 +53,15 @@ public static void main(String[] args) { System.out.println("Ignite Client Finish."); } + /** + * @param masterUrl Master url. + */ private static void sparkSession(String masterUrl) { //Creating spark session. try (SparkSession spark = SparkSession.builder() - .appName("SparkApplication") - .master(masterUrl) - .getOrCreate()) { + .appName("SparkApplication") + .master(masterUrl) + .getOrCreate()) { spark.sparkContext().addJar(HOME + "/modules/core/target/ignite-core-" + VER + ".jar"); spark.sparkContext().addJar(HOME + "/modules/spring/target/ignite-spring-" + VER + ".jar"); spark.sparkContext().addJar(HOME + "/modules/log4j/target/ignite-log4j-" + VER + ".jar"); @@ -66,6 +78,9 @@ private static void sparkSession(String masterUrl) { } } + /** + * @param masterUrl Master url. + */ private static void igniteSession(String masterUrl) { //Creating spark session. try (IgniteSparkSession spark = IgniteSparkSession.builder() @@ -91,6 +106,9 @@ private static void igniteSession(String masterUrl) { } } + /** + * @param spark Spark. + */ private static void sparkDSLExample(SparkSession spark) { System.out.println("Querying using Spark DSL."); diff --git a/pom.xml b/pom.xml index e56fd2f7679c5..fe71482557bc9 100644 --- a/pom.xml +++ b/pom.xml @@ -104,6 +104,7 @@ examples modules/benchmarks modules/compatibility + modules/ducktests modules/geospatial modules/hibernate-4.2 modules/hibernate-5.1 @@ -134,6 +135,13 @@ + + ducktests + + modules/ducktests + + + test @@ -617,7 +625,6 @@ benchmarks modules/benchmarks - modules/integration-tests diff --git a/tests/docker/clean_up.sh b/tests/docker/clean_up.sh new file mode 100644 index 0000000000000..34ce05d7ae019 --- /dev/null +++ b/tests/docker/clean_up.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker stop $(docker ps -a -q) +docker rm $(docker ps -a -q) \ No newline at end of file From dc98ee9df90b25eb5d928090b0e78b48cae2392e Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Thu, 28 May 2020 14:04:25 +0300 Subject: [PATCH 18/78] Ignite ducktape (#7866) * naming -> ducktest docker clean-up script codestyle fixes modules fix * tests folder relocated * build.sh ducktape 0.7.7 2.8.0 from archive py folder as source root apache license * docfix --- modules/ducktests/licenses/apache-2.0.txt | 202 ++++++++++++++++++ modules/ducktests/pom.xml | 19 ++ .../ducktests/tests}/MANIFEST.in | 0 .../ducktests/tests}/docker/Dockerfile | 5 +- .../tests}/docker/build/cluster.json | 0 .../ducktests/tests/docker/build/node_hosts | 7 + .../ducktests/tests}/docker/clean_up.sh | 0 .../ducktests/tests}/docker/ducker-ignite | 10 +- .../ducktests/tests}/docker/run_tests.sh | 0 .../ducktests/tests}/docker/ssh-config | 0 .../tests}/docker/ssh/authorized_keys | 0 .../ducktests/tests}/docker/ssh/config | 0 .../ducktests/tests}/docker/ssh/id_rsa | 0 .../ducktests/tests}/docker/ssh/id_rsa.pub | 0 .../ducktests/tests}/ignitetest/__init__.py | 0 .../ignitetest/ignite_utils/__init__.py | 0 .../ignitetest/ignite_utils/ignite_config.py | 0 .../ignitetest/ignite_utils/ignite_path.py | 0 .../tests}/ignitetest/services/__init__.py | 0 .../tests}/ignitetest/services/ignite.py | 0 .../ignitetest/services/ignite_client_app.py | 0 .../tests}/ignitetest/services/spark.py | 0 .../tests}/ignitetest/suites/__init__.py | 0 .../suites/add_node_rebalance_test.py | 0 .../suites/spark_integration_test.py | 0 .../ducktests/tests}/ignitetest/version.py | 0 {tests => modules/ducktests/tests}/setup.cfg | 0 {tests => modules/ducktests/tests}/setup.py | 2 +- scripts/build.sh | 25 +++ tests/docker/build/node_hosts | 7 - 30 files changed, 262 insertions(+), 15 deletions(-) create mode 100644 modules/ducktests/licenses/apache-2.0.txt rename {tests => modules/ducktests/tests}/MANIFEST.in (100%) rename {tests => modules/ducktests/tests}/docker/Dockerfile (96%) rename {tests => modules/ducktests/tests}/docker/build/cluster.json (100%) create mode 100644 modules/ducktests/tests/docker/build/node_hosts rename {tests => modules/ducktests/tests}/docker/clean_up.sh (100%) rename {tests => modules/ducktests/tests}/docker/ducker-ignite (97%) rename {tests => modules/ducktests/tests}/docker/run_tests.sh (100%) rename {tests => modules/ducktests/tests}/docker/ssh-config (100%) rename {tests => modules/ducktests/tests}/docker/ssh/authorized_keys (100%) rename {tests => modules/ducktests/tests}/docker/ssh/config (100%) rename {tests => modules/ducktests/tests}/docker/ssh/id_rsa (100%) rename {tests => modules/ducktests/tests}/docker/ssh/id_rsa.pub (100%) rename {tests => modules/ducktests/tests}/ignitetest/__init__.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/ignite_utils/__init__.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/ignite_utils/ignite_config.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/ignite_utils/ignite_path.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/services/__init__.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/services/ignite.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/services/ignite_client_app.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/services/spark.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/suites/__init__.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/suites/add_node_rebalance_test.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/suites/spark_integration_test.py (100%) rename {tests => modules/ducktests/tests}/ignitetest/version.py (100%) rename {tests => modules/ducktests/tests}/setup.cfg (100%) rename {tests => modules/ducktests/tests}/setup.py (96%) create mode 100755 scripts/build.sh delete mode 100644 tests/docker/build/node_hosts diff --git a/modules/ducktests/licenses/apache-2.0.txt b/modules/ducktests/licenses/apache-2.0.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/modules/ducktests/licenses/apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml index 8e5d5b3c4ef0c..d0aea24c5c7f3 100644 --- a/modules/ducktests/pom.xml +++ b/modules/ducktests/pom.xml @@ -136,6 +136,25 @@ + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-sources + generate-sources + + add-source + + + + tests + + + + + diff --git a/tests/MANIFEST.in b/modules/ducktests/tests/MANIFEST.in similarity index 100% rename from tests/MANIFEST.in rename to modules/ducktests/tests/MANIFEST.in diff --git a/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile similarity index 96% rename from tests/docker/Dockerfile rename to modules/ducktests/tests/docker/Dockerfile index 593806bc7dec6..790331cfe344f 100644 --- a/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -34,7 +34,7 @@ LABEL ducker.creator=$ducker_creator # Update Linux and install necessary utilities. RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute mc && apt-get -y clean RUN python -m pip install -U pip==9.0.3; -RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.6 +RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.7 # Set up ssh COPY ./ssh-config /root/.ssh/config @@ -45,13 +45,14 @@ RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config RUN chmod a+wr /opt ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" +ARG APACHE_ARCHIVE="https://archive.apache.org/dist/" # Install binary test dependencies. ARG IGNITE_VERSION="2.8.0" ARG IGNITE_NAME="ignite-$IGNITE_VERSION" ARG IGNITE_RELEASE_NAME="apache-ignite-$IGNITE_VERSION-bin" -ADD $APACHE_MIRROR/ignite/$IGNITE_VERSION/$IGNITE_RELEASE_NAME.zip /opt/ +ADD $APACHE_ARCHIVE/ignite/$IGNITE_VERSION/$IGNITE_RELEASE_NAME.zip /opt/ RUN cd /opt && unzip $IGNITE_RELEASE_NAME.zip && rm $IGNITE_RELEASE_NAME.zip RUN mv /opt/$IGNITE_RELEASE_NAME /opt/$IGNITE_NAME RUN chmod a+wr /opt/$IGNITE_NAME -R diff --git a/tests/docker/build/cluster.json b/modules/ducktests/tests/docker/build/cluster.json similarity index 100% rename from tests/docker/build/cluster.json rename to modules/ducktests/tests/docker/build/cluster.json diff --git a/modules/ducktests/tests/docker/build/node_hosts b/modules/ducktests/tests/docker/build/node_hosts new file mode 100644 index 0000000000000..035d22b4ca6ae --- /dev/null +++ b/modules/ducktests/tests/docker/build/node_hosts @@ -0,0 +1,7 @@ +172.26.0.2 ducker01 +172.26.0.3 ducker02 +172.26.0.4 ducker03 +172.26.0.5 ducker04 +172.26.0.6 ducker05 +172.26.0.7 ducker06 +.60 ignite-4 \ No newline at end of file diff --git a/tests/docker/clean_up.sh b/modules/ducktests/tests/docker/clean_up.sh similarity index 100% rename from tests/docker/clean_up.sh rename to modules/ducktests/tests/docker/clean_up.sh diff --git a/tests/docker/ducker-ignite b/modules/ducktests/tests/docker/ducker-ignite similarity index 97% rename from tests/docker/ducker-ignite rename to modules/ducktests/tests/docker/ducker-ignite index c2f2cf8f279c5..806e5295bca60 100755 --- a/tests/docker/ducker-ignite +++ b/modules/ducktests/tests/docker/ducker-ignite @@ -29,7 +29,7 @@ script_path="${0}" ducker_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # The absolute path to the root Ignite directory -ignite_dir="$( cd "${ducker_dir}/../.." && pwd )" +ignite_dir="$( cd "${ducker_dir}/../../../.." && pwd )" # The memory consumption to allow during the docker build. # This does not include swap. @@ -270,7 +270,7 @@ setup_custom_ducktape() { docker_run ducker01 "${image_name}" local running_container="$(docker ps -f=network=ducknet -q)" must_do -v -o docker cp "${custom_ducktape}" "${running_container}:/opt/ducktape" - docker exec --user=root ducker01 bash -c 'set -x && cd /opt/ignite-dev/tests && sudo python ./setup.py develop install && cd /opt/ducktape && sudo python ./setup.py develop install' + docker exec --user=root ducker01 bash -c 'set -x && cd /opt/ignite-dev/modules/ducktests/tests && sudo python ./setup.py develop install && cd /opt/ducktape && sudo python ./setup.py develop install' [[ $? -ne 0 ]] && die "failed to install the new ducktape." must_do -v -o docker commit ducker01 "${image_name}" must_do -v docker kill "${running_container}" @@ -352,7 +352,7 @@ attempting to start new ones." for n in $(seq -f %02g 1 ${num_nodes}); do local node="ducker${n}" docker exec --user=root "${node}" \ - bash -c "grep -v ${node} /opt/ignite-dev/tests/docker/build/node_hosts >> /etc/hosts" + bash -c "grep -v ${node} /opt/ignite-dev/modules/ducktests/tests/docker/build/node_hosts >> /etc/hosts" [[ $? -ne 0 ]] && die "failed to append to the /etc/hosts file on ${node}" done @@ -432,7 +432,7 @@ ducker_test() { local regex=".*\/ignitetest\/(.*)" if [[ $arg =~ $regex ]]; then local ignpath=${BASH_REMATCH[1]} - args="${args} ./tests/ignitetest/${ignpath}" + args="${args} ./modules/ducktests/tests/ignitetest/${ignpath}" else args="${args} ${arg}" fi @@ -440,7 +440,7 @@ ducker_test() { must_pushd "${ignite_dir}" #(test mvn) && mvn package -DskipTests -Dmaven.javadoc.skip=true -Plgpl,-examples,-clean-libs,-release,-scala,-clientDocs must_popd - cmd="cd /opt/ignite-dev && ducktape --cluster-file /opt/ignite-dev/tests/docker/build/cluster.json $args" + cmd="cd /opt/ignite-dev && ducktape --cluster-file /opt/ignite-dev/modules/ducktests/tests/docker/build/cluster.json $args" echo "docker exec ducker01 bash -c \"${cmd}\"" exec docker exec --user=ducker ducker01 bash -c "${cmd}" } diff --git a/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh similarity index 100% rename from tests/docker/run_tests.sh rename to modules/ducktests/tests/docker/run_tests.sh diff --git a/tests/docker/ssh-config b/modules/ducktests/tests/docker/ssh-config similarity index 100% rename from tests/docker/ssh-config rename to modules/ducktests/tests/docker/ssh-config diff --git a/tests/docker/ssh/authorized_keys b/modules/ducktests/tests/docker/ssh/authorized_keys similarity index 100% rename from tests/docker/ssh/authorized_keys rename to modules/ducktests/tests/docker/ssh/authorized_keys diff --git a/tests/docker/ssh/config b/modules/ducktests/tests/docker/ssh/config similarity index 100% rename from tests/docker/ssh/config rename to modules/ducktests/tests/docker/ssh/config diff --git a/tests/docker/ssh/id_rsa b/modules/ducktests/tests/docker/ssh/id_rsa similarity index 100% rename from tests/docker/ssh/id_rsa rename to modules/ducktests/tests/docker/ssh/id_rsa diff --git a/tests/docker/ssh/id_rsa.pub b/modules/ducktests/tests/docker/ssh/id_rsa.pub similarity index 100% rename from tests/docker/ssh/id_rsa.pub rename to modules/ducktests/tests/docker/ssh/id_rsa.pub diff --git a/tests/ignitetest/__init__.py b/modules/ducktests/tests/ignitetest/__init__.py similarity index 100% rename from tests/ignitetest/__init__.py rename to modules/ducktests/tests/ignitetest/__init__.py diff --git a/tests/ignitetest/ignite_utils/__init__.py b/modules/ducktests/tests/ignitetest/ignite_utils/__init__.py similarity index 100% rename from tests/ignitetest/ignite_utils/__init__.py rename to modules/ducktests/tests/ignitetest/ignite_utils/__init__.py diff --git a/tests/ignitetest/ignite_utils/ignite_config.py b/modules/ducktests/tests/ignitetest/ignite_utils/ignite_config.py similarity index 100% rename from tests/ignitetest/ignite_utils/ignite_config.py rename to modules/ducktests/tests/ignitetest/ignite_utils/ignite_config.py diff --git a/tests/ignitetest/ignite_utils/ignite_path.py b/modules/ducktests/tests/ignitetest/ignite_utils/ignite_path.py similarity index 100% rename from tests/ignitetest/ignite_utils/ignite_path.py rename to modules/ducktests/tests/ignitetest/ignite_utils/ignite_path.py diff --git a/tests/ignitetest/services/__init__.py b/modules/ducktests/tests/ignitetest/services/__init__.py similarity index 100% rename from tests/ignitetest/services/__init__.py rename to modules/ducktests/tests/ignitetest/services/__init__.py diff --git a/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py similarity index 100% rename from tests/ignitetest/services/ignite.py rename to modules/ducktests/tests/ignitetest/services/ignite.py diff --git a/tests/ignitetest/services/ignite_client_app.py b/modules/ducktests/tests/ignitetest/services/ignite_client_app.py similarity index 100% rename from tests/ignitetest/services/ignite_client_app.py rename to modules/ducktests/tests/ignitetest/services/ignite_client_app.py diff --git a/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py similarity index 100% rename from tests/ignitetest/services/spark.py rename to modules/ducktests/tests/ignitetest/services/spark.py diff --git a/tests/ignitetest/suites/__init__.py b/modules/ducktests/tests/ignitetest/suites/__init__.py similarity index 100% rename from tests/ignitetest/suites/__init__.py rename to modules/ducktests/tests/ignitetest/suites/__init__.py diff --git a/tests/ignitetest/suites/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/suites/add_node_rebalance_test.py similarity index 100% rename from tests/ignitetest/suites/add_node_rebalance_test.py rename to modules/ducktests/tests/ignitetest/suites/add_node_rebalance_test.py diff --git a/tests/ignitetest/suites/spark_integration_test.py b/modules/ducktests/tests/ignitetest/suites/spark_integration_test.py similarity index 100% rename from tests/ignitetest/suites/spark_integration_test.py rename to modules/ducktests/tests/ignitetest/suites/spark_integration_test.py diff --git a/tests/ignitetest/version.py b/modules/ducktests/tests/ignitetest/version.py similarity index 100% rename from tests/ignitetest/version.py rename to modules/ducktests/tests/ignitetest/version.py diff --git a/tests/setup.cfg b/modules/ducktests/tests/setup.cfg similarity index 100% rename from tests/setup.cfg rename to modules/ducktests/tests/setup.cfg diff --git a/tests/setup.py b/modules/ducktests/tests/setup.py similarity index 96% rename from tests/setup.py rename to modules/ducktests/tests/setup.py index 3e477167e1c7e..e82ed2bae75fd 100644 --- a/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -51,7 +51,7 @@ def run_tests(self): license="apache2.0", packages=find_packages(), include_package_data=True, - install_requires=["ducktape==0.7.6", "requests==2.20.0"], + install_requires=["ducktape==0.7.7", "requests==2.20.0"], tests_require=["pytest", "mock"], cmdclass={'test': PyTest} ) diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000000000..333ba9e046373 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Builds project. +# Run in Ignite sources root directory. +# Usage: ./build.sh +# + +mvn clean package -Pall-java,all-scala -DskipTests -Dmaven.javadoc.skip=true \ No newline at end of file diff --git a/tests/docker/build/node_hosts b/tests/docker/build/node_hosts deleted file mode 100644 index 173c478cbfe11..0000000000000 --- a/tests/docker/build/node_hosts +++ /dev/null @@ -1,7 +0,0 @@ -172.18.0.2 ducker01 -172.18.0.3 ducker02 -172.18.0.4 ducker03 -172.18.0.5 ducker04 -172.18.0.6 ducker05 -172.18.0.7 ducker06 -.60 ignite-4 \ No newline at end of file From 4bfa3aabc6b648216a38370fa78e62b273de146c Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Thu, 28 May 2020 17:35:56 +0300 Subject: [PATCH 19/78] Ignite ducktape (#7868) * naming -> ducktest docker clean-up script codestyle fixes modules fix * tests folder relocated * build.sh ducktape 0.7.7 2.8.0 from archive py folder as source root apache license * docfix * travis fix (rat) build -> gitignore * newline %) * minor package refactoring --- .gitignore | 2 + .../ducktests/tests/docker/build/cluster.json | 75 ------------------- .../ducktests/tests/docker/build/node_hosts | 7 -- .../{ignite_utils => benchmarks}/__init__.py | 0 .../add_node_rebalance_test.py | 0 .../tests/ignitetest/services/ignite.py | 4 +- .../ignitetest/services/ignite_client_app.py | 4 +- .../tests/ignitetest/services/spark.py | 17 ++++- .../{suites => services/utils}/__init__.py | 0 .../utils}/ignite_config.py | 0 .../utils}/ignite_path.py | 0 .../tests/ignitetest/tests/__init__.py | 14 ++++ .../spark_integration_test.py | 15 ++++ parent/pom.xml | 1 + 14 files changed, 52 insertions(+), 87 deletions(-) delete mode 100644 modules/ducktests/tests/docker/build/cluster.json delete mode 100644 modules/ducktests/tests/docker/build/node_hosts rename modules/ducktests/tests/ignitetest/{ignite_utils => benchmarks}/__init__.py (100%) rename modules/ducktests/tests/ignitetest/{suites => benchmarks}/add_node_rebalance_test.py (100%) rename modules/ducktests/tests/ignitetest/{suites => services/utils}/__init__.py (100%) rename modules/ducktests/tests/ignitetest/{ignite_utils => services/utils}/ignite_config.py (100%) rename modules/ducktests/tests/ignitetest/{ignite_utils => services/utils}/ignite_path.py (100%) create mode 100644 modules/ducktests/tests/ignitetest/tests/__init__.py rename modules/ducktests/tests/ignitetest/{suites => tests}/spark_integration_test.py (59%) diff --git a/.gitignore b/.gitignore index b0cc5b790e3db..2269a3feb3b0a 100644 --- a/.gitignore +++ b/.gitignore @@ -108,7 +108,9 @@ packages #NodeJs files /modules/platforms/nodejs/node_modules +#Ducktape /results .ducktape *.pyc /tests/venv +modules/ducktests/tests/docker/build/** diff --git a/modules/ducktests/tests/docker/build/cluster.json b/modules/ducktests/tests/docker/build/cluster.json deleted file mode 100644 index 4332532b80485..0000000000000 --- a/modules/ducktests/tests/docker/build/cluster.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "_comment": [ - "Licensed to the Apache Software Foundation (ASF) under one or more", - "contributor license agreements. See the NOTICE file distributed with", - "this work for additional information regarding copyright ownership.", - "The ASF licenses this file to You under the Apache License, Version 2.0", - "(the \"License\"); you may not use this file except in compliance with", - "the License. You may obtain a copy of the License at", - "", - "http://www.apache.org/licenses/LICENSE-2.0", - "", - "Unless required by applicable law or agreed to in writing, software", - "distributed under the License is distributed on an \"AS IS\" BASIS,", - "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", - "See the License for the specific language governing permissions and", - "limitations under the License." - ], - "nodes": [ - { - "externally_routable_ip": "ducker02", - "ssh_config": { - "host": "ducker02", - "hostname": "ducker02", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker03", - "ssh_config": { - "host": "ducker03", - "hostname": "ducker03", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker04", - "ssh_config": { - "host": "ducker04", - "hostname": "ducker04", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker05", - "ssh_config": { - "host": "ducker05", - "hostname": "ducker05", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - }, - { - "externally_routable_ip": "ducker06", - "ssh_config": { - "host": "ducker06", - "hostname": "ducker06", - "identityfile": "/home/ducker/.ssh/id_rsa", - "password": "", - "port": 22, - "user": "ducker" - } - } - ] -} diff --git a/modules/ducktests/tests/docker/build/node_hosts b/modules/ducktests/tests/docker/build/node_hosts deleted file mode 100644 index 035d22b4ca6ae..0000000000000 --- a/modules/ducktests/tests/docker/build/node_hosts +++ /dev/null @@ -1,7 +0,0 @@ -172.26.0.2 ducker01 -172.26.0.3 ducker02 -172.26.0.4 ducker03 -172.26.0.5 ducker04 -172.26.0.6 ducker05 -172.26.0.7 ducker06 -.60 ignite-4 \ No newline at end of file diff --git a/modules/ducktests/tests/ignitetest/ignite_utils/__init__.py b/modules/ducktests/tests/ignitetest/benchmarks/__init__.py similarity index 100% rename from modules/ducktests/tests/ignitetest/ignite_utils/__init__.py rename to modules/ducktests/tests/ignitetest/benchmarks/__init__.py diff --git a/modules/ducktests/tests/ignitetest/suites/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/benchmarks/add_node_rebalance_test.py similarity index 100% rename from modules/ducktests/tests/ignitetest/suites/add_node_rebalance_test.py rename to modules/ducktests/tests/ignitetest/benchmarks/add_node_rebalance_test.py diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index f864f88c438d6..555905994fded 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -20,8 +20,8 @@ from ducktape.services.service import Service from ducktape.utils.util import wait_until -from ignitetest.ignite_utils.ignite_config import IgniteConfig -from ignitetest.ignite_utils.ignite_path import IgnitePath +from ignitetest.services.utils.ignite_config import IgniteConfig +from ignitetest.services.utils.ignite_path import IgnitePath from ignitetest.version import DEV_BRANCH diff --git a/modules/ducktests/tests/ignitetest/services/ignite_client_app.py b/modules/ducktests/tests/ignitetest/services/ignite_client_app.py index 228e76b9203d4..2474abc5db36e 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_client_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_client_app.py @@ -17,8 +17,8 @@ from ducktape.services.background_thread import BackgroundThreadService -from ignitetest.ignite_utils.ignite_config import IgniteConfig -from ignitetest.ignite_utils.ignite_path import IgnitePath +from ignitetest.services.utils.ignite_config import IgniteConfig +from ignitetest.services.utils.ignite_path import IgnitePath from ignitetest.version import DEV_BRANCH """ diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py index 58f25a4ef434d..9945fd590ef66 100644 --- a/modules/ducktests/tests/ignitetest/services/spark.py +++ b/modules/ducktests/tests/ignitetest/services/spark.py @@ -1,9 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os.path from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.services.service import Service -from ignitetest.ignite_utils.ignite_config import IgniteConfig +from ignitetest.services.utils.ignite_config import IgniteConfig from ignitetest.services.ignite_client_app import IgniteClientApp, create_client_configs diff --git a/modules/ducktests/tests/ignitetest/suites/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/__init__.py similarity index 100% rename from modules/ducktests/tests/ignitetest/suites/__init__.py rename to modules/ducktests/tests/ignitetest/services/utils/__init__.py diff --git a/modules/ducktests/tests/ignitetest/ignite_utils/ignite_config.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py similarity index 100% rename from modules/ducktests/tests/ignitetest/ignite_utils/ignite_config.py rename to modules/ducktests/tests/ignitetest/services/utils/ignite_config.py diff --git a/modules/ducktests/tests/ignitetest/ignite_utils/ignite_path.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py similarity index 100% rename from modules/ducktests/tests/ignitetest/ignite_utils/ignite_path.py rename to modules/ducktests/tests/ignitetest/services/utils/ignite_path.py diff --git a/modules/ducktests/tests/ignitetest/tests/__init__.py b/modules/ducktests/tests/ignitetest/tests/__init__.py new file mode 100644 index 0000000000000..ec2014340d78f --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/modules/ducktests/tests/ignitetest/suites/spark_integration_test.py b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py similarity index 59% rename from modules/ducktests/tests/ignitetest/suites/spark_integration_test.py rename to modules/ducktests/tests/ignitetest/tests/spark_integration_test.py index 57c741cc38ed7..a239a4a91ac34 100644 --- a/modules/ducktests/tests/ignitetest/suites/spark_integration_test.py +++ b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from ducktape.tests.test import Test from ignitetest.services.ignite import IgniteService diff --git a/parent/pom.xml b/parent/pom.xml index 1fdccb4d31b81..9567c1923e709 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -866,6 +866,7 @@ **/keystore/ca/*.txt.attr **/keystore/ca/*serial **/META-INF/services/** + **/id_rsa** .travis.yml .github/PULL_REQUEST_TEMPLATE.md From aeb5db9afe9b117d36ebf8590ff3df1df114f15b Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Thu, 16 Jul 2020 13:34:28 +0300 Subject: [PATCH 20/78] Duck is a duck (#7967) --- .../ducktest/DataGenerationApplication.java | 53 +++++ .../ducktest/LongTxStreamerApplication.java | 107 ++++++++++ .../SampleDataStreamerApplication.java | 61 ++++++ .../SingleKeyTxStreamerApplication.java | 85 ++++++++ .../{test => ducktest}/SparkApplication.java | 52 ++--- .../utils/IgniteApplicationService.java | 63 ++++++ .../utils/IgniteAwareApplication.java | 185 ++++++++++++++++++ .../utils/IgniteAwareApplicationService.java | 47 +++++ .../internal/test/IgniteApplication.java | 73 ------- .../src/main/resources/log4j.properties | 25 +++ modules/ducktests/tests/docker/Dockerfile | 18 +- modules/ducktests/tests/docker/ducker-ignite | 4 +- .../benchmarks/add_node_rebalance_test.py | 56 ------ .../tests/ignitetest/services/__init__.py | 1 - .../tests/ignitetest/services/ignite.py | 86 +++----- .../tests/ignitetest/services/ignite_app.py | 38 ++++ .../ignitetest/services/ignite_client_app.py | 151 -------------- .../ignitetest/services/ignite_spark_app.py | 31 +++ .../tests/ignitetest/services/spark.py | 26 ++- .../ignitetest/services/utils/ignite_aware.py | 105 ++++++++++ .../services/utils/ignite_aware_app.py | 108 ++++++++++ .../services/utils/ignite_config.py | 6 +- .../ignitetest/services/utils/ignite_path.py | 6 +- .../{ => tests}/benchmarks/__init__.py | 0 .../benchmarks/add_node_rebalance_test.py | 94 +++++++++ .../tests/benchmarks/pme_free_switch_test.py | 117 +++++++++++ .../tests/spark_integration_test.py | 30 ++- .../tests/ignitetest/tests/utils/__init__.py | 14 ++ .../ignitetest/tests/utils/ignite_test.py | 24 +++ modules/ducktests/tests/ignitetest/version.py | 5 + scripts/build-module.sh | 25 +++ scripts/build.sh | 2 +- 32 files changed, 1285 insertions(+), 413 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/DataGenerationApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SampleDataStreamerApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SingleKeyTxStreamerApplication.java rename modules/ducktests/src/main/java/org/apache/ignite/internal/{test => ducktest}/SparkApplication.java (85%) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java delete mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java create mode 100644 modules/ducktests/src/main/resources/log4j.properties delete mode 100644 modules/ducktests/tests/ignitetest/benchmarks/add_node_rebalance_test.py create mode 100644 modules/ducktests/tests/ignitetest/services/ignite_app.py delete mode 100644 modules/ducktests/tests/ignitetest/services/ignite_client_app.py create mode 100644 modules/ducktests/tests/ignitetest/services/ignite_spark_app.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py rename modules/ducktests/tests/ignitetest/{ => tests}/benchmarks/__init__.py (100%) create mode 100644 modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py create mode 100644 modules/ducktests/tests/ignitetest/tests/benchmarks/pme_free_switch_test.py create mode 100644 modules/ducktests/tests/ignitetest/tests/utils/__init__.py create mode 100644 modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py create mode 100755 scripts/build-module.sh diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/DataGenerationApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/DataGenerationApplication.java new file mode 100644 index 0000000000000..4c5ffcfd9dfdf --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/DataGenerationApplication.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * + */ +public class DataGenerationApplication extends IgniteAwareApplication { + /** + * @param ignite Ignite. + */ + public DataGenerationApplication(Ignite ignite) { + super(ignite); + } + + /** {@inheritDoc} */ + @Override protected void run(String[] args) { + log.info("Creating cache..."); + + IgniteCache cache = ignite.createCache(args[0]); + + try (IgniteDataStreamer stmr = ignite.dataStreamer(cache.getName())) { + for (int i = 0; i < Integer.parseInt(args[1]); i++) { + stmr.addData(i, i); + + if (i % 10_000 == 0) + log.info("Streamed " + i + " entries"); + } + } + + markSyncExecutionComplete(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java new file mode 100644 index 0000000000000..4bbc078abb0fd --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest; + +import java.util.Collection; +import java.util.concurrent.CountDownLatch; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionState; + +/** + * + */ +public class LongTxStreamerApplication extends IgniteAwareApplication { + /** Tx count. */ + private static final int TX_CNT = 100; + + /** Started. */ + private static final CountDownLatch started = new CountDownLatch(TX_CNT); + + /** + * @param ignite Ignite. + */ + public LongTxStreamerApplication(Ignite ignite) { + super(ignite); + } + + /** {@inheritDoc} */ + @Override public void run(String[] args) throws InterruptedException { + IgniteCache cache = ignite.getOrCreateCache(args[0]); + + log.info("Starting Long Tx..."); + + for (int i = 0; i < TX_CNT; i++) { + int finalI = i; + + new Thread(() -> { + Transaction tx = ignite.transactions().txStart(); + + cache.put(finalI, finalI); + + log.info("Long Tx started [key=" + finalI + "]"); + + started.countDown(); + + while (!terminated()) { + if (tx.state() != TransactionState.ACTIVE) { + log.info("Transaction broken. [key=" + finalI + "]"); + + break; + } + + try { + U.sleep(1000); + } + catch (IgniteInterruptedCheckedException ignored) { + // No-op. + } + } + + log.info("Stopping tx thread [state=" + tx.state() + "]"); + + }).start(); + } + + started.await(); + + markInitialized(); + + while (!terminated()) { + Collection active = + ((IgniteEx)ignite).context().cache().context().tm().activeTransactions(); + + log.info("Long Txs are in progress [txs=" + active.size() + "]"); + + try { + U.sleep(100); // Keeping node/txs alive. + } + catch (IgniteInterruptedCheckedException ignored) { + log.info("Waiting interrupted."); + } + } + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SampleDataStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SampleDataStreamerApplication.java new file mode 100644 index 0000000000000..281d202fccd70 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SampleDataStreamerApplication.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * + */ +public class SampleDataStreamerApplication extends IgniteAwareApplication { + /** + * @param ignite Ignite. + */ + public SampleDataStreamerApplication(Ignite ignite) { + super(ignite); + } + + /** + * @param cache Cache. + * @param qry Query. + * @param args Args. + */ + private static void executeSql(IgniteCache cache, String qry, Object... args) { + cache.query(new SqlFieldsQuery(qry).setArgs(args)).getAll(); + } + + /** {@inheritDoc} */ + @Override protected void run(String[] args) { + System.out.println("Creating cache..."); + + IgniteCache cache = ignite.createCache(args[0]); + + for (int i = 0; i < Integer.parseInt(args[1]); i++) + cache.put(i, i); + + executeSql(cache, "CREATE TABLE person(id INT, fio VARCHAR, PRIMARY KEY(id))"); + executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 1, "Ivanov Ivan"); + executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 2, "Petrov Petr"); + executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 3, "Sidorov Sidr"); + + markSyncExecutionComplete(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SingleKeyTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SingleKeyTxStreamerApplication.java new file mode 100644 index 0000000000000..bff4e7e920427 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SingleKeyTxStreamerApplication.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * + */ +public class SingleKeyTxStreamerApplication extends IgniteAwareApplication { + /** + * @param ignite Ignite. + */ + public SingleKeyTxStreamerApplication(Ignite ignite) { + super(ignite); + } + + /** {@inheritDoc} */ + @Override public void run(String[] args) { + IgniteCache cache = ignite.getOrCreateCache(args[0]); + + int warmup = Integer.parseInt(args[1]); + + long max = -1; + + int key = 10_000_000; + + int cnt = 0; + + long initTime = 0; + + boolean record = false; + + while (!terminated()) { + cnt++; + + long start = System.currentTimeMillis(); + + cache.put(key++, key); + + long finish = System.currentTimeMillis(); + + long time = finish - start; + + if (!record && cnt > warmup) { + record = true; + + initTime = System.currentTimeMillis();; + + markInitialized(); + } + + if (record) { + if (max < time) + max = time; + } + + if (cnt % 1000 == 0) + log.info("Streamed " + cnt + " transactions [max=" + max + "]"); + } + + recordResult("WORST_LATENCY", max); + recordResult("STREAMED", cnt - warmup); + recordResult("MEASURE_DURATION", System.currentTimeMillis() - initTime); + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SparkApplication.java similarity index 85% rename from modules/ducktests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java rename to modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SparkApplication.java index fc41f7f6ed356..0823a823ed046 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/test/SparkApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SparkApplication.java @@ -15,20 +15,19 @@ * limitations under the License. */ -package org.apache.ignite.internal.test; +package org.apache.ignite.internal.ducktest; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; import org.apache.ignite.spark.IgniteDataFrameSettings; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.ignite.IgniteSparkSession; -import static org.apache.ignite.internal.test.IgniteApplication.CONFIG_PATH; - /** * */ -public class SparkApplication { +public class SparkApplication extends IgniteAwareApplication { /** Home. */ public static final String HOME = "/opt/ignite-dev"; @@ -38,25 +37,10 @@ public class SparkApplication { /** Spring version. */ public static final String SPRING_VER = "4.3.26.RELEASE"; - /** - * @param args Args. - */ - public static void main(String[] args) { - System.out.println("SparkApplication.main - args"); - for (String arg : args) - System.out.println("SparkApplication.main - " + arg); - - sparkSession(args[0]); - - igniteSession(args[0]); - - System.out.println("Ignite Client Finish."); - } - /** * @param masterUrl Master url. */ - private static void sparkSession(String masterUrl) { + private static void sparkSession(String cfgPath, String masterUrl) { //Creating spark session. try (SparkSession spark = SparkSession.builder() .appName("SparkApplication") @@ -74,18 +58,19 @@ private static void sparkSession(String masterUrl) { spark.sparkContext().addJar(HOME + "/modules/core/target/libs/cache-api-1.0.0.jar"); spark.sparkContext().addJar(HOME + "/modules/indexing/target/libs/h2-1.4.197.jar"); - sparkDSLExample(spark); + sparkDSLExample(cfgPath, spark); } } /** * @param masterUrl Master url. + * @param cfgPath Config path. */ - private static void igniteSession(String masterUrl) { + private static void igniteSession(String cfgPath, String masterUrl) { //Creating spark session. try (IgniteSparkSession spark = IgniteSparkSession.builder() .appName("SparkApplication") - .igniteConfig(CONFIG_PATH) + .igniteConfig(cfgPath) .master(masterUrl) .getOrCreate()) { spark.sparkContext().addJar(HOME + "/modules/core/target/ignite-core-" + VER + ".jar"); @@ -102,20 +87,21 @@ private static void igniteSession(String masterUrl) { spark.catalog().listTables().show(); - sparkDSLExample(spark); + sparkDSLExample(cfgPath, spark); } } /** * @param spark Spark. + * @param cfgPath Config path. */ - private static void sparkDSLExample(SparkSession spark) { + private static void sparkDSLExample(String cfgPath, SparkSession spark) { System.out.println("Querying using Spark DSL."); Dataset igniteDF = spark.read() .format(IgniteDataFrameSettings.FORMAT_IGNITE()) //Data source type. .option(IgniteDataFrameSettings.OPTION_TABLE(), "person") //Table to read. - .option(IgniteDataFrameSettings.OPTION_CONFIG_FILE(), CONFIG_PATH) //Ignite config. + .option(IgniteDataFrameSettings.OPTION_CONFIG_FILE(), cfgPath) //Ignite config. .load(); System.out.println("Data frame schema:"); @@ -126,4 +112,18 @@ private static void sparkDSLExample(SparkSession spark) { igniteDF.show(); //Printing query results to console. } + + /** {@inheritDoc} */ + @Override protected void run(String[] args) throws Exception { + System.out.println("SparkApplication.main - args"); + + for (String arg : args) + System.out.println("SparkApplication.main - " + arg); + + sparkSession(args[0], args[1]); + + igniteSession(args[0], args[1]); + + markSyncExecutionComplete(); + } } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java new file mode 100644 index 0000000000000..a41f27c9f2a6c --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.utils; + +import java.util.Arrays; +import org.apache.ignite.Ignite; +import org.apache.ignite.Ignition; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgnitionEx; +import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +/** + * + */ +public class IgniteApplicationService { + /** Logger. */ + private static final Logger log = LogManager.getLogger(IgniteApplicationService.class.getName()); + + /** + * @param args Args. + */ + public static void main(String[] args) throws Exception { + log.info("Starting Application... [params=" + args[0] + "]"); + + String[] params = args[0].split(","); + + Class clazz = Class.forName(params[0]); + + IgniteBiTuple cfgs = IgnitionEx.loadConfiguration(params[1]); + + IgniteConfiguration cfg = cfgs.get1(); + + assert cfg.isClientMode(); + + log.info("Starting Ignite node..."); + + try (Ignite ignite = Ignition.start(cfg)) { + IgniteAwareApplication app = (IgniteAwareApplication)clazz.getConstructor(Ignite.class).newInstance(ignite); + + String[] appParams = Arrays.copyOfRange(params, 2, params.length); + + app.start(appParams); + } + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java new file mode 100644 index 0000000000000..525c76d717d39 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.utils; + +import java.util.Arrays; +import org.apache.ignite.Ignite; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +/** + * + */ +public abstract class IgniteAwareApplication { + /** Logger. */ + protected static final Logger log = LogManager.getLogger(IgniteAwareApplication.class.getName()); + + /** App inited. */ + private static final String APP_INITED = "IGNITE_APPLICATION_INITIALIZED"; + + /** App finished. */ + private static final String APP_FINISHED = "IGNITE_APPLICATION_FINISHED"; + + /** App terminated. */ + private static final String APP_TERMINATED = "IGNITE_APPLICATION_TERMINATED"; + + /** Inited. */ + private static volatile boolean inited; + + /** Finished. */ + private static volatile boolean finished; + + /** Terminated. */ + private static volatile boolean terminated; + + /** Ignite. */ + protected final Ignite ignite; + + /** + * Default constructor. + */ + protected IgniteAwareApplication() { + ignite = null; + } + + /** + * + */ + protected IgniteAwareApplication(Ignite ignite) { + this.ignite = ignite; + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + terminate(); + + while (!finished()) { + log.info("Waiting for graceful termnation."); + + try { + U.sleep(100); + } + catch (IgniteInterruptedCheckedException e) { + e.printStackTrace(); + } + } + + log.info("SIGTERM recorded."); + })); + + log.info("ShutdownHook registered."); + } + + /** + * Used to marks as started to perform actions. Suitable for async runs. + */ + protected void markInitialized() { + assert !inited; + + log.info(APP_INITED); + + inited = true; + } + + /** + * + */ + protected void markFinished() { + assert !finished; + + log.info(APP_FINISHED); + + finished = true; + } + + /** + * + */ + protected void markSyncExecutionComplete() { + markInitialized(); + markFinished(); + } + + /** + * + */ + private boolean finished() { + return finished; + } + + /** + * + */ + private void terminate() { + assert !terminated; + + log.info(APP_TERMINATED); + + terminated = true; + } + + /** + * + */ + protected boolean terminated() { + return terminated; + } + + /** + * @param name Name. + * @param val Value. + */ + protected void recordResult(String name, String val) { + assert !finished; + + log.info(name + "->" + val + "<-"); + } + + /** + * @param name Name. + * @param val Value. + */ + protected void recordResult(String name, long val) { + recordResult(name, String.valueOf(val)); + } + + /** + * + */ + protected abstract void run(String[] args) throws Exception; + + /** + * @param args Args. + */ + public void start(String[] args) { + try { + log.info("Application params: " + Arrays.toString(args)); + + run(args); + + assert inited : "Was not properly initialized."; + assert finished : "Was not properly finished."; + } + catch (Throwable th) { + log.error("Unexpected Application failure... ", th); + } + finally { + log.info("Application finished."); + } + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java new file mode 100644 index 0000000000000..a7836c2945f4e --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.utils; + +import java.util.Arrays; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +/** + * + */ +public class IgniteAwareApplicationService { + /** Logger. */ + private static final Logger log = LogManager.getLogger(IgniteAwareApplicationService.class.getName()); + + /** + * @param args Args. + */ + public static void main(String[] args) throws Exception { + log.info("Starting Application... [params=" + args[0] + "]"); + + String[] params = args[0].split(","); + + Class clazz = Class.forName(params[0]); + + IgniteAwareApplication app = (IgniteAwareApplication)clazz.getConstructor().newInstance(); + + String[] appParams = Arrays.copyOfRange(params, 1, params.length); + + app.start(appParams); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java deleted file mode 100644 index d6d710fbbf97f..0000000000000 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/test/IgniteApplication.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.test; - -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.Ignition; -import org.apache.ignite.cache.query.SqlFieldsQuery; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.IgnitionEx; -import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; -import org.apache.ignite.lang.IgniteBiTuple; - -/** - * - */ -public class IgniteApplication { - /** Config path. */ - public static final String CONFIG_PATH = "/mnt/client_app/ignite-client-config.xml"; - - /** - * @param args Args. - */ - public static void main(String[] args) throws IgniteCheckedException { - IgniteBiTuple cfgs = IgnitionEx.loadConfiguration(CONFIG_PATH); - IgniteConfiguration cfg = cfgs.get1(); - - cfg.setClientMode(true); - - System.out.println("Starting Ignite client..."); - - try (Ignite ign = Ignition.start(cfg)) { - System.out.println("Creating cache..."); - - IgniteCache cache = ign.createCache("test-cache"); - - for (int i = 0; i < 1000; i++) - cache.put(i, i); - - executeSql(cache, "CREATE TABLE person(id INT, fio VARCHAR, PRIMARY KEY(id))"); - executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 1, "Ivanov Ivan"); - executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 2, "Petrov Petr"); - executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 3, "Sidorov Sidr"); - - System.out.println("Ignite Client Finish."); - } - } - - /** - * @param cache Cache. - * @param query Query. - * @param args Args. - */ - private static void executeSql(IgniteCache cache, String query, Object... args) { - cache.query(new SqlFieldsQuery(query).setArgs(args)).getAll(); - } -} diff --git a/modules/ducktests/src/main/resources/log4j.properties b/modules/ducktests/src/main/resources/log4j.properties new file mode 100644 index 0000000000000..ecfe84af1dd3c --- /dev/null +++ b/modules/ducktests/src/main/resources/log4j.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601}][%-5p][%t][%c{1}] %m%n diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index 790331cfe344f..d083ff3e8c553 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -20,7 +20,7 @@ MAINTAINER Apache Ignite dev@ignite.apache.org VOLUME ["/opt/ignite-dev"] # Set the timezone. -ENV TZ="/usr/share/zoneinfo/America/Los_Angeles" +ENV TZ="/usr/share/zoneinfo/Europe/Moscow" # Do not ask for confirmations when running apt-get, etc. ENV DEBIAN_FRONTEND noninteractive @@ -42,28 +42,22 @@ COPY ./ssh-config /root/.ssh/config RUN ssh-keygen -m PEM -q -t rsa -N '' -f /root/.ssh/id_rsa && cp -f /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config -RUN chmod a+wr /opt - ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" ARG APACHE_ARCHIVE="https://archive.apache.org/dist/" # Install binary test dependencies. -ARG IGNITE_VERSION="2.8.0" -ARG IGNITE_NAME="ignite-$IGNITE_VERSION" -ARG IGNITE_RELEASE_NAME="apache-ignite-$IGNITE_VERSION-bin" +RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.7.6/apache-ignite-2.7.6-bin.zip && unzip apache-ignite-2.7.6-bin.zip && mv /opt/apache-ignite-2.7.6-bin /opt/ignite-2.7.6 +RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.8.0/apache-ignite-2.8.0-bin.zip && unzip apache-ignite-2.8.0-bin.zip && mv /opt/apache-ignite-2.8.0-bin /opt/ignite-2.8.0 +RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.8.1/apache-ignite-2.8.1-bin.zip && unzip apache-ignite-2.8.1-bin.zip && mv /opt/apache-ignite-2.8.1-bin /opt/ignite-2.8.1 -ADD $APACHE_ARCHIVE/ignite/$IGNITE_VERSION/$IGNITE_RELEASE_NAME.zip /opt/ -RUN cd /opt && unzip $IGNITE_RELEASE_NAME.zip && rm $IGNITE_RELEASE_NAME.zip -RUN mv /opt/$IGNITE_RELEASE_NAME /opt/$IGNITE_NAME -RUN chmod a+wr /opt/$IGNITE_NAME -R +RUN rm /opt/apache-ignite-*-bin.zip # Install spark ARG SPARK_VERSION="2.3.4" ARG SPARK_NAME="spark-$SPARK_VERSION" ARG SPARK_RELEASE_NAME="spark-$SPARK_VERSION-bin-hadoop2.7" -ADD $APACHE_MIRROR/spark/$SPARK_NAME/$SPARK_RELEASE_NAME.tgz /opt/ -RUN cd /opt && tar xvf $SPARK_RELEASE_NAME.tgz && rm $SPARK_RELEASE_NAME.tgz +RUN cd /opt && curl -O $APACHE_MIRROR/spark/$SPARK_NAME/$SPARK_RELEASE_NAME.tgz && tar xvf $SPARK_RELEASE_NAME.tgz && rm $SPARK_RELEASE_NAME.tgz RUN mv /opt/$SPARK_RELEASE_NAME /opt/$SPARK_NAME RUN chmod a+wr /opt/$SPARK_NAME -R diff --git a/modules/ducktests/tests/docker/ducker-ignite b/modules/ducktests/tests/docker/ducker-ignite index 806e5295bca60..e41212c7db1b7 100755 --- a/modules/ducktests/tests/docker/ducker-ignite +++ b/modules/ducktests/tests/docker/ducker-ignite @@ -197,7 +197,7 @@ must_do() { # Ask the user a yes/no question. # # $1: The prompt to use -# $_return: 0 if the user answered no; 1 if the user anМинус - создаст дополнительную нагрузку на core team.swered yes. +# $_return: 0 if the user answered no; 1 if the user answered yes. ask_yes_no() { local prompt="${1}" while true; do @@ -258,7 +258,7 @@ docker_run() { must_do -v docker run --privileged \ -d -t -h "${node}" --network ducknet "${expose_ports}" \ --memory=${docker_run_memory_limit} --memory-swappiness=1 \ - -v "${ignite_dir}:/opt/ignite-dev" --name "${node}" -- "${image_name}" + -v "${ignite_dir}:/opt/ignite-dev:delegated" --name "${node}" -- "${image_name}" } setup_custom_ducktape() { diff --git a/modules/ducktests/tests/ignitetest/benchmarks/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/benchmarks/add_node_rebalance_test.py deleted file mode 100644 index 828ec5a560618..0000000000000 --- a/modules/ducktests/tests/ignitetest/benchmarks/add_node_rebalance_test.py +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ducktape.tests.test import Test - -from ignitetest.services.ignite import IgniteService -from ignitetest.services.ignite_client_app import IgniteClientApp - - -class AddNodeRebalanceTest(Test): - NUM_NODES = 3 - REBALANCE_TIMEOUT = 600 - - """ - Test performs rebalance tests. - """ - def __init__(self, test_context): - super(AddNodeRebalanceTest, self).__init__(test_context=test_context) - self.ignite = IgniteService(test_context, num_nodes=AddNodeRebalanceTest.NUM_NODES) - - def setUp(self): - # starting all nodes except last. - for i in range(AddNodeRebalanceTest.NUM_NODES-1): - self.ignite.start_node(self.ignite.nodes[i]) - - def teardown(self): - self.ignite.stop() - - def test_add_node(self): - """ - Test performs add node rebalance test which consists of following steps: - * Start cluster. - * Put data to it via IgniteClientApp. - * Start one more node and awaits for rebalance to finish. - """ - self.logger.info("Start add node rebalance test.") - - # This client just put some data to the cache. - IgniteClientApp(self.test_context, - java_class_name="org.apache.ignite.internal.test.IgniteApplication").run() - - self.ignite.start_node(self.ignite.nodes[AddNodeRebalanceTest.NUM_NODES-1], - timeout_sec=AddNodeRebalanceTest.REBALANCE_TIMEOUT, - wait_for_rebalance = True) diff --git a/modules/ducktests/tests/ignitetest/services/__init__.py b/modules/ducktests/tests/ignitetest/services/__init__.py index e556dc9592384..ec2014340d78f 100644 --- a/modules/ducktests/tests/ignitetest/services/__init__.py +++ b/modules/ducktests/tests/ignitetest/services/__init__.py @@ -12,4 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index 555905994fded..07b562fe8450b 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -20,22 +20,17 @@ from ducktape.services.service import Service from ducktape.utils.util import wait_until -from ignitetest.services.utils.ignite_config import IgniteConfig -from ignitetest.services.utils.ignite_path import IgnitePath +from ignitetest.services.utils.ignite_aware import IgniteAwareService from ignitetest.version import DEV_BRANCH -class IgniteService(Service): - PERSISTENT_ROOT = "/mnt/ignite" - WORK_DIR = os.path.join(PERSISTENT_ROOT, "work") - CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-config.xml") - LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-log4j.xml") - HEAP_DUMP_FILE = os.path.join(PERSISTENT_ROOT, "ignite-heap.bin") - STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") +class IgniteService(IgniteAwareService): + APP_SERVICE_CLASS = "org.apache.ignite.startup.cmdline.CommandLineStartup" + HEAP_DUMP_FILE = os.path.join(IgniteAwareService.PERSISTENT_ROOT, "ignite-heap.bin") logs = { "console_log": { - "path": STDOUT_STDERR_CAPTURE, + "path": IgniteAwareService.STDOUT_STDERR_CAPTURE, "collect_default": True}, "heap_dump": { @@ -43,56 +38,35 @@ class IgniteService(Service): "collect_default": False} } - def __init__(self, context, num_nodes=3, version=DEV_BRANCH): - """ - :param context: test context - :param num_nodes: number of Ignite nodes. - """ - Service.__init__(self, context, num_nodes) + def __init__(self, context, num_nodes, version=DEV_BRANCH, properties=""): + IgniteAwareService.__init__(self, context, num_nodes, version, properties) - self.log_level = "DEBUG" - self.config = IgniteConfig() - self.path = IgnitePath() - - for node in self.nodes: - node.version = version - - def start(self): + def start(self, timeout_sec=180): Service.start(self) - self.logger.info("Waiting for Ignite to start...") + self.logger.info("Waiting for Ignite(s) to start...") + + for node in self.nodes: + self.await_node_stated(node, timeout_sec) def start_cmd(self, node): jvm_opts = "-J-DIGNITE_SUCCESS_FILE=" + IgniteService.PERSISTENT_ROOT + "/success_file " - jvm_opts += "-J-Dlog4j.configDebug=true" + jvm_opts += "-J-Dlog4j.configDebug=true " + jvm_opts += "-J-XX:+UnlockExperimentalVMOptions -J-XX:+UseCGroupMemoryLimitForHeap" # java8 docker fix cmd = "export EXCLUDE_TEST_CLASSES=true; " cmd += "export IGNITE_LOG_DIR=" + IgniteService.PERSISTENT_ROOT + "; " + cmd += "export USER_LIBS=%s/libs/optional/ignite-log4j/*; " % self.path.home(self.version) cmd += "%s %s %s 1>> %s 2>> %s &" % \ - (self.path.script("ignite.sh", node), - jvm_opts, - IgniteService.CONFIG_FILE, - IgniteService.STDOUT_STDERR_CAPTURE, - IgniteService.STDOUT_STDERR_CAPTURE) + (self.path.script("ignite.sh", node), + jvm_opts, + IgniteService.CONFIG_FILE, + IgniteService.STDOUT_STDERR_CAPTURE, + IgniteService.STDOUT_STDERR_CAPTURE) return cmd - def start_node(self, node, timeout_sec=180, wait_for_rebalance=False): - node.account.mkdirs(IgniteService.PERSISTENT_ROOT) - node.account.create_file(IgniteService.CONFIG_FILE, - self.config.render(IgniteService.PERSISTENT_ROOT, IgniteService.WORK_DIR)) - node.account.create_file(IgniteService.LOG4J_CONFIG_FILE, self.config.render_log4j(IgniteService.WORK_DIR)) - - cmd = self.start_cmd(node) - self.logger.debug("Attempting to start IgniteService on %s with command: %s" % (str(node.account), cmd)) - - wait_for_message = "Topology snapshot" - if wait_for_rebalance: - wait_for_message = "Completed (final) rebalancing \[grp=test-cache" - - with node.account.monitor_log(IgniteService.STDOUT_STDERR_CAPTURE) as monitor: - node.account.ssh(cmd) - monitor.wait_until(wait_for_message, timeout_sec=timeout_sec, backoff_sec=5, - err_msg="Ignite server didn't finish startup in %d seconds" % timeout_sec) + def await_node_stated(self, node, timeout_sec): + self.await_event_on_node("Topology snapshot", node, timeout_sec, from_the_beginning=True) if len(self.pids(node)) == 0: raise Exception("No process ids recorded on node %s" % node.account.hostname) @@ -112,8 +86,7 @@ def stop_node(self, node, clean_shutdown=True, timeout_sec=60): raise def clean_node(self, node): - node.account.kill_java_processes(self.java_class_name(), - clean_shutdown=False, allow_fail=True) + node.account.kill_java_processes(self.APP_SERVICE_CLASS, clean_shutdown=False, allow_fail=True) node.account.ssh("sudo rm -rf -- %s" % IgniteService.PERSISTENT_ROOT, allow_fail=False) def thread_dump(self, node): @@ -126,19 +99,8 @@ def thread_dump(self, node): def pids(self, node): """Return process ids associated with running processes on the given node.""" try: - cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.java_class_name() + cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.APP_SERVICE_CLASS pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] return pid_arr except (RemoteCommandError, ValueError) as e: return [] - - def java_class_name(self): - return "org.apache.ignite.startup.cmdline.CommandLineStartup" - - def set_version(self, version): - for node in self.nodes: - node.version = version - - def alive(self, node): - return len(self.pids(node)) > 0 - diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py new file mode 100644 index 0000000000000..87cb8e1215975 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ducktape.services.service import Service + +from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService +from ignitetest.version import DEV_BRANCH + +""" +The Ignite application service allows to perform custom logic writen on java. +""" + + +class IgniteApplicationService(IgniteAwareApplicationService): + def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60): + IgniteAwareApplicationService.__init__( + self, context, java_class_name, version, properties, params, timeout_sec, + service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteApplicationService") + + def start(self): + Service.start(self) + + self.logger.info("Waiting for Ignite Application (%s) to start..." % self.java_class_name) + + self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True) + self.await_event("IGNITE_APPLICATION_INITIALIZED", self.timeout_sec, from_the_beginning=True) diff --git a/modules/ducktests/tests/ignitetest/services/ignite_client_app.py b/modules/ducktests/tests/ignitetest/services/ignite_client_app.py deleted file mode 100644 index 2474abc5db36e..0000000000000 --- a/modules/ducktests/tests/ignitetest/services/ignite_client_app.py +++ /dev/null @@ -1,151 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from ducktape.services.background_thread import BackgroundThreadService - -from ignitetest.services.utils.ignite_config import IgniteConfig -from ignitetest.services.utils.ignite_path import IgnitePath -from ignitetest.version import DEV_BRANCH - -""" -The Ignite client application is a main class that implements custom logic. -First CMD param is an absolute path to the Ignite config file. -""" - - -class IgniteClientApp(BackgroundThreadService): - # Root directory for persistent output - PERSISTENT_ROOT = "/mnt/client_app" - STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") - WORK_DIR = os.path.join(PERSISTENT_ROOT, "work") - CLIENT_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-client-config.xml") - LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-log4j.xml") - - logs = { - "console_log": { - "path": STDOUT_STDERR_CAPTURE, - "collect_default": True} - } - - def __init__(self, context, java_class_name, version=DEV_BRANCH, num_nodes=1): - """ - Args: - num_nodes: number of nodes to use (this should be 1) - """ - BackgroundThreadService.__init__(self, context, num_nodes) - - self.log_level = "DEBUG" - self.config = IgniteConfig() - self.path = IgnitePath() - self.java_class_name = java_class_name - self.timeout_sec = 60 - self.stop_timeout_sec = 10 - - for node in self.nodes: - node.version = version - - def start_cmd(self, node): - """Return the start command appropriate for the given node.""" - - cmd = self.env() - cmd += "%s %s %s 1>> %s 2>> %s " % \ - (self.path.script("ignite.sh", node), - self.jvm_opts(), - self.app_args(), - IgniteClientApp.STDOUT_STDERR_CAPTURE, - IgniteClientApp.STDOUT_STDERR_CAPTURE) - return cmd - - def start_node(self, node): - BackgroundThreadService.start_node(self, node) - - def stop_node(self, node): - self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) - node.account.kill_java_processes(self.java_class_name, - clean_shutdown=True, - allow_fail=True) - - stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec) - assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ - (str(node.account), str(self.stop_timeout_sec)) - - def clean_node(self, node): - if self.alive(node): - self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." % - (self.__class__.__name__, node.account)) - - node.account.kill_java_processes(self.java_class_name, - clean_shutdown=False, - allow_fail=True) - - node.account.ssh("rm -rf %s" % IgniteClientApp.PERSISTENT_ROOT, allow_fail=False) - - def pids(self, node): - return node.account.java_pids(self.java_class_name) - - def alive(self, node): - return len(self.pids(node)) > 0 - - def _worker(self, idx, node): - create_client_configs(node, self.config) - - # Just run application. - cmd = self.start_cmd(node) - self.logger.info("Ignite client application command: %s", cmd) - - with node.account.monitor_log(IgniteClientApp.STDOUT_STDERR_CAPTURE) as monitor: - node.account.ssh(cmd, allow_fail=False) - monitor.wait_until("Ignite Client Finish.", timeout_sec=self.timeout_sec, backoff_sec=5, - err_msg="Ignite client don't finish before timeout %s" % self.timeout_sec) - - def app_args(self): - return IgniteClientApp.CLIENT_CONFIG_FILE - - def jvm_opts(self): - return "-J-DIGNITE_SUCCESS_FILE=" + IgniteClientApp.PERSISTENT_ROOT + "/success_file " + \ - "-J-Dlog4j.configDebug=true " \ - "-J-Xmx1G" - - def env(self): - return "export MAIN_CLASS={main_class}; ".format(main_class=self.java_class_name) + \ - "export EXCLUDE_TEST_CLASSES=true; " + \ - "export IGNITE_LOG_DIR={log_dir}; ".format(log_dir=IgniteClientApp.PERSISTENT_ROOT) - - -class SparkIgniteClientApp(IgniteClientApp): - def __init__(self, context, master_node): - IgniteClientApp.__init__(self, context, java_class_name="org.apache.ignite.internal.test.SparkApplication") - self.master_node = master_node - self.timeout_sec = 120 - - def app_args(self): - return " spark://" + self.master_node.account.hostname + ":7077" - - def env(self): - return IgniteClientApp.env(self) + \ - "export EXCLUDE_MODULES=\"kubernetes,aws,gce,mesos,rest-http,web-agent,zookeeper,serializers,store," \ - "rocketmq\"; " - - -def create_client_configs(node, config): - node.account.mkdirs(IgniteClientApp.PERSISTENT_ROOT) - node.account.create_file(IgniteClientApp.CLIENT_CONFIG_FILE, - config.render(IgniteClientApp.PERSISTENT_ROOT, - IgniteClientApp.WORK_DIR, - "true")) - node.account.create_file(IgniteClientApp.LOG4J_CONFIG_FILE, - config.render_log4j(IgniteClientApp.WORK_DIR)) diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py new file mode 100644 index 0000000000000..dffe7ace7f48a --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The Ignite-Spark application service. +""" +from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService +from ignitetest.version import DEV_BRANCH + + +class SparkIgniteApplicationService(IgniteAwareApplicationService): + def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60): + IgniteAwareApplicationService.__init__( + self, context, java_class_name, version, properties, params, timeout_sec) + + def env(self): + return IgniteAwareApplicationService.env(self) + \ + "export EXCLUDE_MODULES=\"kubernetes,aws,gce,mesos,rest-http,web-agent,zookeeper,serializers,store," \ + "rocketmq\"; " diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py index 9945fd590ef66..731bcd5cd24a7 100644 --- a/modules/ducktests/tests/ignitetest/services/spark.py +++ b/modules/ducktests/tests/ignitetest/services/spark.py @@ -18,22 +18,23 @@ from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.services.service import Service +from ignitetest.services.utils.ignite_aware import IgniteAwareService from ignitetest.services.utils.ignite_config import IgniteConfig -from ignitetest.services.ignite_client_app import IgniteClientApp, create_client_configs +from ignitetest.version import DEV_BRANCH -class SparkService(Service): +class SparkService(IgniteAwareService): INSTALL_DIR = "/opt/spark-{version}".format(version="2.3.4") - PERSISTENT_ROOT = "/mnt/spark" + SPARK_PERSISTENT_ROOT = "/mnt/spark" logs = {} - def __init__(self, context, num_nodes=3): + def __init__(self, context, version=DEV_BRANCH, num_nodes=3, properties=""): """ :param context: test context :param num_nodes: number of Ignite nodes. """ - Service.__init__(self, context, num_nodes) + IgniteAwareService.__init__(self, context, num_nodes, version, properties) self.log_level = "DEBUG" self.ignite_config = IgniteConfig() @@ -61,14 +62,14 @@ def start_cmd(self, node): start_script = os.path.join(SparkService.INSTALL_DIR, "sbin", script) - cmd = "export SPARK_LOG_DIR={spark_dir}; ".format(spark_dir=SparkService.PERSISTENT_ROOT) - cmd += "export SPARK_WORKER_DIR={spark_dir}; ".format(spark_dir=SparkService.PERSISTENT_ROOT) + cmd = "export SPARK_LOG_DIR={spark_dir}; ".format(spark_dir=SparkService.SPARK_PERSISTENT_ROOT) + cmd += "export SPARK_WORKER_DIR={spark_dir}; ".format(spark_dir=SparkService.SPARK_PERSISTENT_ROOT) cmd += "{start_script} &".format(start_script=start_script) return cmd def start_node(self, node, timeout_sec=30): - create_client_configs(node, self.ignite_config) + self.init_persistent(node) cmd = self.start_cmd(node) self.logger.debug("Attempting to start SparkService on %s with command: %s" % (str(node.account), cmd)) @@ -99,7 +100,7 @@ def stop_node(self, node, clean_shutdown=True, timeout_sec=60): def clean_node(self, node): node.account.kill_java_processes(self.java_class_name(node), clean_shutdown=False, allow_fail=True) - node.account.ssh("sudo rm -rf -- %s" % SparkService.PERSISTENT_ROOT, allow_fail=False) + node.account.ssh("sudo rm -rf -- %s" % SparkService.SPARK_PERSISTENT_ROOT, allow_fail=False) def pids(self, node): """Return process ids associated with running processes on the given node.""" @@ -116,19 +117,16 @@ def java_class_name(self, node): else: return "org.apache.spark.deploy.worker.Worker" - def alive(self, node): - return len(self.pids(node)) > 0 - def master_log_path(self, node): return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.master.Master-{instance}-{host}.out".format( - SPARK_LOG_DIR=SparkService.PERSISTENT_ROOT, + SPARK_LOG_DIR=SparkService.SPARK_PERSISTENT_ROOT, userID=node.account.user, instance=1, host=node.account.hostname) def slave_log_path(self, node): return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.worker.Worker-{instance}-{host}.out".format( - SPARK_LOG_DIR=SparkService.PERSISTENT_ROOT, + SPARK_LOG_DIR=SparkService.SPARK_PERSISTENT_ROOT, userID=node.account.user, instance=1, host=node.account.hostname) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py new file mode 100644 index 0000000000000..dd82febfd6583 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from abc import abstractmethod + +from ducktape.services.background_thread import BackgroundThreadService + +from ignitetest.services.utils.ignite_config import IgniteConfig +from ignitetest.services.utils.ignite_path import IgnitePath + +""" +The base class to build services aware of Ignite. +""" + + +class IgniteAwareService(BackgroundThreadService): + # Root directory for persistent output + PERSISTENT_ROOT = "/mnt/service" + STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") + WORK_DIR = os.path.join(PERSISTENT_ROOT, "work") + CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-config.xml") + LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-log4j.xml") + + logs = { + "console_log": { + "path": STDOUT_STDERR_CAPTURE, + "collect_default": True} + } + + def __init__(self, context, num_nodes, version, properties): + BackgroundThreadService.__init__(self, context, num_nodes) + + self.log_level = "DEBUG" + self.config = IgniteConfig() + self.path = IgnitePath() + self.properties = properties + self.version = version + + for node in self.nodes: + node.version = version + + def start_node(self, node): + self.init_persistent(node) + + BackgroundThreadService.start_node(self, node) + + def init_persistent(self, node): + node.account.mkdirs(self.PERSISTENT_ROOT) + node.account.create_file(self.CONFIG_FILE, self.config.render( + self.PERSISTENT_ROOT, self.WORK_DIR, properties=self.properties)) + node.account.create_file(self.LOG4J_CONFIG_FILE, self.config.render_log4j(self.WORK_DIR)) + + @abstractmethod + def start_cmd(self, node): + raise NotImplementedError + + @abstractmethod + def pids(self, node): + raise NotImplementedError + + def _worker(self, idx, node): + cmd = self.start_cmd(node) + + self.logger.debug("Attempting to start Application Service on %s with command: %s" % (str(node.account), cmd)) + + node.account.ssh(cmd) + + def alive(self, node): + return len(self.pids(node)) > 0 + + def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning=False, backoff_sec=5): + with node.account.monitor_log(self.STDOUT_STDERR_CAPTURE) as monitor: + if from_the_beginning: + monitor.offset = 0 + + monitor.wait_until(evt_message, timeout_sec=timeout_sec, backoff_sec=backoff_sec, + err_msg="Event [%s] was not triggered in %d seconds" % (evt_message, timeout_sec)) + + def await_event(self, evt_message, timeout_sec, from_the_beginning=False, backoff_sec=5): + assert len(self.nodes) == 1 + + self.await_event_on_node(evt_message, self.nodes[0], timeout_sec, from_the_beginning=from_the_beginning, + backoff_sec=backoff_sec) + + def execute(self, command): + for node in self.nodes: + cmd = "%s 1>> %s 2>> %s" % \ + (self.path.script(command, node), + self.STDOUT_STDERR_CAPTURE, + self.STDOUT_STDERR_CAPTURE) + + node.account.ssh(cmd) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py new file mode 100644 index 0000000000000..f569c120ecece --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +from ducktape.services.service import Service + +from ignitetest.services.utils.ignite_aware import IgniteAwareService + +""" +The base class to build Ignite aware application written on java. +""" + + +class IgniteAwareApplicationService(IgniteAwareService): + def __init__(self, context, java_class_name, version, properties, params, timeout_sec, + service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): + IgniteAwareService.__init__(self, context, 1, version, properties) + + self.servicejava_class_name = service_java_class_name + self.java_class_name = java_class_name + self.timeout_sec = timeout_sec + self.stop_timeout_sec = 10 + self.params = params + + def start(self): + Service.start(self) + + self.logger.info("Waiting for Ignite aware Application (%s) to start..." % self.java_class_name) + + self.await_event("IGNITE_APPLICATION_INITIALIZED", self.timeout_sec, from_the_beginning=True) + + def start_cmd(self, node): + cmd = self.env() + cmd += "%s %s %s 1>> %s 2>> %s &" % \ + (self.path.script("ignite.sh", node), + self.jvm_opts(), + self.app_args(), + self.STDOUT_STDERR_CAPTURE, + self.STDOUT_STDERR_CAPTURE) + return cmd + + def stop_node(self, node, clean_shutdown=True, timeout_sec=20): + self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) + node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=True, allow_fail=True) + + stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec) + assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ + (str(node.account), str(self.stop_timeout_sec)) + + self.await_event("IGNITE_APPLICATION_FINISHED", from_the_beginning=True, timeout_sec=timeout_sec) + + def clean_node(self, node): + if self.alive(node): + self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." % + (self.__class__.__name__, node.account)) + + node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=False, allow_fail=True) + + node.account.ssh("rm -rf %s" % self.PERSISTENT_ROOT, allow_fail=False) + + def app_args(self): + args = self.java_class_name + "," + IgniteAwareApplicationService.CONFIG_FILE + + if self.params != "": + args += "," + self.params + + return args + + def pids(self, node): + return node.account.java_pids(self.servicejava_class_name) + + def jvm_opts(self): + return "-J-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file " + \ + "-J-Dlog4j.configDebug=true " \ + "-J-Xmx1G " \ + "-J-ea " \ + "-J-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false " \ + "-J-XX:+UnlockExperimentalVMOptions -J-XX:+UseCGroupMemoryLimitForHeap" # java8 docker fix + + def env(self): + return "export MAIN_CLASS={main_class}; ".format(main_class=self.servicejava_class_name) + \ + "export EXCLUDE_TEST_CLASSES=true; " + \ + "export IGNITE_LOG_DIR={log_dir}; ".format(log_dir=self.PERSISTENT_ROOT) + \ + "export USER_LIBS=%s/libs/optional/ignite-log4j/*:/opt/ignite-dev/modules/ducktests/target/*; " \ + % self.path.home(self.version) + + def extract_result(self, name): + res = "" + + output = self.nodes[0].account.ssh_capture( + "grep '%s' %s" % (name + "->", self.STDOUT_STDERR_CAPTURE), allow_fail=False) + + for line in output: + res = re.search("%s(.*)%s" % (name + "->", "<-"), line).group(1) + + return res diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py index f8bd92b6c4def..5ffaa81881b73 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py @@ -22,7 +22,7 @@ class IgniteConfig: def __init__(self, project="ignite"): self.project = project - def render(self, config_dir, work_dir, client_mode="false"): + def render(self, config_dir, work_dir, properties=""): return """ - + {properties} """.format(config_dir=config_dir, work_dir=work_dir, - client_mode=client_mode) + properties=properties) def render_log4j(self, work_dir): return """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py index 63071758a69c9..ed1a3fbcc1d33 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py @@ -15,7 +15,7 @@ import os -from ignitetest.version import get_version, IgniteVersion, DEV_BRANCH +from ignitetest.version import get_version, IgniteVersion """ This module provides Ignite path methods @@ -37,7 +37,7 @@ class IgnitePath: def __init__(self, project="ignite"): self.project = project - def home(self, node_or_version=DEV_BRANCH, project=None): + def home(self, node_or_version, project=None): version = self._version(node_or_version) home_dir = project or self.project if version is not None: @@ -45,7 +45,7 @@ def home(self, node_or_version=DEV_BRANCH, project=None): return os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir) - def script(self, script_name, node_or_version=DEV_BRANCH, project=None): + def script(self, script_name, node_or_version, project=None): version = self._version(node_or_version) return os.path.join(self.home(version, project=project), "bin", script_name) diff --git a/modules/ducktests/tests/ignitetest/benchmarks/__init__.py b/modules/ducktests/tests/ignitetest/tests/benchmarks/__init__.py similarity index 100% rename from modules/ducktests/tests/ignitetest/benchmarks/__init__.py rename to modules/ducktests/tests/ignitetest/tests/benchmarks/__init__.py diff --git a/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py new file mode 100644 index 0000000000000..2c725d728c833 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import time + +from ducktape.mark import parametrize +from ducktape.mark.resource import cluster + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.tests.utils.ignite_test import IgniteTest +from ignitetest.version import DEV_BRANCH, IgniteVersion, LATEST + + +class AddNodeRebalanceTest(IgniteTest): + NUM_NODES = 4 + PRELOAD_TIMEOUT = 60 + DATA_AMOUNT = 1000000 + REBALANCE_TIMEOUT = 60 + + """ + Test performs rebalance tests. + """ + + @staticmethod + def properties(client_mode="false"): + return """ + + """.format(client_mode=client_mode) + + def __init__(self, test_context): + super(AddNodeRebalanceTest, self).__init__(test_context=test_context) + + def setUp(self): + pass + + def teardown(self): + pass + + @cluster(num_nodes=NUM_NODES + 1) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST)) + def test_add_node(self, version): + """ + Test performs add node rebalance test which consists of following steps: + * Start cluster. + * Put data to it via IgniteClientApp. + * Start one more node and awaits for rebalance to finish. + """ + ignite_version = IgniteVersion(version) + + self.stage("Start Ignite nodes") + + ignites = IgniteService(self.test_context, num_nodes=AddNodeRebalanceTest.NUM_NODES - 1, version=ignite_version) + + ignites.start() + + self.stage("Starting DataGenerationApplication") + + # This client just put some data to the cache. + IgniteApplicationService(self.test_context, + java_class_name="org.apache.ignite.internal.ducktest.DataGenerationApplication", + properties=self.properties(client_mode="true"), + version=ignite_version, + params="test-cache,%d" % self.DATA_AMOUNT, + timeout_sec=self.PRELOAD_TIMEOUT).run() + + ignite = IgniteService(self.test_context, num_nodes=1, version=ignite_version) + + self.stage("Starting Ignite node") + + ignite.start() + + start = time.time() + + ignite.await_event("rebalanced=true, wasRebalanced=false", + timeout_sec=AddNodeRebalanceTest.REBALANCE_TIMEOUT, + from_the_beginning=True, + backoff_sec=1) + + data = {"Rebalanced in (sec)": time.time() - start} + + return data diff --git a/modules/ducktests/tests/ignitetest/tests/benchmarks/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/benchmarks/pme_free_switch_test.py new file mode 100644 index 0000000000000..d06b60338f2c5 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/benchmarks/pme_free_switch_test.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import time + +from ducktape.mark import parametrize +from ducktape.mark.resource import cluster + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.tests.utils.ignite_test import IgniteTest +from ignitetest.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion + + +class PmeFreeSwitchTest(IgniteTest): + NUM_NODES = 3 + + @staticmethod + def properties(client_mode="false"): + return """ + + + + + + + + + + + """.format(client_mode=client_mode) + + def __init__(self, test_context): + super(PmeFreeSwitchTest, self).__init__(test_context=test_context) + + def setUp(self): + pass + + def teardown(self): + pass + + @cluster(num_nodes=NUM_NODES + 2) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_7)) + def test(self, version): + data = {} + + self.stage("Starting nodes") + + ignite_version = IgniteVersion(version) + + ignites = IgniteService( + self.test_context, + num_nodes=self.NUM_NODES, + properties=self.properties(), + version=ignite_version) + + ignites.start() + + self.stage("Starting long_tx_streamer") + + long_tx_streamer = IgniteApplicationService( + self.test_context, + java_class_name="org.apache.ignite.internal.ducktest.LongTxStreamerApplication", + properties=self.properties(client_mode="true"), + params="test-cache", + version=ignite_version) + + long_tx_streamer.start() + + self.stage("Starting single_key_tx_streamer") + + single_key_tx_streamer = IgniteApplicationService( + self.test_context, + java_class_name="org.apache.ignite.internal.ducktest.SingleKeyTxStreamerApplication", + properties=self.properties(client_mode="true"), + params="test-cache,1000", + version=ignite_version) + + single_key_tx_streamer.start() + + if ignite_version >= V_2_8_0: + long_tx_streamer.execute( + "control.sh --host %s --baseline auto_adjust disable --yes" % ignites.nodes[0].account.hostname) + + self.stage("Stopping server node") + + ignites.stop_node(ignites.nodes[1]) + + long_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True) + + time.sleep(30) # keeping txs alive for 30 seconds. + + self.stage("Stopping long_tx_streamer") + + long_tx_streamer.stop() + + self.stage("Stopping single_key_tx_streamer") + + single_key_tx_streamer.stop() + + data["Worst latency (ms)"] = single_key_tx_streamer.extract_result("WORST_LATENCY") + data["Streamed txs"] = single_key_tx_streamer.extract_result("STREAMED") + data["Measure duration (ms)"] = single_key_tx_streamer.extract_result("MEASURE_DURATION") + + return data diff --git a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py index a239a4a91ac34..4fa46bb9823e3 100644 --- a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py +++ b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py @@ -13,14 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ducktape.tests.test import Test - from ignitetest.services.ignite import IgniteService -from ignitetest.services.ignite_client_app import IgniteClientApp, SparkIgniteClientApp +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.ignite_spark_app import SparkIgniteApplicationService from ignitetest.services.spark import SparkService +from ignitetest.tests.utils.ignite_test import IgniteTest -class SparkIntegrationTest(Test): +class SparkIntegrationTest(IgniteTest): """ Test performs: 1. Start of Spark cluster. @@ -28,13 +28,18 @@ class SparkIntegrationTest(Test): 3. Checks results of client application. """ + @staticmethod + def properties(client_mode="false"): + return """ + + """.format(client_mode=client_mode) + def __init__(self, test_context): super(SparkIntegrationTest, self).__init__(test_context=test_context) self.spark = SparkService(test_context, num_nodes=2) self.ignite = IgniteService(test_context, num_nodes=1) def setUp(self): - # starting all nodes except last. self.spark.start() self.ignite.start() @@ -43,9 +48,16 @@ def teardown(self): self.ignite.stop() def test_spark_client(self): - self.logger.info("Spark integration test.") + self.stage("Starting sample data generator") + + IgniteApplicationService(self.test_context, + java_class_name="org.apache.ignite.internal.ducktest.SampleDataStreamerApplication", + params="cache,1000", + properties=self.properties(client_mode="true")).run() - IgniteClientApp(self.test_context, - java_class_name="org.apache.ignite.internal.test.IgniteApplication").run() + self.stage("Starting Spark application") - SparkIgniteClientApp(self.test_context, self.spark.nodes[0]).run() + SparkIgniteApplicationService(self.test_context, + "org.apache.ignite.internal.ducktest.SparkApplication", + params="spark://" + self.spark.nodes[0].account.hostname + ":7077", + timeout_sec=120).run() diff --git a/modules/ducktests/tests/ignitetest/tests/utils/__init__.py b/modules/ducktests/tests/ignitetest/tests/utils/__init__.py new file mode 100644 index 0000000000000..ec2014340d78f --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/utils/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py new file mode 100644 index 0000000000000..1ec2132988e87 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ducktape.tests.test import Test + + +class IgniteTest(Test): + def __init__(self, test_context): + super(IgniteTest, self).__init__(test_context=test_context) + + def stage(self, msg): + self.logger.info("[TEST_STAGE] " + msg + "...") diff --git a/modules/ducktests/tests/ignitetest/version.py b/modules/ducktests/tests/ignitetest/version.py index 31ad73da2124d..9ba5c8e3c2eae 100644 --- a/modules/ducktests/tests/ignitetest/version.py +++ b/modules/ducktests/tests/ignitetest/version.py @@ -67,6 +67,11 @@ def get_version(node=None): # 2.7.x versions V_2_7_6 = IgniteVersion("2.7.6") +LATEST_2_7 = V_2_7_6 # 2.8.0 versions V_2_8_0 = IgniteVersion("2.8.0") +V_2_8_1 = IgniteVersion("2.8.1") +LATEST_2_8 = V_2_8_1 + +LATEST = LATEST_2_8 diff --git a/scripts/build-module.sh b/scripts/build-module.sh new file mode 100755 index 0000000000000..ca788fef155ca --- /dev/null +++ b/scripts/build-module.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Builds project. +# Run in Ignite sources root directory. +# Usage: ./scripts/build-module.sh ducktests +# + +mvn clean package -pl :ignite-$1 -Pall-java,all-scala -DskipTests -Dmaven.javadoc.skip=true -am \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh index 333ba9e046373..d6d83050c901c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -19,7 +19,7 @@ # # Builds project. # Run in Ignite sources root directory. -# Usage: ./build.sh +# Usage: ./scripts/build.sh # mvn clean package -Pall-java,all-scala -DskipTests -Dmaven.javadoc.skip=true \ No newline at end of file From 7bf4d5b12d10717e35623e38bcdb0d42db6e1c7f Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Thu, 16 Jul 2020 16:02:00 +0300 Subject: [PATCH 21/78] Update of the Dockerfile to conform security policies. --- modules/ducktests/tests/docker/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index d083ff3e8c553..b199c3737116b 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -32,6 +32,7 @@ ARG ducker_creator=default LABEL ducker.creator=$ducker_creator # Update Linux and install necessary utilities. +RUN cat /etc/apt/sources.list | sed 's/http:\/\/deb.debian.org/https:\/\/deb.debian.org/g' > /etc/apt/sources.list.2 && mv /etc/apt/sources.list.2 /etc/apt/sources.list RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute mc && apt-get -y clean RUN python -m pip install -U pip==9.0.3; RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.7 @@ -57,7 +58,7 @@ ARG SPARK_VERSION="2.3.4" ARG SPARK_NAME="spark-$SPARK_VERSION" ARG SPARK_RELEASE_NAME="spark-$SPARK_VERSION-bin-hadoop2.7" -RUN cd /opt && curl -O $APACHE_MIRROR/spark/$SPARK_NAME/$SPARK_RELEASE_NAME.tgz && tar xvf $SPARK_RELEASE_NAME.tgz && rm $SPARK_RELEASE_NAME.tgz +RUN cd /opt && curl -O $APACHE_ARCHIVE/spark/$SPARK_NAME/$SPARK_RELEASE_NAME.tgz && tar xvf $SPARK_RELEASE_NAME.tgz && rm $SPARK_RELEASE_NAME.tgz RUN mv /opt/$SPARK_RELEASE_NAME /opt/$SPARK_NAME RUN chmod a+wr /opt/$SPARK_NAME -R @@ -76,4 +77,4 @@ USER ducker CMD sudo service ssh start && tail -f /dev/null # Container port exposure -EXPOSE 11211 47100 47500 49112 10800 8080 \ No newline at end of file +EXPOSE 11211 47100 47500 49112 10800 8080 From 328ef28dc1737d7e0e81b10bcd799716b6d8d563 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Thu, 16 Jul 2020 21:22:56 +0300 Subject: [PATCH 22/78] bump version --- modules/ducktests/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml index d0aea24c5c7f3..81c73cafedc28 100644 --- a/modules/ducktests/pom.xml +++ b/modules/ducktests/pom.xml @@ -31,7 +31,7 @@ ignite-ducktests - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT http://ignite.apache.org From 772bd39a1f2c01c2b9ceed7a7b824a49019b5236 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Thu, 16 Jul 2020 21:24:44 +0300 Subject: [PATCH 23/78] fixing checkstyle --- .../ignite/internal/ducktest/LongTxStreamerApplication.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java index 4bbc078abb0fd..de3dbc7b78a28 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java @@ -23,8 +23,8 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInterruptedCheckedException; -import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; +import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.transactions.Transaction; import org.apache.ignite.transactions.TransactionState; From 73b897b902669fba2b5855c5137a1d9f236f70d4 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 20 Jul 2020 17:35:47 +0300 Subject: [PATCH 24/78] compilation & runtime fix --- modules/ducktests/pom.xml | 2 +- .../org/apache/ignite/internal/ducktest/SparkApplication.java | 2 +- scripts/build-module.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml index d0aea24c5c7f3..81c73cafedc28 100644 --- a/modules/ducktests/pom.xml +++ b/modules/ducktests/pom.xml @@ -31,7 +31,7 @@ ignite-ducktests - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT http://ignite.apache.org diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SparkApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SparkApplication.java index 0823a823ed046..57d1754329a53 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SparkApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SparkApplication.java @@ -32,7 +32,7 @@ public class SparkApplication extends IgniteAwareApplication { public static final String HOME = "/opt/ignite-dev"; /** Version. */ - public static final String VER = "2.9.0-SNAPSHOT"; + public static final String VER = "2.10.0-SNAPSHOT"; /** Spring version. */ public static final String SPRING_VER = "4.3.26.RELEASE"; diff --git a/scripts/build-module.sh b/scripts/build-module.sh index ca788fef155ca..541ac8b762410 100755 --- a/scripts/build-module.sh +++ b/scripts/build-module.sh @@ -22,4 +22,4 @@ # Usage: ./scripts/build-module.sh ducktests # -mvn clean package -pl :ignite-$1 -Pall-java,all-scala -DskipTests -Dmaven.javadoc.skip=true -am \ No newline at end of file +mvn package -pl :ignite-$1 -Pall-java,all-scala -DskipTests -Dmaven.javadoc.skip=true -am \ No newline at end of file From ea7fcd548743171e11d74ae27b9154793b8f871f Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Tue, 21 Jul 2020 12:40:33 +0300 Subject: [PATCH 25/78] Add zookeeper service and basic topology test to ignite-ducktests (#8059) --- modules/ducktests/pom.xml | 36 +++++ modules/ducktests/tests/docker/Dockerfile | 23 +++- modules/ducktests/tests/docker/run_tests.sh | 2 +- .../tests/ignitetest/services/zk/__init__.py | 14 ++ .../services/zk/templates/log4j.properties.j2 | 33 +++++ .../zk/templates/zookeeper.properties.j2 | 25 ++++ .../tests/ignitetest/services/zk/zookeeper.py | 127 ++++++++++++++++++ .../benchmarks/add_node_rebalance_test.py | 5 +- .../tests/ignitetest/tests/discovery_test.py | 120 +++++++++++++++++ .../ignitetest/tests/utils/ignite_test.py | 13 ++ modules/ducktests/tests/setup.py | 2 +- 11 files changed, 390 insertions(+), 10 deletions(-) create mode 100644 modules/ducktests/tests/ignitetest/services/zk/__init__.py create mode 100644 modules/ducktests/tests/ignitetest/services/zk/templates/log4j.properties.j2 create mode 100644 modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2 create mode 100644 modules/ducktests/tests/ignitetest/services/zk/zookeeper.py create mode 100644 modules/ducktests/tests/ignitetest/tests/discovery_test.py diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml index 81c73cafedc28..7bfdc2c99c7e0 100644 --- a/modules/ducktests/pom.xml +++ b/modules/ducktests/pom.xml @@ -51,12 +51,48 @@ org.apache.ignite ignite-spark ${project.version} + + + org.apache.curator + curator-recipes + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-client + + + org.apache.zookeeper + zookeeper + + org.apache.spark spark-core_2.11 ${spark.version} + + + org.apache.curator + curator-recipes + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-client + + + org.apache.zookeeper + zookeeper + + diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index b199c3737116b..0baefdae3be2c 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -35,7 +35,7 @@ LABEL ducker.creator=$ducker_creator RUN cat /etc/apt/sources.list | sed 's/http:\/\/deb.debian.org/https:\/\/deb.debian.org/g' > /etc/apt/sources.list.2 && mv /etc/apt/sources.list.2 /etc/apt/sources.list RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute mc && apt-get -y clean RUN python -m pip install -U pip==9.0.3; -RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 && pip install --upgrade ducktape==0.7.7 +RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 monotonic && pip install --upgrade ducktape==0.7.7 # Set up ssh COPY ./ssh-config /root/.ssh/config @@ -47,12 +47,25 @@ ARG APACHE_MIRROR="https://apache-mirror.rbc.ru/pub/apache/" ARG APACHE_ARCHIVE="https://archive.apache.org/dist/" # Install binary test dependencies. -RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.7.6/apache-ignite-2.7.6-bin.zip && unzip apache-ignite-2.7.6-bin.zip && mv /opt/apache-ignite-2.7.6-bin /opt/ignite-2.7.6 -RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.8.0/apache-ignite-2.8.0-bin.zip && unzip apache-ignite-2.8.0-bin.zip && mv /opt/apache-ignite-2.8.0-bin /opt/ignite-2.8.0 -RUN cd /opt && curl -O $APACHE_ARCHIVE/ignite/2.8.1/apache-ignite-2.8.1-bin.zip && unzip apache-ignite-2.8.1-bin.zip && mv /opt/apache-ignite-2.8.1-bin /opt/ignite-2.8.1 +RUN for v in "2.7.6" "2.8.0" "2.8.1"; \ + do cd /opt; \ + curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\ + unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v;\ + cp -r ./ignite-$v/libs/optional/ignite-zookeeper ./ignite-$v/libs; \ + done RUN rm /opt/apache-ignite-*-bin.zip +#Install zookeeper. +ARG ZOOKEEPER_VERSION="3.5.8" +ARG ZOOKEEPER_NAME="zookeeper-$ZOOKEEPER_VERSION" +ARG ZOOKEEPER_RELEASE_NAME="apache-$ZOOKEEPER_NAME-bin" +ARG ZOOKEEPER_RELEASE_ARTIFACT="$ZOOKEEPER_RELEASE_NAME.tar.gz" +RUN echo $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT +RUN cd /opt && curl -O $APACHE_ARCHIVE/zookeeper/$ZOOKEEPER_NAME/$ZOOKEEPER_RELEASE_ARTIFACT \ + && tar xvf $ZOOKEEPER_RELEASE_ARTIFACT && rm $ZOOKEEPER_RELEASE_ARTIFACT +RUN mv /opt/$ZOOKEEPER_RELEASE_NAME /opt/$ZOOKEEPER_NAME + # Install spark ARG SPARK_VERSION="2.3.4" ARG SPARK_NAME="spark-$SPARK_VERSION" @@ -77,4 +90,4 @@ USER ducker CMD sudo service ssh start && tail -f /dev/null # Container port exposure -EXPOSE 11211 47100 47500 49112 10800 8080 +EXPOSE 11211 47100 47500 49112 10800 8080 2888 3888 2181 diff --git a/modules/ducktests/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh index 477ec8224703a..e392e3d8d6992 100755 --- a/modules/ducktests/tests/docker/run_tests.sh +++ b/modules/ducktests/tests/docker/run_tests.sh @@ -16,7 +16,7 @@ # limitations under the License. SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-6} +IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-11} TC_PATHS=${TC_PATHS:-./ignitetest/} die() { diff --git a/modules/ducktests/tests/ignitetest/services/zk/__init__.py b/modules/ducktests/tests/ignitetest/services/zk/__init__.py new file mode 100644 index 0000000000000..ec2014340d78f --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/zk/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/modules/ducktests/tests/ignitetest/services/zk/templates/log4j.properties.j2 b/modules/ducktests/tests/ignitetest/services/zk/templates/log4j.properties.j2 new file mode 100644 index 0000000000000..507ec9852ef14 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/zk/templates/log4j.properties.j2 @@ -0,0 +1,33 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} + +zookeeper.root.logger=INFO, FILE +zookeeper.console.threshold=INFO + +zookeeper.log.dir={{ PERSISTENT_ROOT }} +zookeeper.log.file=zookeeper.log +zookeeper.log.threshold=INFO +zookeeper.log.maxfilesize=256MB +zookeeper.log.maxbackupindex=20 + +log4j.rootLogger=${zookeeper.root.logger} + +log4j.appender.FILE=org.apache.log4j.FileAppender +log4j.appender.FILE.Threshold=${zookeeper.log.threshold} +log4j.appender.FILE.File=${zookeeper.log.dir}/${zookeeper.log.file} +log4j.appender.FILE.layout=org.apache.log4j.PatternLayout +log4j.appender.FILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n diff --git a/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2 b/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2 new file mode 100644 index 0000000000000..2c80e843e41b5 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2 @@ -0,0 +1,25 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} + +tickTime={{ settings.tick_time }} +initLimit={{ settings.init_limit }} +syncLimit={{ settings.sync_limit }} +dataDir={{ DATA_DIR }} +clientPort={{ settings.client_port }} +{% for node in nodes %} +server.{{ loop.index }}={{ node.account.hostname }}:2888:3888 +{% endfor %} diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py new file mode 100644 index 0000000000000..7767ce9ed6989 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path + +from ducktape.cluster.remoteaccount import RemoteCommandError +from ducktape.services.service import Service + + +class ZookeeperSettings: + def __init__(self, tick_time=1000, init_limit=10, sync_limit=5, client_port=2181): + self.tick_time = tick_time + self.init_limit = init_limit + self.sync_limit = sync_limit + self.client_port = client_port + + +class ZookeeperService(Service): + PERSISTENT_ROOT = "/mnt/zookeeper" + CONFIG_ROOT = os.path.join(PERSISTENT_ROOT, "conf") + LOG_FILE = os.path.join(PERSISTENT_ROOT, "zookeeper.log") + DATA_DIR = os.path.join(PERSISTENT_ROOT, "data") + CONFIG_FILE = os.path.join(CONFIG_ROOT, "zookeeper.properties") + LOG_CONFIG_FILE = os.path.join(CONFIG_ROOT, "log4j.properties") + ZK_LIB_DIR = "/opt/zookeeper-3.5.8/lib" + + logs = { + "zk_log": { + "path": LOG_FILE, + "collect_default": True + } + } + + def __init__(self, context, num_nodes, settings=ZookeeperSettings()): + super(ZookeeperService, self).__init__(context, num_nodes) + self.settings = settings + + def start(self, timeout_sec=60): + Service.start(self) + self.logger.info("Waiting for Zookeeper quorum...") + + for node in self.nodes: + self.await_quorum(node, timeout_sec) + + self.logger.info("Zookeeper quorum is formed.") + + def start_node(self, node): + idx = self.idx(node) + + self.logger.info("Starting Zookeeper node %d on %s", idx, node.account.hostname) + + node.account.ssh("mkdir -p %s" % self.DATA_DIR) + node.account.ssh("mkdir -p %s" % self.CONFIG_ROOT) + node.account.ssh("echo %d > %s/myid" % (idx, self.DATA_DIR)) + + config_file = self.render('zookeeper.properties.j2', settings=self.settings) + node.account.create_file(self.CONFIG_FILE, config_file) + self.logger.info("ZK config %s", config_file) + + log_config_file = self.render('log4j.properties.j2') + node.account.create_file(self.LOG_CONFIG_FILE, log_config_file) + + start_cmd = "nohup java -cp %s/*:%s org.apache.zookeeper.server.quorum.QuorumPeerMain %s >/dev/null 2>&1 &" % \ + (self.ZK_LIB_DIR, self.CONFIG_ROOT, self.CONFIG_FILE) + + node.account.ssh(start_cmd) + + def wait_node(self, node, timeout_sec=20): + idx = self.idx(node) + + with node.account.monitor_log(self.LOG_FILE) as monitor: + monitor.offset = 0 + monitor.wait_until( + "binding to port", + timeout_sec=timeout_sec, + err_msg="Zookeeper service didn't finish startup on %s" % node.account.hostname + ) + + self.logger.info("Zookeeper node %d started on %s", idx, node.account.hostname) + + def await_quorum(self, node, timeout): + with node.account.monitor_log(self.LOG_FILE) as monitor: + monitor.offset = 0 + monitor.wait_until( + "LEADER ELECTION TOOK", + timeout_sec=timeout, + err_msg="Zookeeper quorum was not formed on %s" % node.account.hostname + ) + + def pids(self, node): + try: + cmd = "ps ax | grep -i zookeeper | grep java | grep -v grep | awk '{print $1}'" + pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] + return pid_arr + except (RemoteCommandError, ValueError) as e: + return [] + + def alive(self, node): + return len(self.pids(node)) > 0 + + def connection_string(self): + return ','.join([node.account.hostname + ":" + str(2181) for node in self.nodes]) + + def stop_node(self, node): + idx = self.idx(node) + self.logger.info("Stopping %s node %d on %s" % (type(self).__name__, idx, node.account.hostname)) + node.account.kill_process("zookeeper", allow_fail=False) + + def clean_node(self, node): + self.logger.info("Cleaning Zookeeper node %d on %s", self.idx(node), node.account.hostname) + if self.alive(node): + self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." % + (self.__class__.__name__, node.account)) + node.account.kill_process("zookeeper", clean_shutdown=False, allow_fail=True) + node.account.ssh("rm -rf %s %s %s" % (self.CONFIG_ROOT, self.DATA_DIR, self.LOG_FILE), allow_fail=False) diff --git a/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py index 2c725d728c833..ba5b324a1118b 100644 --- a/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import time from ducktape.mark import parametrize from ducktape.mark.resource import cluster @@ -82,13 +81,13 @@ def test_add_node(self, version): ignite.start() - start = time.time() + start = self.monotonic() ignite.await_event("rebalanced=true, wasRebalanced=false", timeout_sec=AddNodeRebalanceTest.REBALANCE_TIMEOUT, from_the_beginning=True, backoff_sec=1) - data = {"Rebalanced in (sec)": time.time() - start} + data = {"Rebalanced in (sec)": self.monotonic() - start} return data diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py new file mode 100644 index 0000000000000..42228e36e6ee2 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random + +from ducktape.mark import parametrize +from ducktape.mark.resource import cluster + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.zk.zookeeper import ZookeeperService +from ignitetest.version import DEV_BRANCH, LATEST_2_7 +from ignitetest.tests.utils.ignite_test import IgniteTest + +from jinja2 import Template + +class DiscoveryTest(IgniteTest): + NUM_NODES = 7 + + CONFIG_TEMPLATE = """ + + {% if zookeeper_settings %} + {% with zk = zookeeper_settings %} + + + + + + + + + {% endwith %} + {% endif %} + """ + + def __init__(self, test_context): + super(DiscoveryTest, self).__init__(test_context=test_context) + self.zk = None + self.servers = None + + @staticmethod + def properties(client_mode="false", zookeeper_settings=None): + return Template(DiscoveryTest.CONFIG_TEMPLATE) \ + .render(client_mode=client_mode, zookeeper_settings=zookeeper_settings) + + def setUp(self): + pass + + def teardown(self): + if self.zk: + self.zk.stop() + + if self.servers: + self.servers.stop() + + @cluster(num_nodes=NUM_NODES) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_7)) + def test_tcp(self, version): + return self.__basic_test__(version, False) + + @cluster(num_nodes=NUM_NODES + 3) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_7)) + def test_zk(self, version): + return self.__basic_test__(version, True) + + def __basic_test__(self, version, with_zk=False): + if with_zk: + self.zk = ZookeeperService(self.test_context, 3) + self.stage("Starting Zookeper quorum") + self.zk.start() + properties = self.properties(zookeeper_settings={'connection_string': self.zk.connection_string()}) + self.stage("Zookeper quorum started") + else: + properties = self.properties() + + self.servers = IgniteService( + self.test_context, + num_nodes=self.NUM_NODES, + properties=properties, + version=version) + + self.stage("Starting ignite cluster") + + start = self.monotonic() + self.servers.start() + data = {'Ignite cluster start time (s)': self.monotonic() - start } + self.stage("Topology is ready") + + # Node failure detection + fail_node, survived_node = self.choose_random_node_to_kill(self.servers) + self.servers.stop_node(fail_node, clean_shutdown=False) + + start = self.monotonic() + self.servers.await_event_on_node("Node FAILED", random.choice(survived_node), 60, True) + + data['Failure of node detected in time (s)'] = self.monotonic() - start + + return data + + @staticmethod + def choose_random_node_to_kill(service): + idx = random.randint(0, len(service.nodes) - 1) + + survive = [node for i, node in enumerate(service.nodes) if i != idx] + kill = service.nodes[idx] + + return kill, survive diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py index 1ec2132988e87..a9c13d941d20d 100644 --- a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py +++ b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py @@ -14,6 +14,7 @@ # limitations under the License. from ducktape.tests.test import Test +from monotonic import monotonic class IgniteTest(Test): @@ -22,3 +23,15 @@ def __init__(self, test_context): def stage(self, msg): self.logger.info("[TEST_STAGE] " + msg + "...") + + @staticmethod + def monotonic(): + """ + monotonic() -> float + + :return: + The value (in fractional seconds) of a monotonic clock, i.e. a clock that cannot go backwards. + The clock is not affected by system clock updates. The reference point of the returned value is undefined, + so that only the difference between the results of consecutive calls is valid. + """ + return monotonic() diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index e82ed2bae75fd..2eef5141ef4ee 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -52,6 +52,6 @@ def run_tests(self): packages=find_packages(), include_package_data=True, install_requires=["ducktape==0.7.7", "requests==2.20.0"], - tests_require=["pytest", "mock"], + tests_require=["pytest", "mock", "monotonic"], cmdclass={'test': PyTest} ) From df2428490f83a14eb69480dbc70f23717ee3c30e Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Tue, 21 Jul 2020 13:05:02 +0300 Subject: [PATCH 26/78] mem limit fix redundant jvm fix removal --- modules/ducktests/tests/docker/ducker-ignite | 4 ++-- modules/ducktests/tests/ignitetest/services/ignite.py | 1 - .../tests/ignitetest/services/utils/ignite_aware_app.py | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/modules/ducktests/tests/docker/ducker-ignite b/modules/ducktests/tests/docker/ducker-ignite index e41212c7db1b7..e7287cad5fd89 100755 --- a/modules/ducktests/tests/docker/ducker-ignite +++ b/modules/ducktests/tests/docker/ducker-ignite @@ -33,10 +33,10 @@ ignite_dir="$( cd "${ducker_dir}/../../../.." && pwd )" # The memory consumption to allow during the docker build. # This does not include swap. -docker_build_memory_limit="3200m" +docker_build_memory_limit="8000m" # The maximum mmemory consumption to allow in containers. -docker_run_memory_limit="2000m" +docker_run_memory_limit="8000m" # The default number of cluster nodes to bring up if a number is not specified. default_num_nodes=4 diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index 07b562fe8450b..a2855e3a661a6 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -52,7 +52,6 @@ def start(self, timeout_sec=180): def start_cmd(self, node): jvm_opts = "-J-DIGNITE_SUCCESS_FILE=" + IgniteService.PERSISTENT_ROOT + "/success_file " jvm_opts += "-J-Dlog4j.configDebug=true " - jvm_opts += "-J-XX:+UnlockExperimentalVMOptions -J-XX:+UseCGroupMemoryLimitForHeap" # java8 docker fix cmd = "export EXCLUDE_TEST_CLASSES=true; " cmd += "export IGNITE_LOG_DIR=" + IgniteService.PERSISTENT_ROOT + "; " diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index f569c120ecece..916d1550b5acd 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -86,8 +86,7 @@ def jvm_opts(self): "-J-Dlog4j.configDebug=true " \ "-J-Xmx1G " \ "-J-ea " \ - "-J-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false " \ - "-J-XX:+UnlockExperimentalVMOptions -J-XX:+UseCGroupMemoryLimitForHeap" # java8 docker fix + "-J-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false " def env(self): return "export MAIN_CLASS={main_class}; ".format(main_class=self.servicejava_class_name) + \ From 57aa7780f34167f17b5365b5a77d5478d0d5daef Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Thu, 23 Jul 2020 15:02:00 +0300 Subject: [PATCH 27/78] Add jmx client and some basic discovery stuff fo IgniteClusterNode. (#8070) --- modules/ducktests/tests/docker/Dockerfile | 7 + .../tests/ignitetest/services/ignite.py | 9 +- .../tests/ignitetest/services/ignite_app.py | 15 +-- .../ignitetest/services/ignite_spark_app.py | 4 +- .../ignitetest/services/utils/decorators.py | 32 +++++ .../ignitetest/services/utils/ignite_aware.py | 11 +- .../services/utils/ignite_aware_app.py | 5 +- .../ignitetest/services/utils/jmx_utils.py | 124 ++++++++++++++++++ .../tests/ignitetest/services/zk/zookeeper.py | 14 +- .../tests/ignitetest/tests/discovery_test.py | 19 ++- 10 files changed, 208 insertions(+), 32 deletions(-) create mode 100644 modules/ducktests/tests/ignitetest/services/utils/decorators.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index 0baefdae3be2c..fedea542476e6 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -83,6 +83,13 @@ ARG KIBOSH_VERSION="8841dd392e6fbf02986e2fb1f1ebf04df344b65a" RUN apt-get install fuse RUN cd /opt && git clone -q https://github.com/confluentinc/kibosh.git && cd "/opt/kibosh" && git reset --hard $KIBOSH_VERSION && mkdir "/opt/kibosh/build" && cd "/opt/kibosh/build" && ../configure && make -j 2 +#Install jmxterm +ARG JMXTERM_NAME="jmxterm" +ARG JMXTERM_VERSION="1.0.1" +ARG JMXTERM_ARTIFACT="$JMXTERM_NAME-$JMXTERM_VERSION-uber.jar" +RUN cd /opt && curl -OL https://github.com/jiaqi/jmxterm/releases/download/v$JMXTERM_VERSION/$JMXTERM_ARTIFACT \ + && mv $JMXTERM_ARTIFACT $JMXTERM_NAME.jar + # Set up the ducker user. RUN useradd -ms /bin/bash ducker && mkdir -p /home/ducker/ && rsync -aiq /root/.ssh/ /home/ducker/.ssh && chown -R ducker /home/ducker/ /mnt/ /var/log/ && echo "PATH=$(runuser -l ducker -c 'echo $PATH'):$JAVA_HOME/bin" >> /home/ducker/.ssh/environment && echo 'PATH=$PATH:'"$JAVA_HOME/bin" >> /home/ducker/.profile && echo 'ducker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers USER ducker diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index a2855e3a661a6..e68d0bb27ac63 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -17,7 +17,6 @@ import signal from ducktape.cluster.remoteaccount import RemoteCommandError -from ducktape.services.service import Service from ducktape.utils.util import wait_until from ignitetest.services.utils.ignite_aware import IgniteAwareService @@ -39,15 +38,15 @@ class IgniteService(IgniteAwareService): } def __init__(self, context, num_nodes, version=DEV_BRANCH, properties=""): - IgniteAwareService.__init__(self, context, num_nodes, version, properties) + super(IgniteService, self).__init__(context, num_nodes, version, properties) def start(self, timeout_sec=180): - Service.start(self) + super(IgniteService, self).start() self.logger.info("Waiting for Ignite(s) to start...") for node in self.nodes: - self.await_node_stated(node, timeout_sec) + self.await_node_started(node, timeout_sec) def start_cmd(self, node): jvm_opts = "-J-DIGNITE_SUCCESS_FILE=" + IgniteService.PERSISTENT_ROOT + "/success_file " @@ -64,7 +63,7 @@ def start_cmd(self, node): IgniteService.STDOUT_STDERR_CAPTURE) return cmd - def await_node_stated(self, node, timeout_sec): + def await_node_started(self, node, timeout_sec): self.await_event_on_node("Topology snapshot", node, timeout_sec, from_the_beginning=True) if len(self.pids(node)) == 0: diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 87cb8e1215975..237436d14a42a 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -24,15 +24,8 @@ class IgniteApplicationService(IgniteAwareApplicationService): - def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60): - IgniteAwareApplicationService.__init__( - self, context, java_class_name, version, properties, params, timeout_sec, - service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteApplicationService") - - def start(self): - Service.start(self) + service_java_class_name = "org.apache.ignite.internal.ducktest.utils.IgniteApplicationService" - self.logger.info("Waiting for Ignite Application (%s) to start..." % self.java_class_name) - - self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True) - self.await_event("IGNITE_APPLICATION_INITIALIZED", self.timeout_sec, from_the_beginning=True) + def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60): + super(IgniteApplicationService, self).__init__(context, java_class_name, version, properties, params, + timeout_sec, self.service_java_class_name) diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py index dffe7ace7f48a..c2f0496cc0db6 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py @@ -22,8 +22,8 @@ class SparkIgniteApplicationService(IgniteAwareApplicationService): def __init__(self, context, java_class_name, version=DEV_BRANCH, properties="", params="", timeout_sec=60): - IgniteAwareApplicationService.__init__( - self, context, java_class_name, version, properties, params, timeout_sec) + super(SparkIgniteApplicationService, self).__init__(context, java_class_name, version, properties, params, + timeout_sec) def env(self): return IgniteAwareApplicationService.env(self) + \ diff --git a/modules/ducktests/tests/ignitetest/services/utils/decorators.py b/modules/ducktests/tests/ignitetest/services/utils/decorators.py new file mode 100644 index 0000000000000..788ddba24d139 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/decorators.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import functools +from threading import RLock + + +def memoize(func): + cache = func.cache = {} + lock = RLock() + + @functools.wraps(func) + def memoized_func(*args, **kwargs): + key = str(args) + str(kwargs) + if key not in cache: + with lock: + if key not in cache: + cache[key] = func(*args, **kwargs) + return cache[key] + + return memoized_func diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index dd82febfd6583..106493c92f77a 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -12,14 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import os from abc import abstractmethod from ducktape.services.background_thread import BackgroundThreadService +from ducktape.utils.util import wait_until from ignitetest.services.utils.ignite_config import IgniteConfig from ignitetest.services.utils.ignite_path import IgnitePath +from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin """ The base class to build services aware of Ignite. @@ -41,7 +42,7 @@ class IgniteAwareService(BackgroundThreadService): } def __init__(self, context, num_nodes, version, properties): - BackgroundThreadService.__init__(self, context, num_nodes) + super(IgniteAwareService, self).__init__(context, num_nodes) self.log_level = "DEBUG" self.config = IgniteConfig() @@ -55,7 +56,11 @@ def __init__(self, context, num_nodes, version, properties): def start_node(self, node): self.init_persistent(node) - BackgroundThreadService.start_node(self, node) + super(IgniteAwareService, self).start_node(node) + + wait_until(lambda: len(self.pids(node)) > 0, timeout_sec=10) + + ignite_jmx_mixin(node, self.pids(node)) def init_persistent(self, node): node.account.mkdirs(self.PERSISTENT_ROOT) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index 916d1550b5acd..7b376333af755 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -26,7 +26,7 @@ class IgniteAwareApplicationService(IgniteAwareService): def __init__(self, context, java_class_name, version, properties, params, timeout_sec, service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): - IgniteAwareService.__init__(self, context, 1, version, properties) + super(IgniteAwareApplicationService, self).__init__(context, 1, version, properties) self.servicejava_class_name = service_java_class_name self.java_class_name = java_class_name @@ -35,10 +35,11 @@ def __init__(self, context, java_class_name, version, properties, params, timeou self.params = params def start(self): - Service.start(self) + super(IgniteAwareApplicationService, self).start() self.logger.info("Waiting for Ignite aware Application (%s) to start..." % self.java_class_name) + self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True) self.await_event("IGNITE_APPLICATION_INITIALIZED", self.timeout_sec, from_the_beginning=True) def start_cmd(self, node): diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py new file mode 100644 index 0000000000000..6843c945dfcad --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +from ignitetest.services.utils.decorators import memoize + + +def ignite_jmx_mixin(node, pids): + setattr(node, 'pids', pids) + base_cls = node.__class__ + base_cls_name = node.__class__.__name__ + node.__class__ = type(base_cls_name, (base_cls, IgniteJmxMixin), {}) + + +class JmxMBean(object): + def __init__(self, client, name): + self.client = client + self.name = name + + def __getattr__(self, attr): + return self.client.mbean_attribute(self.name, attr) + + +class JmxClient(object): + jmx_util_cmd = 'java -jar /opt/jmxterm.jar -v silent -n' + + def __init__(self, node): + self.node = node + self.pid = node.pids[0] + + @memoize + def find_mbean(self, pattern, domain='org.apache'): + cmd = "echo $'open %s\\n beans -d %s \\n close' | %s | grep -o '%s'" \ + % (self.pid, domain, self.jmx_util_cmd, pattern) + + name = next(self.run_cmd(cmd)).strip() + + return JmxMBean(self, name) + + def mbean_attribute(self, mbean, attr): + cmd = "echo $'open %s\\n get -b %s %s \\n close' | %s | sed 's/%s = \\(.*\\);/\\1/'" \ + % (self.pid, mbean, attr, self.jmx_util_cmd, attr) + + return iter(s.strip() for s in self.run_cmd(cmd)) + + def run_cmd(self, cmd): + return self.node.account.ssh_capture(cmd, allow_fail=False, callback=str) + + +class DiscoveryInfo(object): + def __init__(self, coordinator, local_raw): + self._local_raw = local_raw + self._coordinator = coordinator + + @property + def id(self): + return self.__find__("id=([^\\s]+),") + + @property + def coordinator(self): + return self._coordinator + + @property + def consistent_id(self): + return self.__find__("consistentId=([^\\s]+),") + + @property + def is_client(self): + return self.__find__("isClient=([^\\s]+),") == "true" + + @property + def order(self): + return int(self.__find__("order=(\\d+),")) + + @property + def int_order(self): + val = self.__find__("intOrder=(\\d+),") + return int(val) if val else -1 + + def __find__(self, pattern): + res = re.search(pattern, self._local_raw) + return res.group(1) if res else None + + +class IgniteJmxMixin(object): + @memoize + def jmx_client(self): + # noinspection PyTypeChecker + return JmxClient(self) + + @memoize + def id(self): + return next(self.kernal_mbean().LocalNodeId).strip() + + def discovery_info(self): + disco_mbean = self.disco_mbean() + crd = next(disco_mbean.Coordinator).strip() + local = next(disco_mbean.LocalNodeFormatted).strip() + + return DiscoveryInfo(crd, local) + + def kernal_mbean(self): + return self.jmx_client().find_mbean('.*group=Kernal,name=IgniteKernal') + + @memoize + def disco_mbean(self): + disco_spi = next(self.kernal_mbean().DiscoverySpiFormatted).strip() + + if 'ZookeeperDiscoverySpi' in disco_spi: + return self.jmx_client().find_mbean('.*group=SPIs,name=ZookeeperDiscoverySpi') + else: + return self.jmx_client().find_mbean('.*group=SPIs,name=TcpDiscoverySpi') diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py index 7767ce9ed6989..d32cbbc18e486 100644 --- a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py +++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py @@ -12,10 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import os.path -from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.services.service import Service @@ -99,13 +97,13 @@ def await_quorum(self, node, timeout): err_msg="Zookeeper quorum was not formed on %s" % node.account.hostname ) + @staticmethod + def java_class_name(): + """ The class name of the Zookeeper quorum peers. """ + return "org.apache.zookeeper.server.quorum.QuorumPeerMain" + def pids(self, node): - try: - cmd = "ps ax | grep -i zookeeper | grep java | grep -v grep | awk '{print $1}'" - pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] - return pid_arr - except (RemoteCommandError, ValueError) as e: - return [] + return node.account.java_pids(self.java_class_name()) def alive(self, node): return len(self.pids(node)) > 0 diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 42228e36e6ee2..dd252053e6ec1 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import random from ducktape.mark import parametrize @@ -25,6 +24,7 @@ from jinja2 import Template + class DiscoveryTest(IgniteTest): NUM_NODES = 7 @@ -101,6 +101,23 @@ def __basic_test__(self, version, with_zk=False): # Node failure detection fail_node, survived_node = self.choose_random_node_to_kill(self.servers) + + data["nodes"] = [node.id() for node in self.servers.nodes] + + disco_infos = [] + for node in self.servers.nodes: + disco_info = node.discovery_info() + disco_infos.append({ + "id": disco_info.id, + "consistent_id": disco_info.consistent_id, + "coordinator": disco_info.coordinator, + "order": disco_info.order, + "int_order": disco_info.int_order, + "is_client": disco_info.is_client + }) + + data["node_disco_info"] = disco_infos + self.servers.stop_node(fail_node, clean_shutdown=False) start = self.monotonic() From c84478fa5787628d907a048888f485cd95d1fdd4 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Thu, 23 Jul 2020 16:45:17 +0300 Subject: [PATCH 28/78] cleanup --- .../{ => tests}/DataGenerationApplication.java | 2 +- .../LongTxStreamerApplication.java | 2 +- .../SingleKeyTxStreamerApplication.java | 2 +- .../SampleDataStreamerApplication.java | 2 +- .../SparkApplication.java | 2 +- .../tests/ignitetest/services/ignite.py | 2 +- .../tests/ignitetest/services/ignite_app.py | 4 +--- .../ignitetest/services/ignite_spark_app.py | 2 +- .../tests/ignitetest/services/spark.py | 2 +- .../ignitetest/services/utils/ignite_path.py | 2 +- .../add_node_rebalance_test.py | 4 ++-- .../ignitetest/tests/benchmarks/__init__.py | 14 -------------- .../tests/ignitetest/tests/discovery_test.py | 5 ++--- .../{benchmarks => }/pme_free_switch_test.py | 6 +++--- .../ignitetest/tests/spark_integration_test.py | 18 ++++++++++-------- .../ignitetest/{ => tests/utils}/version.py | 0 16 files changed, 27 insertions(+), 42 deletions(-) rename modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/{ => tests}/DataGenerationApplication.java (97%) rename modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/{ => tests/pme_free_switch_test}/LongTxStreamerApplication.java (97%) rename modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/{ => tests/pme_free_switch_test}/SingleKeyTxStreamerApplication.java (97%) rename modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/{ => tests/spark_integration_test}/SampleDataStreamerApplication.java (96%) rename modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/{ => tests/spark_integration_test}/SparkApplication.java (98%) rename modules/ducktests/tests/ignitetest/tests/{benchmarks => }/add_node_rebalance_test.py (96%) delete mode 100644 modules/ducktests/tests/ignitetest/tests/benchmarks/__init__.py rename modules/ducktests/tests/ignitetest/tests/{benchmarks => }/pme_free_switch_test.py (95%) rename modules/ducktests/tests/ignitetest/{ => tests/utils}/version.py (100%) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/DataGenerationApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java similarity index 97% rename from modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/DataGenerationApplication.java rename to modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java index 4c5ffcfd9dfdf..c1661d2f4b0b2 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/DataGenerationApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.ducktest; +package org.apache.ignite.internal.ducktest.tests; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java similarity index 97% rename from modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java rename to modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java index de3dbc7b78a28..14b96a8dad669 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/LongTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.ducktest; +package org.apache.ignite.internal.ducktest.tests.pme_free_switch_test; import java.util.Collection; import java.util.concurrent.CountDownLatch; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SingleKeyTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java similarity index 97% rename from modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SingleKeyTxStreamerApplication.java rename to modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java index bff4e7e920427..42e4d24a4afd2 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SingleKeyTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.ducktest; +package org.apache.ignite.internal.ducktest.tests.pme_free_switch_test; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SampleDataStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SampleDataStreamerApplication.java similarity index 96% rename from modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SampleDataStreamerApplication.java rename to modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SampleDataStreamerApplication.java index 281d202fccd70..691953c21c762 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SampleDataStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SampleDataStreamerApplication.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.ducktest; +package org.apache.ignite.internal.ducktest.tests.spark_integration_test; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SparkApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SparkApplication.java similarity index 98% rename from modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SparkApplication.java rename to modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SparkApplication.java index 57d1754329a53..565b85e8be773 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/SparkApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SparkApplication.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal.ducktest; +package org.apache.ignite.internal.ducktest.tests.spark_integration_test; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; import org.apache.ignite.spark.IgniteDataFrameSettings; diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index e68d0bb27ac63..375c2ade559eb 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -20,7 +20,7 @@ from ducktape.utils.util import wait_until from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.version import DEV_BRANCH +from ignitetest.tests.utils.version import DEV_BRANCH class IgniteService(IgniteAwareService): diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 237436d14a42a..a073fb36e3cb4 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -13,10 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ducktape.services.service import Service - from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService -from ignitetest.version import DEV_BRANCH +from ignitetest.tests.utils.version import DEV_BRANCH """ The Ignite application service allows to perform custom logic writen on java. diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py index c2f0496cc0db6..5cc66c348e58d 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py @@ -17,7 +17,7 @@ The Ignite-Spark application service. """ from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService -from ignitetest.version import DEV_BRANCH +from ignitetest.tests.utils.version import DEV_BRANCH class SparkIgniteApplicationService(IgniteAwareApplicationService): diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py index 731bcd5cd24a7..1ac937dc92046 100644 --- a/modules/ducktests/tests/ignitetest/services/spark.py +++ b/modules/ducktests/tests/ignitetest/services/spark.py @@ -20,7 +20,7 @@ from ignitetest.services.utils.ignite_aware import IgniteAwareService from ignitetest.services.utils.ignite_config import IgniteConfig -from ignitetest.version import DEV_BRANCH +from ignitetest.tests.utils.version import DEV_BRANCH class SparkService(IgniteAwareService): diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py index ed1a3fbcc1d33..8e4603a288ac2 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py @@ -15,7 +15,7 @@ import os -from ignitetest.version import get_version, IgniteVersion +from ignitetest.tests.utils.version import get_version, IgniteVersion """ This module provides Ignite path methods diff --git a/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py similarity index 96% rename from modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py rename to modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py index ba5b324a1118b..a55863803cb0f 100644 --- a/modules/ducktests/tests/ignitetest/tests/benchmarks/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py @@ -19,7 +19,7 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.tests.utils.ignite_test import IgniteTest -from ignitetest.version import DEV_BRANCH, IgniteVersion, LATEST +from ignitetest.tests.utils.version import DEV_BRANCH, IgniteVersion, LATEST class AddNodeRebalanceTest(IgniteTest): @@ -69,7 +69,7 @@ def test_add_node(self, version): # This client just put some data to the cache. IgniteApplicationService(self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.DataGenerationApplication", + java_class_name="org.apache.ignite.internal.ducktest.tests.DataGenerationApplication", properties=self.properties(client_mode="true"), version=ignite_version, params="test-cache,%d" % self.DATA_AMOUNT, diff --git a/modules/ducktests/tests/ignitetest/tests/benchmarks/__init__.py b/modules/ducktests/tests/ignitetest/tests/benchmarks/__init__.py deleted file mode 100644 index ec2014340d78f..0000000000000 --- a/modules/ducktests/tests/ignitetest/tests/benchmarks/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index dd252053e6ec1..246416a6ce26f 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -16,13 +16,12 @@ from ducktape.mark import parametrize from ducktape.mark.resource import cluster +from jinja2 import Template from ignitetest.services.ignite import IgniteService from ignitetest.services.zk.zookeeper import ZookeeperService -from ignitetest.version import DEV_BRANCH, LATEST_2_7 from ignitetest.tests.utils.ignite_test import IgniteTest - -from jinja2 import Template +from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7 class DiscoveryTest(IgniteTest): diff --git a/modules/ducktests/tests/ignitetest/tests/benchmarks/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py similarity index 95% rename from modules/ducktests/tests/ignitetest/tests/benchmarks/pme_free_switch_test.py rename to modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index d06b60338f2c5..89653e9f9b8c6 100644 --- a/modules/ducktests/tests/ignitetest/tests/benchmarks/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -20,7 +20,7 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.tests.utils.ignite_test import IgniteTest -from ignitetest.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion +from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion class PmeFreeSwitchTest(IgniteTest): @@ -72,7 +72,7 @@ def test(self, version): long_tx_streamer = IgniteApplicationService( self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.LongTxStreamerApplication", + java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.LongTxStreamerApplication", properties=self.properties(client_mode="true"), params="test-cache", version=ignite_version) @@ -83,7 +83,7 @@ def test(self, version): single_key_tx_streamer = IgniteApplicationService( self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.SingleKeyTxStreamerApplication", + java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.SingleKeyTxStreamerApplication", properties=self.properties(client_mode="true"), params="test-cache,1000", version=ignite_version) diff --git a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py index 4fa46bb9823e3..45c0a385e22ad 100644 --- a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py +++ b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py @@ -50,14 +50,16 @@ def teardown(self): def test_spark_client(self): self.stage("Starting sample data generator") - IgniteApplicationService(self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.SampleDataStreamerApplication", - params="cache,1000", - properties=self.properties(client_mode="true")).run() + IgniteApplicationService( + self.test_context, + java_class_name="org.apache.ignite.internal.ducktest.tests.spark_integration_test.SampleDataStreamerApplication", + params="cache,1000", + properties=self.properties(client_mode="true")).run() self.stage("Starting Spark application") - SparkIgniteApplicationService(self.test_context, - "org.apache.ignite.internal.ducktest.SparkApplication", - params="spark://" + self.spark.nodes[0].account.hostname + ":7077", - timeout_sec=120).run() + SparkIgniteApplicationService( + self.test_context, + "org.apache.ignite.internal.ducktest.tests.spark_integration_test.SparkApplication", + params="spark://" + self.spark.nodes[0].account.hostname + ":7077", + timeout_sec=120).run() diff --git a/modules/ducktests/tests/ignitetest/version.py b/modules/ducktests/tests/ignitetest/tests/utils/version.py similarity index 100% rename from modules/ducktests/tests/ignitetest/version.py rename to modules/ducktests/tests/ignitetest/tests/utils/version.py From 5f2ca372fa8baa07b4df44a8929cc676e6cce54e Mon Sep 17 00:00:00 2001 From: Maksim Timonin Date: Wed, 29 Jul 2020 16:06:15 +0300 Subject: [PATCH 29/78] Make ducktape work with ISE / ISP (#8071) --- modules/ducktests/tests/README.md | 13 ++ modules/ducktests/tests/docker/clean_up.sh | 4 +- modules/ducktests/tests/docker/ducker-ignite | 140 +++++++++++++----- modules/ducktests/tests/docker/run_tests.sh | 123 ++++++++++++++- .../ducktests/tests/ignitetest/__init__.py | 2 +- .../tests/ignitetest/services/ignite.py | 8 +- .../tests/ignitetest/services/ignite_app.py | 7 +- .../ignitetest/services/ignite_spark_app.py | 7 +- .../tests/ignitetest/services/spark.py | 6 +- .../services/utils/config/ignite.xml.j2 | 36 +++++ .../services/utils/config/log4j.xml.j2 | 57 +++++++ .../ignitetest/services/utils/ignite_aware.py | 23 ++- .../services/utils/ignite_aware_app.py | 7 +- .../services/utils/ignite_config.py | 112 ++++++-------- .../ignitetest/services/utils/ignite_path.py | 4 +- .../tests/add_node_rebalance_test.py | 7 - .../tests/ignitetest/tests/discovery_test.py | 5 +- .../ignitetest/tests/pme_free_switch_test.py | 9 +- .../tests/spark_integration_test.py | 28 ++-- .../tests/ignitetest/tests/utils/version.py | 14 +- 20 files changed, 444 insertions(+), 168 deletions(-) create mode 100644 modules/ducktests/tests/README.md mode change 100644 => 100755 modules/ducktests/tests/docker/clean_up.sh create mode 100644 modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 create mode 100644 modules/ducktests/tests/ignitetest/services/utils/config/log4j.xml.j2 diff --git a/modules/ducktests/tests/README.md b/modules/ducktests/tests/README.md new file mode 100644 index 0000000000000..cbf74823cec58 --- /dev/null +++ b/modules/ducktests/tests/README.md @@ -0,0 +1,13 @@ +## Overview +The `ignitetest` framework provides basic functionality and services +to write integration tests for Apache Ignite. This framework bases on +the `ducktape` test framework, for information about it check the links: +- https://github.com/confluentinc/ducktape - source code of the `ducktape`; +- http://ducktape-docs.readthedocs.io - documentation to the `ducktape`. + +Structure of the `ignitetest` directory is: +- `./ignitetest/services` contains basic services functionality; +- `./ignitetest/tests/utils` contains utils for testing. + +## Review Checklist +1. All tests must be parameterized with `version`. diff --git a/modules/ducktests/tests/docker/clean_up.sh b/modules/ducktests/tests/docker/clean_up.sh old mode 100644 new mode 100755 index 34ce05d7ae019..a5efe2d12e38b --- a/modules/ducktests/tests/docker/clean_up.sh +++ b/modules/ducktests/tests/docker/clean_up.sh @@ -15,5 +15,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -docker stop $(docker ps -a -q) -docker rm $(docker ps -a -q) \ No newline at end of file +bash ./ducker-ignite down + diff --git a/modules/ducktests/tests/docker/ducker-ignite b/modules/ducktests/tests/docker/ducker-ignite index e7287cad5fd89..9a92247a58e13 100755 --- a/modules/ducktests/tests/docker/ducker-ignite +++ b/modules/ducktests/tests/docker/ducker-ignite @@ -60,10 +60,18 @@ Usage: ${script_path} [command] [options] help|-h|--help Display this help message +build [-j|--jdk JDK] [-c|--context] [image-name] + Build a docker image that represents ducker node. Image is tagged with specified ${image_name}. + + If --jdk is specified then we will use this argument as base image for ducker docker images. + Otherwise ${default_jdk} is used. + + If --context is specified then build docker image from this path. Context directory must contain Dockerfile. + up [-n|--num-nodes NUM_NODES] [-f|--force] [docker-image] - [-C|--custom-ducktape DIR] [-e|--expose-ports ports] + [-C|--custom-ducktape DIR] [-e|--expose-ports ports] [-j|--jdk JDK_VERSION] Bring up a cluster with the specified amount of nodes (defaults to ${default_num_nodes}). - The docker image name defaults to ${default_image_name}. If --force is specified, we will + The docker image name defaults to ${default_image_name}. If --force is specified, we will attempt to bring up an image even some parameters are not valid. If --custom-ducktape is specified, we will install the provided custom @@ -75,6 +83,9 @@ up [-n|--num-nodes NUM_NODES] [-f|--force] [docker-image] or a combination of port/port-range separated by comma (like 2181,9092 or 2181,5005-5008). By default no port is exposed. See README.md for more detail on this option. + If --jdk is specified then we will use this argument as base image for ducktest's docker images. + Otherwise ${default_jdk} is used. + test [test-name(s)] Run a test or set of tests inside the currently active Ducker nodes. For example, to run the system test produce_bench_test, you would run: @@ -96,6 +107,11 @@ down [-q|--quiet] [-f|--force] purge [--f|--force] Purge Docker images created by ducker-ignite. This will free disk space. If --force is set, we run 'docker rmi -f'. + +compare [docker-image] + Compare image id of last run and last build for specified docker-image. If they are different then cluster runs + non-actual version of image. Rerun is required. + EOF exit "${exit_status}" } @@ -189,9 +205,9 @@ must_do() { *) break;; esac done - local cmd="${@}" - [[ "${verbose}" -eq 1 ]] && echo "${cmd}" - ${cmd} >${output} || die "${1} failed" + + [[ "${verbose}" -eq 1 ]] && echo "${@}" + eval "${@}" >${output} || die "${1} failed" } # Ask the user a yes/no question. @@ -214,9 +230,11 @@ ask_yes_no() { # Build a docker image. # -# $1: The name of the image to build. -ducker_build() { - local image_name="${1}" +# $1: The docker build context +# $2: The name of the image to build. +ducker_build_image() { + local docker_context="${1}" + local image_name="${2}" # Use SECONDS, a builtin bash variable that gets incremented each second, to measure the docker # build duration. @@ -227,16 +245,38 @@ ducker_build() { # (for example java.lang.NoClassDefFoundError), add --no-cache flag to the build shall give you a clean start. must_do -v -o docker build --memory="${docker_build_memory_limit}" \ --build-arg "ducker_creator=${user_name}" --build-arg "jdk_version=${jdk_version}" -t "${image_name}" \ - -f "${ducker_dir}/Dockerfile" ${docker_args} -- . + -f "${docker_context}/Dockerfile" -- "${docker_context}" docker_status=$? must_popd duration="${SECONDS}" if [[ ${docker_status} -ne 0 ]]; then - die "** ERROR: Failed to build ${what} image after $((${duration} / 60))m \ -$((${duration} % 60))s. See ${build_log} for details." + die "** ERROR: Failed to build ${what} image after $((duration / 60))m $((duration % 60))s." fi - echo "** Successfully built ${what} image in $((${duration} / 60))m \ -$((${duration} % 60))s. See ${build_log} for details." + + # Save docker image id to the file. Then could use this file to find version of docker image built last time. + # It could be useful if we don't confident about necessity of stoping the cluster. + get_image_id "${image_name}" > "${ducker_dir}/build/image_${image_name}.build" + + echo "** Successfully built ${what} image in $((duration / 60))m $((duration % 60))s." +} + +ducker_build() { + require_commands docker + + local docker_context= + while [[ $# -ge 1 ]]; do + case "${1}" in + -j|--jdk) set_once jdk_version "${2}" "the OpenJDK base image"; shift 2;; + -c|--context) docker_context="${2}"; shift 2;; + *) set_once image_name "${1}" "docker image name"; shift;; + esac + done + + [[ -n "${jdk_version}" ]] || jdk_version="${default_jdk}" + [[ -n "${image_name}" ]] || image_name="${default_image_name}-${jdk_version/:/-}" + [[ -n "${docker_context}" ]] || docker_context="${ducker_dir}" + + ducker_build_image "${docker_context}" "${image_name}" } docker_run() { @@ -256,9 +296,9 @@ docker_run() { # and mount FUSE filesystems inside the container. We also need it to # run iptables inside the container. must_do -v docker run --privileged \ - -d -t -h "${node}" --network ducknet "${expose_ports}" \ + -d -t -h "${node}" --network ducknet ${expose_ports} \ --memory=${docker_run_memory_limit} --memory-swappiness=1 \ - -v "${ignite_dir}:/opt/ignite-dev:delegated" --name "${node}" -- "${image_name}" + --mount type=bind,source="${ignite_dir}",target=/opt/ignite-dev,consistency=delegated --name "${node}" -- "${image_name}" } setup_custom_ducktape() { @@ -268,7 +308,8 @@ setup_custom_ducktape() { [[ -f "${custom_ducktape}/ducktape/__init__.py" ]] || \ die "You must supply a valid ducktape directory to --custom-ducktape" docker_run ducker01 "${image_name}" - local running_container="$(docker ps -f=network=ducknet -q)" + local running_container + running_container=$(docker ps -f=network=ducknet -q) must_do -v -o docker cp "${custom_ducktape}" "${running_container}:/opt/ducktape" docker exec --user=root ducker01 bash -c 'set -x && cd /opt/ignite-dev/modules/ducktests/tests && sudo python ./setup.py develop install && cd /opt/ducktape && sudo python ./setup.py develop install' [[ $? -ne 0 ]] && die "failed to install the new ducktape." @@ -284,14 +325,12 @@ ducker_up() { -C|--custom-ducktape) set_once custom_ducktape "${2}" "the custom ducktape directory"; shift 2;; -f|--force) force=1; shift;; -n|--num-nodes) set_once num_nodes "${2}" "number of nodes"; shift 2;; - -j|--jdk) set_once jdk_version "${2}" "the OpenJDK base image"; shift 2;; -e|--expose-ports) set_once expose_ports "${2}" "the ports to expose"; shift 2;; *) set_once image_name "${1}" "docker image name"; shift;; esac done [[ -n "${num_nodes}" ]] || num_nodes="${default_num_nodes}" - [[ -n "${jdk_version}" ]] || jdk_version="${default_jdk}" - [[ -n "${image_name}" ]] || image_name="${default_image_name}-${jdk_version/:/-}" + [[ -n "${image_name}" ]] || image_name="${default_image_name}-${default_jdk/:/-}" [[ "${num_nodes}" =~ ^-?[0-9]+$ ]] || \ die "ducker_up: the number of nodes must be an integer." [[ "${num_nodes}" -gt 0 ]] || die "ducker_up: the number of nodes must be greater than 0." @@ -306,21 +345,6 @@ use only ${num_nodes}." docker ps >/dev/null || die "ducker_up: failed to run docker. Please check that the daemon is started." - ducker_build "${image_name}" - - docker inspect --format='{{.Config.Labels}}' --type=image "${image_name}" | grep -q 'ducker.type' - local docker_status=${PIPESTATUS[0]} - local grep_status=${PIPESTATUS[1]} - [[ "${docker_status}" -eq 0 ]] || die "ducker_up: failed to inspect image ${image_name}. \ -Please check that it exists." - if [[ "${grep_status}" -ne 0 ]]; then - if [[ "${force}" -ne 1 ]]; then - echo "ducker_up: ${image_name} does not appear to be a ducker image. It lacks the \ -ducker.type label. If you think this is a mistake, you can use --force to attempt to bring \ -it up anyway." - exit 1 - fi - fi local running_containers="$(docker ps -f=network=ducknet -q)" local num_running_containers=$(count ${running_containers}) if [[ ${num_running_containers} -gt 0 ]]; then @@ -330,6 +354,9 @@ attempting to start new ones." fi echo "ducker_up: Bringing up ${image_name} with ${num_nodes} nodes..." + docker image inspect "${image_name}" &>/dev/null || \ + must_do -v -o docker pull "${image_name}" + if docker network inspect ducknet &>/dev/null; then must_do -v docker network rm ducknet fi @@ -359,6 +386,11 @@ attempting to start new ones." echo "ducker_up: added the latest entries to /etc/hosts on each node." generate_cluster_json_file "${num_nodes}" "${ducker_dir}/build/cluster.json" echo "ducker_up: successfully wrote ${ducker_dir}/build/cluster.json" + + # Save docker image id to the file. Then could use this file to find version of docker image that is running. + # It could be useful if we don't confident about necessity of rebuilding image. + get_image_id "${image_name}" > "${ducker_dir}/build/image_id.up" + echo "** ducker_up: successfully brought up ${num_nodes} nodes." } @@ -508,7 +540,8 @@ ducker_down() { running_containers="$(docker ps -f=network=ducknet -q)" [[ $? -eq 0 ]] || die "ducker_down: docker command failed. Is the docker daemon running?" running_containers=${running_containers//$'\n'/ } - local all_containers="$(docker ps -a -f=network=ducknet -q)" + local all_containers + all_containers=$(docker ps -a -f=network=ducknet -q) all_containers=${all_containers//$'\n'/ } if [[ -z "${all_containers}" ]]; then maybe_echo "${verbose}" "No ducker containers found." @@ -519,13 +552,14 @@ ducker_down() { verbose_flag="-v" fi if [[ -n "${running_containers}" ]]; then - must_do ${verbose_flag} docker kill "${running_containers}" + must_do ${verbose_flag} docker kill "${running_containers[@]}" fi must_do ${verbose_flag} docker rm ${force_str} "${all_containers}" must_do ${verbose_flag} -o rm -f -- "${ducker_dir}/build/node_hosts" "${ducker_dir}/build/cluster.json" if docker network inspect ducknet &>/dev/null; then must_do -v docker network rm ducknet fi + rm "${ducker_dir}/build/image_id.up" maybe_echo "${verbose}" "ducker_down: removed $(count ${all_containers}) containers." } @@ -559,6 +593,38 @@ ducker_purge() { must_do -v -o docker rmi ${force_str} ${images} } +get_image_id() { + require_commands docker + local image_name="${1}" + + must_do -o docker image inspect --format "{{.Id}}" "${image_name}" +} + +ducker_compare() { + local cmd="" + + local verbose=1 + local force_str="" + + while [[ $# -ge 1 ]]; do + case "${1}" in + -q|--quiet) verbose=0; cmd="${cmd} ${1}"; shift;; + -f|--force) force_str="-f"; cmd="${cmd} ${1}"; shift;; + *) set_once image_name "${1}" "docker image name"; shift;; + esac + done + + [ -n "${image_name}" ] || image_name="${default_image_name}-${default_jdk/:/-}" + + cmp -s "${ducker_dir}/build/image_${image_name}.build" "${ducker_dir}/build/image_id.up" + local ret="$?" + + if [[ $ret != "0" ]]; then + echo "Docker image ${image_name} is outdated. Stop the cluster" + ducker_down ${cmd} + fi +} + # Parse command-line arguments [[ $# -lt 1 ]] && usage 0 # Display the help text if -h or --help appears in the command line @@ -574,7 +640,7 @@ shift case "${action}" in help) usage 0;; - up|test|ssh|down|purge) + build|up|test|ssh|down|purge|compare) ducker_${action} "${@}"; exit 0;; *) echo "Unknown command '${action}'. Type '${script_path} --help' for usage information." diff --git a/modules/ducktests/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh index e392e3d8d6992..e5dc56113f357 100755 --- a/modules/ducktests/tests/docker/run_tests.sh +++ b/modules/ducktests/tests/docker/run_tests.sh @@ -16,15 +16,128 @@ # limitations under the License. SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +### +# DuckerUp parameters are specified with env variables + +# Num of cotainers that ducktape will prepare for tests IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-11} -TC_PATHS=${TC_PATHS:-./ignitetest/} + +# Image name to run nodes +default_image_name="ducker-ignite-openjdk-8" +IMAGE_NAME="${IMAGE_NAME:-$default_image_name}" + +### +# DuckerTest parameters are specified with options to the script + +# Path to ducktests +TC_PATHS="./ignitetest/" +# Global parameters to pass to ducktape util with --global param +GLOBALS="{}" +# Ducktests parameters to pass to ducktape util with --parameters param +PARAMETERS="{}" + +### +# RunTests parameters +# Force flag: +# - skips ducker-ignite compare step; +# - sends to duck-ignite scripts. +FORCE= + +usage() { + cat < + + + + + + + + + + + + + + + {{ properties }} + + diff --git a/modules/ducktests/tests/ignitetest/services/utils/config/log4j.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/config/log4j.xml.j2 new file mode 100644 index 0000000000000..e8b3be8bbfef1 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/config/log4j.xml.j2 @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 106493c92f77a..578d72864c393 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -18,7 +18,7 @@ from ducktape.services.background_thread import BackgroundThreadService from ducktape.utils.util import wait_until -from ignitetest.services.utils.ignite_config import IgniteConfig +from ignitetest.services.utils.ignite_config import IgniteLoggerConfig, IgniteServerConfig, IgniteClientConfig from ignitetest.services.utils.ignite_path import IgnitePath from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin @@ -41,14 +41,17 @@ class IgniteAwareService(BackgroundThreadService): "collect_default": True} } - def __init__(self, context, num_nodes, version, properties): + def __init__(self, context, num_nodes, client_mode, version, properties): super(IgniteAwareService, self).__init__(context, num_nodes) + self.path = IgnitePath(context) + self.jvm_options = context.globals.get("jvm_opts", "") + self.log_level = "DEBUG" - self.config = IgniteConfig() - self.path = IgnitePath() self.properties = properties self.version = version + self.logger_config = IgniteLoggerConfig() + self.client_mode = client_mode for node in self.nodes: node.version = version @@ -64,9 +67,9 @@ def start_node(self, node): def init_persistent(self, node): node.account.mkdirs(self.PERSISTENT_ROOT) - node.account.create_file(self.CONFIG_FILE, self.config.render( - self.PERSISTENT_ROOT, self.WORK_DIR, properties=self.properties)) - node.account.create_file(self.LOG4J_CONFIG_FILE, self.config.render_log4j(self.WORK_DIR)) + node.account.create_file(self.CONFIG_FILE, self.config().render( + config_dir=self.PERSISTENT_ROOT, work_dir=self.WORK_DIR, properties=self.properties)) + node.account.create_file(self.LOG4J_CONFIG_FILE, self.logger_config.render(work_dir=self.WORK_DIR)) @abstractmethod def start_cmd(self, node): @@ -76,6 +79,12 @@ def start_cmd(self, node): def pids(self, node): raise NotImplementedError + def config(self): + if self.client_mode: + return IgniteClientConfig(self.context) + else: + return IgniteServerConfig(self.context) + def _worker(self, idx, node): cmd = self.start_cmd(node) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index 7b376333af755..fd2c860c6938d 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -17,6 +17,7 @@ from ducktape.services.service import Service from ignitetest.services.utils.ignite_aware import IgniteAwareService +from ignitetest.services.utils.ignite_config import IgniteClientConfig """ The base class to build Ignite aware application written on java. @@ -24,9 +25,9 @@ class IgniteAwareApplicationService(IgniteAwareService): - def __init__(self, context, java_class_name, version, properties, params, timeout_sec, + def __init__(self, context, java_class_name, client_mode, version, properties, params, timeout_sec, service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): - super(IgniteAwareApplicationService, self).__init__(context, 1, version, properties) + super(IgniteAwareApplicationService, self).__init__(context, 1, client_mode, version, properties) self.servicejava_class_name = service_java_class_name self.java_class_name = java_class_name @@ -87,7 +88,7 @@ def jvm_opts(self): "-J-Dlog4j.configDebug=true " \ "-J-Xmx1G " \ "-J-ea " \ - "-J-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false " + "-J-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false " + self.jvm_options def env(self): return "export MAIN_CLASS={main_class}; ".format(main_class=self.servicejava_class_name) + \ diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py index 5ffaa81881b73..077c39bab9e6b 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py @@ -17,71 +17,49 @@ This module renders Ignite config and all related artifacts """ +from jinja2 import FileSystemLoader, Environment + +import os + +DEFAULT_CONFIG_PATH = os.path.dirname(os.path.abspath(__file__)) + "/config" +DEFAULT_IGNITE_CONF = DEFAULT_CONFIG_PATH + "/ignite.xml.j2" + + +class Config(object): + def __init__(self, path): + tmpl_dir = os.path.dirname(path) + tmpl_file = os.path.basename(path) + + tmpl_loader = FileSystemLoader(searchpath=tmpl_dir) + env = Environment(loader=tmpl_loader) + + self.template = env.get_template(tmpl_file) + self.default_params = {} + + def render(self, **kwargs): + kwargs.update(self.default_params) + res = self.template.render(**kwargs) + return res + + +class IgniteServerConfig(Config): + def __init__(self, context): + path = DEFAULT_IGNITE_CONF + if 'ignite_server_config_path' in context.globals: + path = context.globals['ignite_server_config_path'] + super(IgniteServerConfig, self).__init__(path) + + +class IgniteClientConfig(Config): + def __init__(self, context): + path = DEFAULT_IGNITE_CONF + if 'ignite_client_config_path' in context.globals: + path = context.globals['ignite_client_config_path'] + super(IgniteClientConfig, self).__init__(path) + self.default_params.update(client_mode=True) + + +class IgniteLoggerConfig(Config): + def __init__(self): + super(IgniteLoggerConfig, self).__init__(DEFAULT_CONFIG_PATH + "/log4j.xml.j2") -class IgniteConfig: - def __init__(self, project="ignite"): - self.project = project - - def render(self, config_dir, work_dir, properties=""): - return """ - - - - - - - - - - {properties} - - - """.format(config_dir=config_dir, - work_dir=work_dir, - properties=properties) - - def render_log4j(self, work_dir): - return """ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - """.format(work_dir=work_dir) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py index 8e4603a288ac2..4eab4a43ecb9d 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py @@ -34,8 +34,8 @@ class IgnitePath: ... """ - def __init__(self, project="ignite"): - self.project = project + def __init__(self, context): + self.project = context.globals.get("project", "ignite") def home(self, node_or_version, project=None): version = self._version(node_or_version) diff --git a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py index a55863803cb0f..a3c9546728741 100644 --- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py @@ -32,12 +32,6 @@ class AddNodeRebalanceTest(IgniteTest): Test performs rebalance tests. """ - @staticmethod - def properties(client_mode="false"): - return """ - - """.format(client_mode=client_mode) - def __init__(self, test_context): super(AddNodeRebalanceTest, self).__init__(test_context=test_context) @@ -70,7 +64,6 @@ def test_add_node(self, version): # This client just put some data to the cache. IgniteApplicationService(self.test_context, java_class_name="org.apache.ignite.internal.ducktest.tests.DataGenerationApplication", - properties=self.properties(client_mode="true"), version=ignite_version, params="test-cache,%d" % self.DATA_AMOUNT, timeout_sec=self.PRELOAD_TIMEOUT).run() diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 246416a6ce26f..01aebe4dda62a 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -28,7 +28,6 @@ class DiscoveryTest(IgniteTest): NUM_NODES = 7 CONFIG_TEMPLATE = """ - {% if zookeeper_settings %} {% with zk = zookeeper_settings %} @@ -49,9 +48,9 @@ def __init__(self, test_context): self.servers = None @staticmethod - def properties(client_mode="false", zookeeper_settings=None): + def properties(zookeeper_settings=None): return Template(DiscoveryTest.CONFIG_TEMPLATE) \ - .render(client_mode=client_mode, zookeeper_settings=zookeeper_settings) + .render(zookeeper_settings=zookeeper_settings) def setUp(self): pass diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index 89653e9f9b8c6..0df2f84dca5d6 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -27,9 +27,8 @@ class PmeFreeSwitchTest(IgniteTest): NUM_NODES = 3 @staticmethod - def properties(client_mode="false"): + def properties(): return """ - @@ -39,7 +38,7 @@ def properties(client_mode="false"): - """.format(client_mode=client_mode) + """ def __init__(self, test_context): super(PmeFreeSwitchTest, self).__init__(test_context=test_context) @@ -73,7 +72,7 @@ def test(self, version): long_tx_streamer = IgniteApplicationService( self.test_context, java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.LongTxStreamerApplication", - properties=self.properties(client_mode="true"), + properties=self.properties(), params="test-cache", version=ignite_version) @@ -84,7 +83,7 @@ def test(self, version): single_key_tx_streamer = IgniteApplicationService( self.test_context, java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.SingleKeyTxStreamerApplication", - properties=self.properties(client_mode="true"), + properties=self.properties(), params="test-cache,1000", version=ignite_version) diff --git a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py index 45c0a385e22ad..7e2b4fc4b854f 100644 --- a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py +++ b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py @@ -13,11 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ducktape.mark import parametrize + from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.ignite_spark_app import SparkIgniteApplicationService from ignitetest.services.spark import SparkService from ignitetest.tests.utils.ignite_test import IgniteTest +from ignitetest.tests.utils.version import DEV_BRANCH class SparkIntegrationTest(IgniteTest): @@ -28,33 +31,33 @@ class SparkIntegrationTest(IgniteTest): 3. Checks results of client application. """ - @staticmethod - def properties(client_mode="false"): - return """ - - """.format(client_mode=client_mode) - def __init__(self, test_context): super(SparkIntegrationTest, self).__init__(test_context=test_context) - self.spark = SparkService(test_context, num_nodes=2) - self.ignite = IgniteService(test_context, num_nodes=1) + self.spark = None + self.ignite = None def setUp(self): - self.spark.start() - self.ignite.start() + pass def teardown(self): self.spark.stop() self.ignite.stop() - def test_spark_client(self): + @parametrize(version=str(DEV_BRANCH)) + def test_spark_client(self, version): + self.spark = SparkService(self.test_context, version=version, num_nodes=2) + self.spark.start() + + self.ignite = IgniteService(self.test_context, version=version, num_nodes=1) + self.ignite.start() + self.stage("Starting sample data generator") IgniteApplicationService( self.test_context, java_class_name="org.apache.ignite.internal.ducktest.tests.spark_integration_test.SampleDataStreamerApplication", params="cache,1000", - properties=self.properties(client_mode="true")).run() + version=version).run() self.stage("Starting Spark application") @@ -62,4 +65,5 @@ def test_spark_client(self): self.test_context, "org.apache.ignite.internal.ducktest.tests.spark_integration_test.SparkApplication", params="spark://" + self.spark.nodes[0].account.hostname + ":7077", + version=version, timeout_sec=120).run() diff --git a/modules/ducktests/tests/ignitetest/tests/utils/version.py b/modules/ducktests/tests/ignitetest/tests/utils/version.py index 9ba5c8e3c2eae..2f1f003f7d7ff 100644 --- a/modules/ducktests/tests/ignitetest/tests/utils/version.py +++ b/modules/ducktests/tests/ignitetest/tests/utils/version.py @@ -15,6 +15,8 @@ from distutils.version import LooseVersion +from ducktape.cluster.cluster import ClusterNode + from ignitetest import __version__ @@ -56,14 +58,16 @@ def get_version(node=None): Return the version attached to the given node. Default to DEV_BRANCH if node or node.version is undefined (aka None) """ - if node is not None and hasattr(node, "version") and node.version is not None: - return node.version - else: - return DEV_BRANCH + if isinstance(node, ClusterNode) and hasattr(node, 'version'): + return getattr(node, 'version') + + if isinstance(node, str) or isinstance(node, unicode): + return node + + return DEV_BRANCH DEV_BRANCH = IgniteVersion("dev") -DEV_VERSION = IgniteVersion("2.9.0-SNAPSHOT") # 2.7.x versions V_2_7_6 = IgniteVersion("2.7.6") From 330c17b084aaf91d19591178885dd2637dc8fa21 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Thu, 30 Jul 2020 12:11:59 +0300 Subject: [PATCH 30/78] Ducktape codestyle (#8098) --- modules/ducktests/tests/.pylintrc | 24 ++++++ modules/ducktests/tests/check_style.sh | 26 ++++++ .../ducktests/tests/ignitetest/__init__.py | 1 + .../tests/ignitetest/services/ignite.py | 27 ++++-- .../tests/ignitetest/services/ignite_app.py | 12 ++- .../ignitetest/services/ignite_spark_app.py | 6 +- .../tests/ignitetest/services/spark.py | 40 ++++++--- .../ignitetest/services/utils/decorators.py | 8 ++ .../ignitetest/services/utils/ignite_aware.py | 59 +++++++++++-- .../services/utils/ignite_aware_app.py | 32 +++++-- .../services/utils/ignite_config.py | 21 ++++- .../ignitetest/services/utils/ignite_path.py | 45 ++++++---- .../ignitetest/services/utils/jmx_utils.py | 85 +++++++++++++++++-- .../tests/ignitetest/services/zk/zookeeper.py | 39 ++++++++- .../tests/add_node_rebalance_test.py | 12 ++- .../tests/ignitetest/tests/discovery_test.py | 44 ++++++++-- .../ignitetest/tests/pme_free_switch_test.py | 18 +++- .../tests/spark_integration_test.py | 11 ++- .../ignitetest/tests/utils/ignite_test.py | 12 +++ .../tests/ignitetest/tests/utils/version.py | 10 ++- 20 files changed, 447 insertions(+), 85 deletions(-) create mode 100644 modules/ducktests/tests/.pylintrc create mode 100755 modules/ducktests/tests/check_style.sh diff --git a/modules/ducktests/tests/.pylintrc b/modules/ducktests/tests/.pylintrc new file mode 100644 index 0000000000000..73536f333a79b --- /dev/null +++ b/modules/ducktests/tests/.pylintrc @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +[BASIC] +min-public-methods=0 +# TODO: Remove after migrating to python 3 +disable=R0205 + +[SIMILARITIES] +ignore-imports=yes + +[FORMAT] +max-line-length=120 diff --git a/modules/ducktests/tests/check_style.sh b/modules/ducktests/tests/check_style.sh new file mode 100755 index 0000000000000..39405a7990037 --- /dev/null +++ b/modules/ducktests/tests/check_style.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if ! command -v pylint &> /dev/null +then + echo "Please, install pylint first" + exit 1 +fi + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +pylint --rcfile="$SCRIPT_DIR"/.pylintrc "$SCRIPT_DIR"/ignitetest diff --git a/modules/ducktests/tests/ignitetest/__init__.py b/modules/ducktests/tests/ignitetest/__init__.py index 1c2357fec3ec4..8cbaed8b6c166 100644 --- a/modules/ducktests/tests/ignitetest/__init__.py +++ b/modules/ducktests/tests/ignitetest/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=missing-module-docstring # This determines the version of ignitetest that can be published to PyPi and installed with pip # # Note that in development, this version name can't follow Ignite's convention of having a trailing "-SNAPSHOT" diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index 05f108efc2213..bfc96ade7db85 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module contains class to start ignite cluster node. +""" + import os.path import signal @@ -20,11 +24,13 @@ from ducktape.utils.util import wait_until from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.services.utils.ignite_config import IgniteServerConfig, IgniteClientConfig from ignitetest.tests.utils.version import DEV_BRANCH class IgniteService(IgniteAwareService): + """ + Ignite node service. + """ APP_SERVICE_CLASS = "org.apache.ignite.startup.cmdline.CommandLineStartup" HEAP_DUMP_FILE = os.path.join(IgniteAwareService.PERSISTENT_ROOT, "ignite-heap.bin") @@ -38,9 +44,11 @@ class IgniteService(IgniteAwareService): "collect_default": False} } + # pylint: disable=R0913 def __init__(self, context, num_nodes, client_mode=False, version=DEV_BRANCH, properties=""): super(IgniteService, self).__init__(context, num_nodes, client_mode, version, properties) + # pylint: disable=W0221 def start(self, timeout_sec=180): super(IgniteService, self).start() @@ -66,11 +74,17 @@ def start_cmd(self, node): return cmd def await_node_started(self, node, timeout_sec): + """ + Await topology ready event on node start. + :param node: Node to wait + :param timeout_sec: Number of seconds to wait event. + """ self.await_event_on_node("Topology snapshot", node, timeout_sec, from_the_beginning=True) if len(self.pids(node)) == 0: raise Exception("No process ids recorded on node %s" % node.account.hostname) + # pylint: disable=W0221 def stop_node(self, node, clean_shutdown=True, timeout_sec=60): pids = self.pids(node) sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL @@ -90,17 +104,20 @@ def clean_node(self, node): node.account.ssh("sudo rm -rf -- %s" % IgniteService.PERSISTENT_ROOT, allow_fail=False) def thread_dump(self, node): + """ + Generate thread dump on node. + :param node: Ignite service node. + """ for pid in self.pids(node): try: node.account.signal(pid, signal.SIGQUIT, allow_fail=True) - except: + except RemoteCommandError: self.logger.warn("Could not dump threads on node") def pids(self, node): - """Return process ids associated with running processes on the given node.""" try: cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.APP_SERVICE_CLASS - pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] + pid_arr = list(node.account.ssh_capture(cmd, allow_fail=True, callback=int)) return pid_arr - except (RemoteCommandError, ValueError) as e: + except (RemoteCommandError, ValueError): return [] diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 64de176456f59..07022210fe814 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -13,17 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService -from ignitetest.tests.utils.version import DEV_BRANCH - """ -The Ignite application service allows to perform custom logic writen on java. +This module contains the ignite application service allows to perform custom logic writen on java. """ +from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService +from ignitetest.tests.utils.version import DEV_BRANCH + class IgniteApplicationService(IgniteAwareApplicationService): + """ + The Ignite application service allows to perform custom logic writen on java. + """ service_java_class_name = "org.apache.ignite.internal.ducktest.utils.IgniteApplicationService" + # pylint: disable=R0913 def __init__(self, context, java_class_name, client_mode=True, version=DEV_BRANCH, properties="", params="", timeout_sec=60): super(IgniteApplicationService, self).__init__(context, java_class_name, client_mode, version, properties, diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py index 775f19c1c1a69..71014b7bc15ea 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py @@ -14,13 +14,17 @@ # limitations under the License. """ -The Ignite-Spark application service. +This module contains the Ignite-Spark application service. """ from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService from ignitetest.tests.utils.version import DEV_BRANCH class SparkIgniteApplicationService(IgniteAwareApplicationService): + """ + The Ignite-Spark application service. + """ + # pylint: disable=R0913 def __init__(self, context, java_class_name, client_mode=True, version=DEV_BRANCH, properties="", params="", timeout_sec=60): super(SparkIgniteApplicationService, self).__init__(context, java_class_name, client_mode, version, properties, diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py index fdd5c738ea356..657a6b6a4b141 100644 --- a/modules/ducktests/tests/ignitetest/services/spark.py +++ b/modules/ducktests/tests/ignitetest/services/spark.py @@ -13,18 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module contains spark service class. +""" + import os.path from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.services.service import Service from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.services.utils.ignite_config import IgniteServerConfig - from ignitetest.tests.utils.version import DEV_BRANCH class SparkService(IgniteAwareService): + """ + Start a spark node. + """ INSTALL_DIR = "/opt/spark-{version}".format(version="2.3.4") SPARK_PERSISTENT_ROOT = "/mnt/spark" @@ -68,6 +73,7 @@ def start_cmd(self, node): return cmd + # pylint: disable=W0221 def start_node(self, node, timeout_sec=30): self.init_persistent(node) @@ -91,7 +97,7 @@ def start_node(self, node, timeout_sec=30): if len(self.pids(node)) == 0: raise Exception("No process ids recorded on node %s" % node.account.hostname) - def stop_node(self, node, clean_shutdown=True, timeout_sec=60): + def stop_node(self, node): if node == self.nodes[0]: node.account.ssh(os.path.join(SparkService.INSTALL_DIR, "sbin", "stop-master.sh")) else: @@ -103,28 +109,40 @@ def clean_node(self, node): node.account.ssh("sudo rm -rf -- %s" % SparkService.SPARK_PERSISTENT_ROOT, allow_fail=False) def pids(self, node): - """Return process ids associated with running processes on the given node.""" try: cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.java_class_name(node) - pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)] - return pid_arr - except (RemoteCommandError, ValueError) as e: + return list(node.account.ssh_capture(cmd, allow_fail=True, callback=int)) + except (RemoteCommandError, ValueError): return [] def java_class_name(self, node): + """ + :param node: Spark node. + :return: Class name depending on node type (master or slave). + """ if node == self.nodes[0]: return "org.apache.spark.deploy.master.Master" - else: - return "org.apache.spark.deploy.worker.Worker" - def master_log_path(self, node): + return "org.apache.spark.deploy.worker.Worker" + + @staticmethod + def master_log_path(node): + """ + :param node: Spark master node. + :return: Path to log file. + """ return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.master.Master-{instance}-{host}.out".format( SPARK_LOG_DIR=SparkService.SPARK_PERSISTENT_ROOT, userID=node.account.user, instance=1, host=node.account.hostname) - def slave_log_path(self, node): + @staticmethod + def slave_log_path(node): + """ + :param node: Spark slave node. + :return: Path to log file. + """ return "{SPARK_LOG_DIR}/spark-{userID}-org.apache.spark.deploy.worker.Worker-{instance}-{host}.out".format( SPARK_LOG_DIR=SparkService.SPARK_PERSISTENT_ROOT, userID=node.account.user, diff --git a/modules/ducktests/tests/ignitetest/services/utils/decorators.py b/modules/ducktests/tests/ignitetest/services/utils/decorators.py index 788ddba24d139..9c767f4efbee4 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/decorators.py +++ b/modules/ducktests/tests/ignitetest/services/utils/decorators.py @@ -12,11 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +This module contains various useful decorators. +""" + import functools from threading import RLock def memoize(func): + """ + Decorate function to memoize first call to thread safe cache. + """ cache = func.cache = {} lock = RLock() diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 578d72864c393..b4d12f0964a22 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -12,6 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +This module contains the base class to build services aware of Ignite. +""" + import os from abc import abstractmethod @@ -22,12 +27,11 @@ from ignitetest.services.utils.ignite_path import IgnitePath from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin -""" -The base class to build services aware of Ignite. -""" - class IgniteAwareService(BackgroundThreadService): + """ + The base class to build services aware of Ignite. + """ # Root directory for persistent output PERSISTENT_ROOT = "/mnt/service" STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") @@ -41,6 +45,7 @@ class IgniteAwareService(BackgroundThreadService): "collect_default": True} } + # pylint: disable=R0913 def __init__(self, context, num_nodes, client_mode, version, properties): super(IgniteAwareService, self).__init__(context, num_nodes) @@ -66,6 +71,10 @@ def start_node(self, node): ignite_jmx_mixin(node, self.pids(node)) def init_persistent(self, node): + """ + Init persistent directory. + :param node: Ignite service node. + """ node.account.mkdirs(self.PERSISTENT_ROOT) node.account.create_file(self.CONFIG_FILE, self.config().render( config_dir=self.PERSISTENT_ROOT, work_dir=self.WORK_DIR, properties=self.properties)) @@ -73,18 +82,30 @@ def init_persistent(self, node): @abstractmethod def start_cmd(self, node): + """ + Start up command for service. + :param node: Ignite service node. + """ raise NotImplementedError @abstractmethod def pids(self, node): + """ + :param node: Ignite service node. + :return: List of service's pids. + """ raise NotImplementedError def config(self): + """ + :return: Ignite node configuration. + """ if self.client_mode: return IgniteClientConfig(self.context) - else: - return IgniteServerConfig(self.context) + return IgniteServerConfig(self.context) + + # pylint: disable=W0613 def _worker(self, idx, node): cmd = self.start_cmd(node) @@ -93,9 +114,23 @@ def _worker(self, idx, node): node.account.ssh(cmd) def alive(self, node): + """ + :param node: Ignite service node. + :return: True if node is alive. + """ return len(self.pids(node)) > 0 + # pylint: disable=R0913 def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning=False, backoff_sec=5): + """ + Await for specific event message in a node's log file. + :param evt_message: Event message. + :param node: Ignite service node. + :param timeout_sec: Number of seconds to check the condition for before failing. + :param from_the_beginning: If True, search for message from the beggining of log file. + :param backoff_sec: Number of seconds to back off between each failure to meet the condition + before checking again. + """ with node.account.monitor_log(self.STDOUT_STDERR_CAPTURE) as monitor: if from_the_beginning: monitor.offset = 0 @@ -104,12 +139,24 @@ def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning err_msg="Event [%s] was not triggered in %d seconds" % (evt_message, timeout_sec)) def await_event(self, evt_message, timeout_sec, from_the_beginning=False, backoff_sec=5): + """ + Await for specific event messages on all nodes. + :param evt_message: Event message. + :param timeout_sec: Number of seconds to check the condition for before failing. + :param from_the_beginning: If True, search for message from the beggining of log file. + :param backoff_sec: Number of seconds to back off between each failure to meet the condition + before checking again. + """ assert len(self.nodes) == 1 self.await_event_on_node(evt_message, self.nodes[0], timeout_sec, from_the_beginning=from_the_beginning, backoff_sec=backoff_sec) def execute(self, command): + """ + Execute command on all nodes. + :param command: Command to execute. + """ for node in self.nodes: cmd = "%s 1>> %s 2>> %s" % \ (self.path.script(command, node), diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index fd2c860c6938d..152b5fe2038a5 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -12,19 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import re - -from ducktape.services.service import Service - -from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.services.utils.ignite_config import IgniteClientConfig """ -The base class to build Ignite aware application written on java. +This module contains the base class to build Ignite aware application written on java. """ +import re + +from ignitetest.services.utils.ignite_aware import IgniteAwareService + class IgniteAwareApplicationService(IgniteAwareService): + """ + The base class to build Ignite aware application written on java. + """ + # pylint: disable=R0913 def __init__(self, context, java_class_name, client_mode, version, properties, params, timeout_sec, service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): super(IgniteAwareApplicationService, self).__init__(context, 1, client_mode, version, properties) @@ -53,9 +55,10 @@ def start_cmd(self, node): self.STDOUT_STDERR_CAPTURE) return cmd + # pylint: disable=W0221 def stop_node(self, node, clean_shutdown=True, timeout_sec=20): self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) - node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=True, allow_fail=True) + node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=clean_shutdown, allow_fail=True) stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec) assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ @@ -73,6 +76,9 @@ def clean_node(self, node): node.account.ssh("rm -rf %s" % self.PERSISTENT_ROOT, allow_fail=False) def app_args(self): + """ + :return: Application arguments. + """ args = self.java_class_name + "," + IgniteAwareApplicationService.CONFIG_FILE if self.params != "": @@ -84,6 +90,9 @@ def pids(self, node): return node.account.java_pids(self.servicejava_class_name) def jvm_opts(self): + """ + :return: Application JVM options. + """ return "-J-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file " + \ "-J-Dlog4j.configDebug=true " \ "-J-Xmx1G " \ @@ -91,6 +100,9 @@ def jvm_opts(self): "-J-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false " + self.jvm_options def env(self): + """ + :return: Export string of additional environment variables. + """ return "export MAIN_CLASS={main_class}; ".format(main_class=self.servicejava_class_name) + \ "export EXCLUDE_TEST_CLASSES=true; " + \ "export IGNITE_LOG_DIR={log_dir}; ".format(log_dir=self.PERSISTENT_ROOT) + \ @@ -98,6 +110,10 @@ def env(self): % self.path.home(self.version) def extract_result(self, name): + """ + :param name: Result parameter's name. + :return: Extracted result of application run. + """ res = "" output = self.nodes[0].account.ssh_capture( diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py index 077c39bab9e6b..90b3bfa970754 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py @@ -14,18 +14,20 @@ # limitations under the License. """ -This module renders Ignite config and all related artifacts +This module contains ignite config classes and utilities. """ +import os from jinja2 import FileSystemLoader, Environment -import os - DEFAULT_CONFIG_PATH = os.path.dirname(os.path.abspath(__file__)) + "/config" DEFAULT_IGNITE_CONF = DEFAULT_CONFIG_PATH + "/ignite.xml.j2" class Config(object): + """ + Basic configuration. + """ def __init__(self, path): tmpl_dir = os.path.dirname(path) tmpl_file = os.path.basename(path) @@ -37,12 +39,18 @@ def __init__(self, path): self.default_params = {} def render(self, **kwargs): + """ + Render configuration. + """ kwargs.update(self.default_params) res = self.template.render(**kwargs) return res class IgniteServerConfig(Config): + """ + Ignite server node configuration. + """ def __init__(self, context): path = DEFAULT_IGNITE_CONF if 'ignite_server_config_path' in context.globals: @@ -51,6 +59,9 @@ def __init__(self, context): class IgniteClientConfig(Config): + """ + Ignite client node configuration. + """ def __init__(self, context): path = DEFAULT_IGNITE_CONF if 'ignite_client_config_path' in context.globals: @@ -60,6 +71,8 @@ def __init__(self, context): class IgniteLoggerConfig(Config): + """ + Ignite logger configuration. + """ def __init__(self): super(IgniteLoggerConfig, self).__init__(DEFAULT_CONFIG_PATH + "/log4j.xml.j2") - diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py index 4eab4a43ecb9d..4d24d36b72658 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py @@ -13,32 +13,36 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module contains ignite path resolve utilities. +""" + import os from ignitetest.tests.utils.version import get_version, IgniteVersion -""" -This module provides Ignite path methods -""" - class IgnitePath: - SCRATCH_ROOT = "/mnt" - IGNITE_INSTALL_ROOT = "/opt" - """Path resolver for Ignite system tests which assumes the following layout: - /opt/ignite-dev # Current version of Ignite under test - /opt/ignite-2.7.6 # Example of an older version of Ignite installed from tarball - /opt/ignite- # Other previous versions of Ignite - ... - """ + /opt/ignite-dev # Current version of Ignite under test + /opt/ignite-2.7.6 # Example of an older version of Ignite installed from tarball + /opt/ignite- # Other previous versions of Ignite + ... + """ + SCRATCH_ROOT = "/mnt" + IGNITE_INSTALL_ROOT = "/opt" def __init__(self, context): self.project = context.globals.get("project", "ignite") def home(self, node_or_version, project=None): - version = self._version(node_or_version) + """ + :param node_or_version: Ignite service node or IgniteVersion instance. + :param project: Project name. + :return: Home directory. + """ + version = self.__version(node_or_version) home_dir = project or self.project if version is not None: home_dir += "-%s" % str(version) @@ -46,11 +50,18 @@ def home(self, node_or_version, project=None): return os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir) def script(self, script_name, node_or_version, project=None): - version = self._version(node_or_version) + """ + :param script_name: Script name. + :param node_or_version: Ignite service node or IgniteVersion instance. + :param project: Project name. + :return: Full path to script. + """ + version = self.__version(node_or_version) return os.path.join(self.home(version, project=project), "bin", script_name) - def _version(self, node_or_version): + @staticmethod + def __version(node_or_version): if isinstance(node_or_version, IgniteVersion): return node_or_version - else: - return get_version(node_or_version) + + return get_version(node_or_version) diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py index 6843c945dfcad..bfbab25f9d00a 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py @@ -12,12 +12,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +This module contains JMX Console client and different utilities and mixins to retrieve ignite node parameters +and attributes. +""" + import re from ignitetest.services.utils.decorators import memoize def ignite_jmx_mixin(node, pids): + """ + Dynamically mixin JMX attributes to Ignite service node. + :param node: Ignite service node. + :param pids: Ignite service node pids. + """ setattr(node, 'pids', pids) base_cls = node.__class__ base_cls_name = node.__class__.__name__ @@ -25,15 +36,25 @@ def ignite_jmx_mixin(node, pids): class JmxMBean(object): + """ + Dynamically exposes JMX MBean attributes. + """ def __init__(self, client, name): self.client = client self.name = name def __getattr__(self, attr): + """ + Retrieves through JMX client MBean attributes. + :param attr: Attribute name. + :return: Attribute value. + """ return self.client.mbean_attribute(self.name, attr) class JmxClient(object): + """JMX client, invokes jmxterm on node locally. + """ jmx_util_cmd = 'java -jar /opt/jmxterm.jar -v silent -n' def __init__(self, node): @@ -42,50 +63,82 @@ def __init__(self, node): @memoize def find_mbean(self, pattern, domain='org.apache'): + """ + Find mbean by specified pattern and domain on node. + :param pattern: MBean name pattern. + :param domain: Domain of MBean + :return: JmxMBean instance + """ cmd = "echo $'open %s\\n beans -d %s \\n close' | %s | grep -o '%s'" \ % (self.pid, domain, self.jmx_util_cmd, pattern) - name = next(self.run_cmd(cmd)).strip() + name = next(self.__run_cmd(cmd)).strip() return JmxMBean(self, name) def mbean_attribute(self, mbean, attr): + """ + Get MBean attribute. + :param mbean: MBean name + :param attr: Attribute name + :return: Attribute value + """ cmd = "echo $'open %s\\n get -b %s %s \\n close' | %s | sed 's/%s = \\(.*\\);/\\1/'" \ % (self.pid, mbean, attr, self.jmx_util_cmd, attr) - return iter(s.strip() for s in self.run_cmd(cmd)) + return iter(s.strip() for s in self.__run_cmd(cmd)) - def run_cmd(self, cmd): + def __run_cmd(self, cmd): return self.node.account.ssh_capture(cmd, allow_fail=False, callback=str) class DiscoveryInfo(object): + """ Ignite service node discovery info, obtained from DiscoverySpi mbean. + """ def __init__(self, coordinator, local_raw): self._local_raw = local_raw self._coordinator = coordinator @property - def id(self): + def node_id(self): + """ + :return: Local node id. + """ return self.__find__("id=([^\\s]+),") @property def coordinator(self): + """ + :return: Coordinator node id. + """ return self._coordinator @property def consistent_id(self): + """ + :return: Node consistent id, if presents (only in TcpDiscovery). + """ return self.__find__("consistentId=([^\\s]+),") @property def is_client(self): + """ + :return: True if node is client. + """ return self.__find__("isClient=([^\\s]+),") == "true" @property def order(self): + """ + :return: Topology order. + """ return int(self.__find__("order=(\\d+),")) @property def int_order(self): + """ + :return: Internal order (TcpDiscovery). + """ val = self.__find__("intOrder=(\\d+),") return int(val) if val else -1 @@ -95,16 +148,28 @@ def __find__(self, pattern): class IgniteJmxMixin(object): + """ + Mixin to IgniteService node, exposing useful properties, obtained from JMX. + """ @memoize def jmx_client(self): + """ + :return: JmxClient instance. + """ # noinspection PyTypeChecker return JmxClient(self) @memoize - def id(self): + def node_id(self): + """ + :return: Local node id. + """ return next(self.kernal_mbean().LocalNodeId).strip() def discovery_info(self): + """ + :return: DiscoveryInfo instance. + """ disco_mbean = self.disco_mbean() crd = next(disco_mbean.Coordinator).strip() local = next(disco_mbean.LocalNodeFormatted).strip() @@ -112,13 +177,19 @@ def discovery_info(self): return DiscoveryInfo(crd, local) def kernal_mbean(self): + """ + :return: IgniteKernal MBean. + """ return self.jmx_client().find_mbean('.*group=Kernal,name=IgniteKernal') @memoize def disco_mbean(self): + """ + :return: DiscoverySpi MBean. + """ disco_spi = next(self.kernal_mbean().DiscoverySpiFormatted).strip() if 'ZookeeperDiscoverySpi' in disco_spi: return self.jmx_client().find_mbean('.*group=SPIs,name=ZookeeperDiscoverySpi') - else: - return self.jmx_client().find_mbean('.*group=SPIs,name=TcpDiscoverySpi') + + return self.jmx_client().find_mbean('.*group=SPIs,name=TcpDiscoverySpi') diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py index d32cbbc18e486..5c29c66cbbbd8 100644 --- a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py +++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py @@ -12,12 +12,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +This module contains classes and utilities to start zookeeper cluster for testing ZookeeperDiscovery. +""" + import os.path from ducktape.services.service import Service class ZookeeperSettings: + """ + Settings for zookeeper quorum nodes. + """ def __init__(self, tick_time=1000, init_limit=10, sync_limit=5, client_port=2181): self.tick_time = tick_time self.init_limit = init_limit @@ -26,6 +34,9 @@ def __init__(self, tick_time=1000, init_limit=10, sync_limit=5, client_port=2181 class ZookeeperService(Service): + """ + Zookeeper service. + """ PERSISTENT_ROOT = "/mnt/zookeeper" CONFIG_ROOT = os.path.join(PERSISTENT_ROOT, "conf") LOG_FILE = os.path.join(PERSISTENT_ROOT, "zookeeper.log") @@ -41,16 +52,17 @@ class ZookeeperService(Service): } } - def __init__(self, context, num_nodes, settings=ZookeeperSettings()): + def __init__(self, context, num_nodes, settings=ZookeeperSettings(), start_timeout_sec=60): super(ZookeeperService, self).__init__(context, num_nodes) self.settings = settings + self.start_timeout_sec = start_timeout_sec - def start(self, timeout_sec=60): - Service.start(self) + def start(self): + super(ZookeeperService, self).start() self.logger.info("Waiting for Zookeeper quorum...") for node in self.nodes: - self.await_quorum(node, timeout_sec) + self.await_quorum(node, self.start_timeout_sec) self.logger.info("Zookeeper quorum is formed.") @@ -89,6 +101,11 @@ def wait_node(self, node, timeout_sec=20): self.logger.info("Zookeeper node %d started on %s", idx, node.account.hostname) def await_quorum(self, node, timeout): + """ + Await quorum formed on node (leader election ready). + :param node: Zookeeper service node. + :param timeout: Wait timeout. + """ with node.account.monitor_log(self.LOG_FILE) as monitor: monitor.offset = 0 monitor.wait_until( @@ -103,12 +120,26 @@ def java_class_name(): return "org.apache.zookeeper.server.quorum.QuorumPeerMain" def pids(self, node): + """ + Get pids of zookeeper service node. + :param node: Zookeeper service node. + :return: List of pids. + """ return node.account.java_pids(self.java_class_name()) def alive(self, node): + """ + Check if zookeeper service node is alive. + :param node: Zookeeper service node. + :return: True if node is alive + """ return len(self.pids(node)) > 0 def connection_string(self): + """ + Form a connection string to zookeeper cluster. + :return: Connection string. + """ return ','.join([node.account.hostname + ":" + str(2181) for node in self.nodes]) def stop_node(self, node): diff --git a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py index a3c9546728741..d8366819e5afd 100644 --- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Module contains node rebalance tests. +""" + from ducktape.mark import parametrize from ducktape.mark.resource import cluster @@ -22,16 +26,16 @@ from ignitetest.tests.utils.version import DEV_BRANCH, IgniteVersion, LATEST +# pylint: disable=W0223 class AddNodeRebalanceTest(IgniteTest): + """ + Test basic rebalance scenarios. + """ NUM_NODES = 4 PRELOAD_TIMEOUT = 60 DATA_AMOUNT = 1000000 REBALANCE_TIMEOUT = 60 - """ - Test performs rebalance tests. - """ - def __init__(self, test_context): super(AddNodeRebalanceTest, self).__init__(test_context=test_context) diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 01aebe4dda62a..d8d4e0b0ff489 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -12,6 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +Module contains discovery tests. +""" + import random from ducktape.mark import parametrize @@ -24,7 +29,14 @@ from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7 +# pylint: disable=W0223 class DiscoveryTest(IgniteTest): + """ + Test basic discovery scenarious (TCP and Zookeeper). + 1. Start of ignite cluster. + 2. Kill random node. + 3. Wait that survived node detects node failure. + """ NUM_NODES = 7 CONFIG_TEMPLATE = """ @@ -44,11 +56,15 @@ class DiscoveryTest(IgniteTest): def __init__(self, test_context): super(DiscoveryTest, self).__init__(test_context=test_context) - self.zk = None + self.zk_quorum = None self.servers = None @staticmethod def properties(zookeeper_settings=None): + """ + :param zookeeper_settings: ZookeperDiscoverySpi settings. If None, TcpDiscoverySpi will be used. + :return: Rendered node's properties. + """ return Template(DiscoveryTest.CONFIG_TEMPLATE) \ .render(zookeeper_settings=zookeeper_settings) @@ -56,8 +72,8 @@ def setUp(self): pass def teardown(self): - if self.zk: - self.zk.stop() + if self.zk_quorum: + self.zk_quorum.stop() if self.servers: self.servers.stop() @@ -66,20 +82,26 @@ def teardown(self): @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_7)) def test_tcp(self, version): + """ + Test basic discovery scenario with TcpDiscoverySpi. + """ return self.__basic_test__(version, False) @cluster(num_nodes=NUM_NODES + 3) @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_7)) def test_zk(self, version): + """ + Test basic discovery scenario with ZookeeperDiscoverySpi. + """ return self.__basic_test__(version, True) def __basic_test__(self, version, with_zk=False): if with_zk: - self.zk = ZookeeperService(self.test_context, 3) + self.zk_quorum = ZookeeperService(self.test_context, 3) self.stage("Starting Zookeper quorum") - self.zk.start() - properties = self.properties(zookeeper_settings={'connection_string': self.zk.connection_string()}) + self.zk_quorum.start() + properties = self.properties(zookeeper_settings={'connection_string': self.zk_quorum.connection_string()}) self.stage("Zookeper quorum started") else: properties = self.properties() @@ -94,19 +116,19 @@ def __basic_test__(self, version, with_zk=False): start = self.monotonic() self.servers.start() - data = {'Ignite cluster start time (s)': self.monotonic() - start } + data = {'Ignite cluster start time (s)': self.monotonic() - start} self.stage("Topology is ready") # Node failure detection fail_node, survived_node = self.choose_random_node_to_kill(self.servers) - data["nodes"] = [node.id() for node in self.servers.nodes] + data["nodes"] = [node.node_id() for node in self.servers.nodes] disco_infos = [] for node in self.servers.nodes: disco_info = node.discovery_info() disco_infos.append({ - "id": disco_info.id, + "id": disco_info.node_id, "consistent_id": disco_info.consistent_id, "coordinator": disco_info.coordinator, "order": disco_info.order, @@ -127,6 +149,10 @@ def __basic_test__(self, version, with_zk=False): @staticmethod def choose_random_node_to_kill(service): + """ + :param service: Service nodes to process. + :return: Tuple of random node to kill and survived nodes + """ idx = random.randint(0, len(service.nodes) - 1) survive = [node for i, node in enumerate(service.nodes) if i != idx] diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index 0df2f84dca5d6..f0cfd7777da91 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -12,6 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +This module contains PME free switch tests. +""" + import time from ducktape.mark import parametrize @@ -23,11 +28,18 @@ from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion +# pylint: disable=W0223 class PmeFreeSwitchTest(IgniteTest): + """ + Tests PME free switch scenarios. + """ NUM_NODES = 3 @staticmethod def properties(): + """ + :return: Rendered configuration properties. + """ return """ @@ -53,6 +65,9 @@ def teardown(self): @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_7)) def test(self, version): + """ + Test PME free scenario (node stop). + """ data = {} self.stage("Starting nodes") @@ -82,7 +97,8 @@ def test(self, version): single_key_tx_streamer = IgniteApplicationService( self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.SingleKeyTxStreamerApplication", + java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test." + "SingleKeyTxStreamerApplication", properties=self.properties(), params="test-cache,1000", version=ignite_version) diff --git a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py index 7e2b4fc4b854f..a5f188bab82fa 100644 --- a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py +++ b/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module contains spark integration test. +""" + from ducktape.mark import parametrize from ignitetest.services.ignite import IgniteService @@ -23,6 +27,7 @@ from ignitetest.tests.utils.version import DEV_BRANCH +# pylint: disable=W0223 class SparkIntegrationTest(IgniteTest): """ Test performs: @@ -45,6 +50,9 @@ def teardown(self): @parametrize(version=str(DEV_BRANCH)) def test_spark_client(self, version): + """ + Performs test scenario. + """ self.spark = SparkService(self.test_context, version=version, num_nodes=2) self.spark.start() @@ -55,7 +63,8 @@ def test_spark_client(self, version): IgniteApplicationService( self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.tests.spark_integration_test.SampleDataStreamerApplication", + java_class_name="org.apache.ignite.internal.ducktest.tests.spark_integration_test." + "SampleDataStreamerApplication", params="cache,1000", version=version).run() diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py index a9c13d941d20d..df0d30eceda77 100644 --- a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py +++ b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py @@ -13,15 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +This module contains basic ignite test. +""" + from ducktape.tests.test import Test from monotonic import monotonic +# pylint: disable=W0223 class IgniteTest(Test): + """ + Basic ignite test. + """ def __init__(self, test_context): super(IgniteTest, self).__init__(test_context=test_context) def stage(self, msg): + """ + Print stage mark. + :param msg: Stage mark message. + """ self.logger.info("[TEST_STAGE] " + msg + "...") @staticmethod diff --git a/modules/ducktests/tests/ignitetest/tests/utils/version.py b/modules/ducktests/tests/ignitetest/tests/utils/version.py index 2f1f003f7d7ff..51731882d8ec7 100644 --- a/modules/ducktests/tests/ignitetest/tests/utils/version.py +++ b/modules/ducktests/tests/ignitetest/tests/utils/version.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Module contains ignite version utility class. +""" from distutils.version import LooseVersion from ducktape.cluster.cluster import ClusterNode @@ -49,8 +52,8 @@ def __init__(self, version_string): def __str__(self): if self.is_dev: return "dev" - else: - return LooseVersion.__str__(self) + + return LooseVersion.__str__(self) def get_version(node=None): @@ -61,7 +64,8 @@ def get_version(node=None): if isinstance(node, ClusterNode) and hasattr(node, 'version'): return getattr(node, 'version') - if isinstance(node, str) or isinstance(node, unicode): + # pylint: disable=E0602 + if isinstance(node, (str, unicode)): return node return DEV_BRANCH From 495e07cb1b712b81292403c6a77a796de14649ae Mon Sep 17 00:00:00 2001 From: Vladsz83 Date: Thu, 30 Jul 2020 14:26:34 +0300 Subject: [PATCH 31/78] ducktape to v0.7.8 (#8100) --- modules/ducktests/tests/docker/Dockerfile | 2 +- modules/ducktests/tests/setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index fedea542476e6..b54578eca7e73 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -35,7 +35,7 @@ LABEL ducker.creator=$ducker_creator RUN cat /etc/apt/sources.list | sed 's/http:\/\/deb.debian.org/https:\/\/deb.debian.org/g' > /etc/apt/sources.list.2 && mv /etc/apt/sources.list.2 /etc/apt/sources.list RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute mc && apt-get -y clean RUN python -m pip install -U pip==9.0.3; -RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 monotonic && pip install --upgrade ducktape==0.7.7 +RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 monotonic && pip install --upgrade ducktape==0.7.8 # Set up ssh COPY ./ssh-config /root/.ssh/config diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index 2eef5141ef4ee..731527d96be37 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -47,11 +47,11 @@ def run_tests(self): version=version, description="Apache Ignite System Tests", author="Apache Ignite", - platforms=["any"], + platforms=["any"], license="apache2.0", packages=find_packages(), include_package_data=True, - install_requires=["ducktape==0.7.7", "requests==2.20.0"], + install_requires=["ducktape==0.7.8", "requests==2.20.0"], tests_require=["pytest", "mock", "monotonic"], cmdclass={'test': PyTest} ) From 6bcbb896ad2eafc9959515db68197a5e8e211ee2 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Fri, 31 Jul 2020 11:47:38 +0300 Subject: [PATCH 32/78] Fix zk wait for finishing node. (#8106) --- .../tests/ignitetest/services/zk/zookeeper.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py index 5c29c66cbbbd8..f4a0158cc8f9c 100644 --- a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py +++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py @@ -20,6 +20,7 @@ import os.path from ducktape.services.service import Service +from ducktape.utils.util import wait_until class ZookeeperSettings: @@ -88,17 +89,9 @@ def start_node(self, node): node.account.ssh(start_cmd) def wait_node(self, node, timeout_sec=20): - idx = self.idx(node) - - with node.account.monitor_log(self.LOG_FILE) as monitor: - monitor.offset = 0 - monitor.wait_until( - "binding to port", - timeout_sec=timeout_sec, - err_msg="Zookeeper service didn't finish startup on %s" % node.account.hostname - ) + wait_until(lambda: not self.alive(node), timeout_sec=timeout_sec) - self.logger.info("Zookeeper node %d started on %s", idx, node.account.hostname) + return not self.alive(node) def await_quorum(self, node, timeout): """ From ef9dda8cf772dd3e4d3496f0d3626a47d6a69917 Mon Sep 17 00:00:00 2001 From: Maksim Timonin Date: Fri, 31 Jul 2020 15:23:08 +0300 Subject: [PATCH 33/78] Support inclusion of optional modules in services (#8097) --- .../tests/smoke_test/SimpleApplication.java | 57 ++++++++ .../SampleDataStreamerApplication.java | 61 --------- .../SparkApplication.java | 129 ------------------ modules/ducktests/tests/docker/Dockerfile | 2 - .../tests/ignitetest/services/ignite.py | 8 +- .../tests/ignitetest/services/ignite_app.py | 8 +- .../ignitetest/services/ignite_spark_app.py | 36 ----- .../tests/ignitetest/services/spark.py | 8 +- .../ignitetest/services/utils/ignite_aware.py | 26 ++-- .../services/utils/ignite_aware_app.py | 9 +- .../ignitetest/services/utils/ignite_path.py | 43 +++--- .../tests/ignitetest/tests/discovery_test.py | 1 + ...park_integration_test.py => smoke_test.py} | 69 +++++----- .../tests/ignitetest/tests/utils/version.py | 17 +-- modules/ducktests/tests/setup.py | 1 + 15 files changed, 149 insertions(+), 326 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java delete mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SampleDataStreamerApplication.java delete mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SparkApplication.java delete mode 100644 modules/ducktests/tests/ignitetest/services/ignite_spark_app.py rename modules/ducktests/tests/ignitetest/tests/{spark_integration_test.py => smoke_test.py} (51%) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java new file mode 100644 index 0000000000000..ec4057d1fec82 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.smoke_test; + +import java.util.UUID; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Simple application that used in smoke tests + */ +public class SimpleApplication extends IgniteAwareApplication { + /** + * @param ignite Ignite. + */ + public SimpleApplication(Ignite ignite) { + super(ignite); + } + + /** {@inheritDoc} */ + @Override public void run(String[] args) { + IgniteCache cache = ignite.getOrCreateCache(UUID.randomUUID().toString()); + + cache.put(1, 2); + + markInitialized(); + + while (!terminated()) { + try { + U.sleep(100); // Keeping node/txs alive. + } + catch (IgniteInterruptedCheckedException ignored) { + log.info("Waiting interrupted."); + } + } + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SampleDataStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SampleDataStreamerApplication.java deleted file mode 100644 index 691953c21c762..0000000000000 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SampleDataStreamerApplication.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.ducktest.tests.spark_integration_test; - -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.query.SqlFieldsQuery; -import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; - -/** - * - */ -public class SampleDataStreamerApplication extends IgniteAwareApplication { - /** - * @param ignite Ignite. - */ - public SampleDataStreamerApplication(Ignite ignite) { - super(ignite); - } - - /** - * @param cache Cache. - * @param qry Query. - * @param args Args. - */ - private static void executeSql(IgniteCache cache, String qry, Object... args) { - cache.query(new SqlFieldsQuery(qry).setArgs(args)).getAll(); - } - - /** {@inheritDoc} */ - @Override protected void run(String[] args) { - System.out.println("Creating cache..."); - - IgniteCache cache = ignite.createCache(args[0]); - - for (int i = 0; i < Integer.parseInt(args[1]); i++) - cache.put(i, i); - - executeSql(cache, "CREATE TABLE person(id INT, fio VARCHAR, PRIMARY KEY(id))"); - executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 1, "Ivanov Ivan"); - executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 2, "Petrov Petr"); - executeSql(cache, "INSERT INTO person(id, fio) VALUES(?, ?)", 3, "Sidorov Sidr"); - - markSyncExecutionComplete(); - } -} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SparkApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SparkApplication.java deleted file mode 100644 index 565b85e8be773..0000000000000 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/spark_integration_test/SparkApplication.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.ducktest.tests.spark_integration_test; - -import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; -import org.apache.ignite.spark.IgniteDataFrameSettings; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.ignite.IgniteSparkSession; - -/** - * - */ -public class SparkApplication extends IgniteAwareApplication { - /** Home. */ - public static final String HOME = "/opt/ignite-dev"; - - /** Version. */ - public static final String VER = "2.10.0-SNAPSHOT"; - - /** Spring version. */ - public static final String SPRING_VER = "4.3.26.RELEASE"; - - /** - * @param masterUrl Master url. - */ - private static void sparkSession(String cfgPath, String masterUrl) { - //Creating spark session. - try (SparkSession spark = SparkSession.builder() - .appName("SparkApplication") - .master(masterUrl) - .getOrCreate()) { - spark.sparkContext().addJar(HOME + "/modules/core/target/ignite-core-" + VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spring/target/ignite-spring-" + VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/log4j/target/ignite-log4j-" + VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spark/target/ignite-spark-" + VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/indexing/target/ignite-indexing-" + VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-beans-" + SPRING_VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-core-" + SPRING_VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-context-" + SPRING_VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-expression-" + SPRING_VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/core/target/libs/cache-api-1.0.0.jar"); - spark.sparkContext().addJar(HOME + "/modules/indexing/target/libs/h2-1.4.197.jar"); - - sparkDSLExample(cfgPath, spark); - } - } - - /** - * @param masterUrl Master url. - * @param cfgPath Config path. - */ - private static void igniteSession(String cfgPath, String masterUrl) { - //Creating spark session. - try (IgniteSparkSession spark = IgniteSparkSession.builder() - .appName("SparkApplication") - .igniteConfig(cfgPath) - .master(masterUrl) - .getOrCreate()) { - spark.sparkContext().addJar(HOME + "/modules/core/target/ignite-core-" + VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spring/target/ignite-spring-" + VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/log4j/target/ignite-log4j-" + VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spark/target/ignite-spark-" + VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/indexing/target/ignite-indexing-" + VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-beans-" + SPRING_VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-core-" + SPRING_VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-context-" + SPRING_VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/spring/target/libs/spring-expression-" + SPRING_VER + ".jar"); - spark.sparkContext().addJar(HOME + "/modules/core/target/libs/cache-api-1.0.0.jar"); - spark.sparkContext().addJar(HOME + "/modules/indexing/target/libs/h2-1.4.197.jar"); - - spark.catalog().listTables().show(); - - sparkDSLExample(cfgPath, spark); - } - } - - /** - * @param spark Spark. - * @param cfgPath Config path. - */ - private static void sparkDSLExample(String cfgPath, SparkSession spark) { - System.out.println("Querying using Spark DSL."); - - Dataset igniteDF = spark.read() - .format(IgniteDataFrameSettings.FORMAT_IGNITE()) //Data source type. - .option(IgniteDataFrameSettings.OPTION_TABLE(), "person") //Table to read. - .option(IgniteDataFrameSettings.OPTION_CONFIG_FILE(), cfgPath) //Ignite config. - .load(); - - System.out.println("Data frame schema:"); - - igniteDF.printSchema(); //Printing query schema to console. - - System.out.println("Data frame content:"); - - igniteDF.show(); //Printing query results to console. - } - - /** {@inheritDoc} */ - @Override protected void run(String[] args) throws Exception { - System.out.println("SparkApplication.main - args"); - - for (String arg : args) - System.out.println("SparkApplication.main - " + arg); - - sparkSession(args[0], args[1]); - - igniteSession(args[0], args[1]); - - markSyncExecutionComplete(); - } -} diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index b54578eca7e73..6de634b068d39 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -51,7 +51,6 @@ RUN for v in "2.7.6" "2.8.0" "2.8.1"; \ do cd /opt; \ curl -O $APACHE_ARCHIVE/ignite/$v/apache-ignite-$v-bin.zip;\ unzip apache-ignite-$v-bin.zip && mv /opt/apache-ignite-$v-bin /opt/ignite-$v;\ - cp -r ./ignite-$v/libs/optional/ignite-zookeeper ./ignite-$v/libs; \ done RUN rm /opt/apache-ignite-*-bin.zip @@ -73,7 +72,6 @@ ARG SPARK_RELEASE_NAME="spark-$SPARK_VERSION-bin-hadoop2.7" RUN cd /opt && curl -O $APACHE_ARCHIVE/spark/$SPARK_NAME/$SPARK_RELEASE_NAME.tgz && tar xvf $SPARK_RELEASE_NAME.tgz && rm $SPARK_RELEASE_NAME.tgz RUN mv /opt/$SPARK_RELEASE_NAME /opt/$SPARK_NAME -RUN chmod a+wr /opt/$SPARK_NAME -R # The version of Kibosh to use for testing. # If you update this, also update vagrant/base.sh diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index bfc96ade7db85..f5054b6a0ea38 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -45,8 +45,8 @@ class IgniteService(IgniteAwareService): } # pylint: disable=R0913 - def __init__(self, context, num_nodes, client_mode=False, version=DEV_BRANCH, properties=""): - super(IgniteService, self).__init__(context, num_nodes, client_mode, version, properties) + def __init__(self, context, num_nodes, modules=None, client_mode=False, version=DEV_BRANCH, properties=""): + super(IgniteService, self).__init__(context, num_nodes, modules, client_mode, version, properties) # pylint: disable=W0221 def start(self, timeout_sec=180): @@ -64,9 +64,9 @@ def start_cmd(self, node): cmd = "export EXCLUDE_TEST_CLASSES=true; " cmd += "export IGNITE_LOG_DIR=" + IgniteService.PERSISTENT_ROOT + "; " - cmd += "export USER_LIBS=%s/libs/optional/ignite-log4j/*; " % self.path.home(self.version) + cmd += "export USER_LIBS=%s; " % self.user_libs cmd += "%s %s %s 1>> %s 2>> %s &" % \ - (self.path.script("ignite.sh", node), + (self.path.script("ignite.sh"), jvm_opts, IgniteService.CONFIG_FILE, IgniteService.STDOUT_STDERR_CAPTURE, diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 07022210fe814..ec9007d2d3cb0 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -28,7 +28,7 @@ class IgniteApplicationService(IgniteAwareApplicationService): service_java_class_name = "org.apache.ignite.internal.ducktest.utils.IgniteApplicationService" # pylint: disable=R0913 - def __init__(self, context, java_class_name, client_mode=True, version=DEV_BRANCH, properties="", params="", - timeout_sec=60): - super(IgniteApplicationService, self).__init__(context, java_class_name, client_mode, version, properties, - params, timeout_sec, self.service_java_class_name) + def __init__(self, context, java_class_name, modules=None, client_mode=True, version=DEV_BRANCH, + properties="", params="", timeout_sec=60): + super(IgniteApplicationService, self).__init__(context, java_class_name, modules, client_mode, version, + properties, params, timeout_sec, self.service_java_class_name) diff --git a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py b/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py deleted file mode 100644 index 71014b7bc15ea..0000000000000 --- a/modules/ducktests/tests/ignitetest/services/ignite_spark_app.py +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This module contains the Ignite-Spark application service. -""" -from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService -from ignitetest.tests.utils.version import DEV_BRANCH - - -class SparkIgniteApplicationService(IgniteAwareApplicationService): - """ - The Ignite-Spark application service. - """ - # pylint: disable=R0913 - def __init__(self, context, java_class_name, client_mode=True, version=DEV_BRANCH, properties="", params="", - timeout_sec=60): - super(SparkIgniteApplicationService, self).__init__(context, java_class_name, client_mode, version, properties, - params, timeout_sec) - - def env(self): - return IgniteAwareApplicationService.env(self) + \ - "export EXCLUDE_MODULES=\"kubernetes,aws,gce,mesos,rest-http,web-agent,zookeeper,serializers,store," \ - "rocketmq\"; " diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py index 657a6b6a4b141..4b4b9cd01048d 100644 --- a/modules/ducktests/tests/ignitetest/services/spark.py +++ b/modules/ducktests/tests/ignitetest/services/spark.py @@ -35,12 +35,16 @@ class SparkService(IgniteAwareService): logs = {} - def __init__(self, context, version=DEV_BRANCH, num_nodes=3, properties=""): + # pylint: disable=R0913 + def __init__(self, context, modules=None, version=DEV_BRANCH, num_nodes=3, properties=""): """ :param context: test context :param num_nodes: number of Ignite nodes. """ - IgniteAwareService.__init__(self, context, num_nodes, False, version, properties) + modules = modules or [] + modules.extend(["ignite-spark"]) + + IgniteAwareService.__init__(self, context, num_nodes, modules, False, version, properties) self.log_level = "DEBUG" diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index b4d12f0964a22..3668ffe23e5f0 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -27,6 +27,8 @@ from ignitetest.services.utils.ignite_path import IgnitePath from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin +from ignitetest.tests.utils.version import IgniteVersion + class IgniteAwareService(BackgroundThreadService): """ @@ -46,20 +48,26 @@ class IgniteAwareService(BackgroundThreadService): } # pylint: disable=R0913 - def __init__(self, context, num_nodes, client_mode, version, properties): + def __init__(self, context, num_nodes, modules, client_mode, version, properties): super(IgniteAwareService, self).__init__(context, num_nodes) - self.path = IgnitePath(context) self.jvm_options = context.globals.get("jvm_opts", "") self.log_level = "DEBUG" self.properties = properties - self.version = version - self.logger_config = IgniteLoggerConfig() + + if isinstance(version, IgniteVersion): + self.version = version + else: + self.version = IgniteVersion(version) + + self.path = IgnitePath(self.version, context) self.client_mode = client_mode - for node in self.nodes: - node.version = version + libs = modules or [] + libs.extend(["ignite-log4j"]) + libs = map(lambda m: self.path.module(m) + "/*", libs) + self.user_libs = ":".join(libs) def start_node(self, node): self.init_persistent(node) @@ -75,10 +83,12 @@ def init_persistent(self, node): Init persistent directory. :param node: Ignite service node. """ + logger_config = IgniteLoggerConfig().render(work_dir=self.WORK_DIR) + node.account.mkdirs(self.PERSISTENT_ROOT) node.account.create_file(self.CONFIG_FILE, self.config().render( config_dir=self.PERSISTENT_ROOT, work_dir=self.WORK_DIR, properties=self.properties)) - node.account.create_file(self.LOG4J_CONFIG_FILE, self.logger_config.render(work_dir=self.WORK_DIR)) + node.account.create_file(self.LOG4J_CONFIG_FILE, logger_config) @abstractmethod def start_cmd(self, node): @@ -159,7 +169,7 @@ def execute(self, command): """ for node in self.nodes: cmd = "%s 1>> %s 2>> %s" % \ - (self.path.script(command, node), + (self.path.script(command), self.STDOUT_STDERR_CAPTURE, self.STDOUT_STDERR_CAPTURE) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index 152b5fe2038a5..d2fedaa1f9b40 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -27,9 +27,9 @@ class IgniteAwareApplicationService(IgniteAwareService): The base class to build Ignite aware application written on java. """ # pylint: disable=R0913 - def __init__(self, context, java_class_name, client_mode, version, properties, params, timeout_sec, + def __init__(self, context, java_class_name, modules, client_mode, version, properties, params, timeout_sec, service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): - super(IgniteAwareApplicationService, self).__init__(context, 1, client_mode, version, properties) + super(IgniteAwareApplicationService, self).__init__(context, 1, modules, client_mode, version, properties) self.servicejava_class_name = service_java_class_name self.java_class_name = java_class_name @@ -48,7 +48,7 @@ def start(self): def start_cmd(self, node): cmd = self.env() cmd += "%s %s %s 1>> %s 2>> %s &" % \ - (self.path.script("ignite.sh", node), + (self.path.script("ignite.sh"), self.jvm_opts(), self.app_args(), self.STDOUT_STDERR_CAPTURE, @@ -106,8 +106,7 @@ def env(self): return "export MAIN_CLASS={main_class}; ".format(main_class=self.servicejava_class_name) + \ "export EXCLUDE_TEST_CLASSES=true; " + \ "export IGNITE_LOG_DIR={log_dir}; ".format(log_dir=self.PERSISTENT_ROOT) + \ - "export USER_LIBS=%s/libs/optional/ignite-log4j/*:/opt/ignite-dev/modules/ducktests/target/*; " \ - % self.path.home(self.version) + "export USER_LIBS=%s:/opt/ignite-dev/modules/ducktests/target/*; " % self.user_libs def extract_result(self, name): """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py index 4d24d36b72658..c1217b094e116 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py @@ -19,8 +19,6 @@ import os -from ignitetest.tests.utils.version import get_version, IgniteVersion - class IgnitePath: """Path resolver for Ignite system tests which assumes the following layout: @@ -33,35 +31,28 @@ class IgnitePath: SCRATCH_ROOT = "/mnt" IGNITE_INSTALL_ROOT = "/opt" - def __init__(self, context): + def __init__(self, version, context): + self.version = version self.project = context.globals.get("project", "ignite") - def home(self, node_or_version, project=None): + home_dir = "%s-%s" % (self.project, str(self.version)) + self._home = os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir) + + def module(self, module_name): """ - :param node_or_version: Ignite service node or IgniteVersion instance. - :param project: Project name. - :return: Home directory. + :param module_name: name of Ignite optional lib + :return: absolute path to the specified module """ - version = self.__version(node_or_version) - home_dir = project or self.project - if version is not None: - home_dir += "-%s" % str(version) + if self.version.is_dev: + module_path = os.path.join("modules", module_name, "target") + else: + module_path = os.path.join("libs", "optional", module_name) - return os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir) + return os.path.join(self._home, module_path) - def script(self, script_name, node_or_version, project=None): + def script(self, script_name): """ - :param script_name: Script name. - :param node_or_version: Ignite service node or IgniteVersion instance. - :param project: Project name. - :return: Full path to script. + :param script_name: name of Ignite script + :return: absolute path to the specified script """ - version = self.__version(node_or_version) - return os.path.join(self.home(version, project=project), "bin", script_name) - - @staticmethod - def __version(node_or_version): - if isinstance(node_or_version, IgniteVersion): - return node_or_version - - return get_version(node_or_version) + return os.path.join(self._home, "bin", script_name) diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index d8d4e0b0ff489..01f6bcbe76290 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -109,6 +109,7 @@ def __basic_test__(self, version, with_zk=False): self.servers = IgniteService( self.test_context, num_nodes=self.NUM_NODES, + modules=["ignite-zookeeper"], properties=properties, version=version) diff --git a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py b/modules/ducktests/tests/ignitetest/tests/smoke_test.py similarity index 51% rename from modules/ducktests/tests/ignitetest/tests/spark_integration_test.py rename to modules/ducktests/tests/ignitetest/tests/smoke_test.py index a5f188bab82fa..a52f38246791f 100644 --- a/modules/ducktests/tests/ignitetest/tests/spark_integration_test.py +++ b/modules/ducktests/tests/ignitetest/tests/smoke_test.py @@ -14,65 +14,66 @@ # limitations under the License. """ -This module contains spark integration test. +This module contains smoke tests that checks that services work """ from ducktape.mark import parametrize from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService -from ignitetest.services.ignite_spark_app import SparkIgniteApplicationService from ignitetest.services.spark import SparkService +from ignitetest.services.zk.zookeeper import ZookeeperService from ignitetest.tests.utils.ignite_test import IgniteTest from ignitetest.tests.utils.version import DEV_BRANCH # pylint: disable=W0223 -class SparkIntegrationTest(IgniteTest): +class SmokeServicesTest(IgniteTest): """ - Test performs: - 1. Start of Spark cluster. - 2. Start of Spark client application. - 3. Checks results of client application. + Tests services implementations """ - def __init__(self, test_context): - super(SparkIntegrationTest, self).__init__(test_context=test_context) - self.spark = None - self.ignite = None + super(SmokeServicesTest, self).__init__(test_context=test_context) def setUp(self): pass def teardown(self): - self.spark.stop() - self.ignite.stop() + pass @parametrize(version=str(DEV_BRANCH)) - def test_spark_client(self, version): + def test_ignite_app_start_stop(self, version): """ - Performs test scenario. + Test that IgniteService and IgniteApplicationService correctly start and stop """ - self.spark = SparkService(self.test_context, version=version, num_nodes=2) - self.spark.start() - - self.ignite = IgniteService(self.test_context, version=version, num_nodes=1) - self.ignite.start() - - self.stage("Starting sample data generator") + ignite = IgniteService( + self.test_context, + num_nodes=1, + version=version) - IgniteApplicationService( + app = IgniteApplicationService( self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.tests.spark_integration_test." - "SampleDataStreamerApplication", - params="cache,1000", - version=version).run() + java_class_name="org.apache.ignite.internal.ducktest.tests.smoke_test.SimpleApplication", + version=version) - self.stage("Starting Spark application") + ignite.start() + app.start() + app.stop() + ignite.stop() - SparkIgniteApplicationService( - self.test_context, - "org.apache.ignite.internal.ducktest.tests.spark_integration_test.SparkApplication", - params="spark://" + self.spark.nodes[0].account.hostname + ":7077", - version=version, - timeout_sec=120).run() + @parametrize(version=str(DEV_BRANCH)) + def test_spark_start_stop(self, version): + """ + Test that SparkService correctly start and stop + """ + spark = SparkService(self.test_context, version=version, num_nodes=2) + spark.start() + spark.stop() + + def test_zk_start_stop(self): + """ + Test that ZookeeperService correctly start and stop + """ + zookeeper = ZookeeperService(self.test_context, num_nodes=2) + zookeeper.start() + zookeeper.stop() diff --git a/modules/ducktests/tests/ignitetest/tests/utils/version.py b/modules/ducktests/tests/ignitetest/tests/utils/version.py index 51731882d8ec7..5131b47404b4a 100644 --- a/modules/ducktests/tests/ignitetest/tests/utils/version.py +++ b/modules/ducktests/tests/ignitetest/tests/utils/version.py @@ -18,7 +18,6 @@ """ from distutils.version import LooseVersion -from ducktape.cluster.cluster import ClusterNode from ignitetest import __version__ @@ -55,20 +54,8 @@ def __str__(self): return LooseVersion.__str__(self) - -def get_version(node=None): - """ - Return the version attached to the given node. - Default to DEV_BRANCH if node or node.version is undefined (aka None) - """ - if isinstance(node, ClusterNode) and hasattr(node, 'version'): - return getattr(node, 'version') - - # pylint: disable=E0602 - if isinstance(node, (str, unicode)): - return node - - return DEV_BRANCH + def __repr__(self): + return "IgniteVersion ('%s')" % str(self) DEV_BRANCH = IgniteVersion("dev") diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index 731527d96be37..94cb914c6b56f 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -42,6 +42,7 @@ def run_tests(self): errno = pytest.main(self.pytest_args) sys.exit(errno) + # Note: when changing the version of ducktape, also revise tests/docker/Dockerfile setup(name="ignitetest", version=version, From 254bb679629a14a64538233792ffa7dd353f6bb9 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Wed, 5 Aug 2020 12:30:20 +0300 Subject: [PATCH 34/78] Json params support (#8109) --- modules/ducktests/pom.xml | 18 ++++++++++++++++++ .../tests/DataGenerationApplication.java | 7 ++++--- .../LongTxStreamerApplication.java | 5 +++-- .../SingleKeyTxStreamerApplication.java | 7 ++++--- .../tests/smoke_test/SimpleApplication.java | 3 ++- .../utils/IgniteApplicationService.java | 11 ++++++++--- .../ducktest/utils/IgniteAwareApplication.java | 14 +++++++------- .../utils/IgniteAwareApplicationService.java | 14 +++++++++++--- .../tests/ignitetest/services/ignite.py | 2 +- .../ignitetest/services/utils/ignite_aware.py | 5 ++--- .../services/utils/ignite_aware_app.py | 13 +++++++++++-- .../ignitetest/services/utils/ignite_path.py | 6 +++--- .../tests/add_node_rebalance_test.py | 2 +- .../ignitetest/tests/pme_free_switch_test.py | 4 ++-- modules/ducktests/tests/setup.py | 1 + 15 files changed, 78 insertions(+), 34 deletions(-) diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml index 7bfdc2c99c7e0..26226c62e788f 100644 --- a/modules/ducktests/pom.xml +++ b/modules/ducktests/pom.xml @@ -131,6 +131,24 @@ 5.0.3 + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + org.codehaus.woodstox stax2-api diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java index c1661d2f4b0b2..f9732b2844aad 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.ducktest.tests; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; @@ -34,13 +35,13 @@ public DataGenerationApplication(Ignite ignite) { } /** {@inheritDoc} */ - @Override protected void run(String[] args) { + @Override protected void run(JsonNode jsonNode) { log.info("Creating cache..."); - IgniteCache cache = ignite.createCache(args[0]); + IgniteCache cache = ignite.createCache(jsonNode.get("cacheName").asText()); try (IgniteDataStreamer stmr = ignite.dataStreamer(cache.getName())) { - for (int i = 0; i < Integer.parseInt(args[1]); i++) { + for (int i = 0; i < jsonNode.get("range").asInt(); i++) { stmr.addData(i, i); if (i % 10_000 == 0) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java index 14b96a8dad669..82596f8d8e39d 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java @@ -19,6 +19,7 @@ import java.util.Collection; import java.util.concurrent.CountDownLatch; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.IgniteEx; @@ -47,8 +48,8 @@ public LongTxStreamerApplication(Ignite ignite) { } /** {@inheritDoc} */ - @Override public void run(String[] args) throws InterruptedException { - IgniteCache cache = ignite.getOrCreateCache(args[0]); + @Override public void run(JsonNode jsonNode) throws InterruptedException { + IgniteCache cache = ignite.getOrCreateCache(jsonNode.get("cacheName").asText()); log.info("Starting Long Tx..."); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java index 42e4d24a4afd2..f38e8111bcb6c 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.ducktest.tests.pme_free_switch_test; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; @@ -33,10 +34,10 @@ public SingleKeyTxStreamerApplication(Ignite ignite) { } /** {@inheritDoc} */ - @Override public void run(String[] args) { - IgniteCache cache = ignite.getOrCreateCache(args[0]); + @Override public void run(JsonNode jsonNode) { + IgniteCache cache = ignite.getOrCreateCache(jsonNode.get("cacheName").asText()); - int warmup = Integer.parseInt(args[1]); + int warmup = jsonNode.get("warmup").asInt(); long max = -1; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java index ec4057d1fec82..379f5b11b4842 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.ducktest.tests.smoke_test; import java.util.UUID; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.IgniteInterruptedCheckedException; @@ -36,7 +37,7 @@ public SimpleApplication(Ignite ignite) { } /** {@inheritDoc} */ - @Override public void run(String[] args) { + @Override public void run(JsonNode jsonNode) { IgniteCache cache = ignite.getOrCreateCache(UUID.randomUUID().toString()); cache.put(1, 2); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java index a41f27c9f2a6c..b4f60b50226c2 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java @@ -17,7 +17,9 @@ package org.apache.ignite.internal.ducktest.utils; -import java.util.Arrays; +import java.util.Base64; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ignite.Ignite; import org.apache.ignite.Ignition; import org.apache.ignite.configuration.IgniteConfiguration; @@ -55,9 +57,12 @@ public static void main(String[] args) throws Exception { try (Ignite ignite = Ignition.start(cfg)) { IgniteAwareApplication app = (IgniteAwareApplication)clazz.getConstructor(Ignite.class).newInstance(ignite); - String[] appParams = Arrays.copyOfRange(params, 2, params.length); + ObjectMapper mapper = new ObjectMapper(); - app.start(appParams); + JsonNode jsonNode = params.length > 2 ? + mapper.readTree(Base64.getDecoder().decode(params[2])) : mapper.createObjectNode(); + + app.start(jsonNode); } } } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java index 525c76d717d39..28c8d20d6ea4d 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java @@ -17,7 +17,7 @@ package org.apache.ignite.internal.ducktest.utils; -import java.util.Arrays; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.Ignite; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.util.typedef.internal.U; @@ -159,18 +159,18 @@ protected void recordResult(String name, long val) { } /** - * + * @param jsonNode JSON node. */ - protected abstract void run(String[] args) throws Exception; + protected abstract void run(JsonNode jsonNode) throws Exception; /** - * @param args Args. + * @param jsonNode JSON node. */ - public void start(String[] args) { + public void start(JsonNode jsonNode) { try { - log.info("Application params: " + Arrays.toString(args)); + log.info("Application params: " + jsonNode); - run(args); + run(jsonNode); assert inited : "Was not properly initialized."; assert finished : "Was not properly finished."; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java index a7836c2945f4e..de13ef1f46750 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java @@ -17,7 +17,10 @@ package org.apache.ignite.internal.ducktest.utils; -import java.util.Arrays; +import java.util.Base64; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -40,8 +43,13 @@ public static void main(String[] args) throws Exception { IgniteAwareApplication app = (IgniteAwareApplication)clazz.getConstructor().newInstance(); - String[] appParams = Arrays.copyOfRange(params, 1, params.length); + ObjectMapper mapper = new ObjectMapper(); - app.start(appParams); + JsonNode jsonNode = params.length > 2 ? + mapper.readTree(Base64.getDecoder().decode(params[2])) : mapper.createObjectNode(); + + ((ObjectNode)jsonNode).put("cfgPath", params[1]); + + app.start(jsonNode); } } diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index f5054b6a0ea38..08732c6a87830 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -64,7 +64,7 @@ def start_cmd(self, node): cmd = "export EXCLUDE_TEST_CLASSES=true; " cmd += "export IGNITE_LOG_DIR=" + IgniteService.PERSISTENT_ROOT + "; " - cmd += "export USER_LIBS=%s; " % self.user_libs + cmd += "export USER_LIBS=%s; " % ":".join(self.user_libs) cmd += "%s %s %s 1>> %s 2>> %s &" % \ (self.path.script("ignite.sh"), jvm_opts, diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 3668ffe23e5f0..7fe97a9a3532d 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -26,7 +26,6 @@ from ignitetest.services.utils.ignite_config import IgniteLoggerConfig, IgniteServerConfig, IgniteClientConfig from ignitetest.services.utils.ignite_path import IgnitePath from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin - from ignitetest.tests.utils.version import IgniteVersion @@ -66,8 +65,8 @@ def __init__(self, context, num_nodes, modules, client_mode, version, properties libs = modules or [] libs.extend(["ignite-log4j"]) - libs = map(lambda m: self.path.module(m) + "/*", libs) - self.user_libs = ":".join(libs) + + self.user_libs = list(map(lambda m: self.path.module(m) + "/*", libs)) def start_node(self, node): self.init_persistent(node) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index d2fedaa1f9b40..eece2a6af65af 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -17,6 +17,8 @@ This module contains the base class to build Ignite aware application written on java. """ +import base64 +import json import re from ignitetest.services.utils.ignite_aware import IgniteAwareService @@ -26,6 +28,7 @@ class IgniteAwareApplicationService(IgniteAwareService): """ The base class to build Ignite aware application written on java. """ + # pylint: disable=R0913 def __init__(self, context, java_class_name, modules, client_mode, version, properties, params, timeout_sec, service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): @@ -82,7 +85,7 @@ def app_args(self): args = self.java_class_name + "," + IgniteAwareApplicationService.CONFIG_FILE if self.params != "": - args += "," + self.params + args += "," + str(base64.b64encode(json.dumps(self.params).encode("UTF-8"))) return args @@ -103,10 +106,16 @@ def env(self): """ :return: Export string of additional environment variables. """ + if not self.version.is_dev: + # Jackson requred to parse application params at java side. Release's version should be used. + for line in self.nodes[0].account.ssh_capture( + "ls -d %s/libs/optional/ignite-aws/* | grep jackson | tr '\n' ':' | sed 's/.$//'" % self.path.home): + self.user_libs.extend([line]) + return "export MAIN_CLASS={main_class}; ".format(main_class=self.servicejava_class_name) + \ "export EXCLUDE_TEST_CLASSES=true; " + \ "export IGNITE_LOG_DIR={log_dir}; ".format(log_dir=self.PERSISTENT_ROOT) + \ - "export USER_LIBS=%s:/opt/ignite-dev/modules/ducktests/target/*; " % self.user_libs + "export USER_LIBS=%s:/opt/ignite-dev/modules/ducktests/target/*; " % (":".join(self.user_libs)) def extract_result(self, name): """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py index c1217b094e116..2177e9bd785de 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py @@ -36,7 +36,7 @@ def __init__(self, version, context): self.project = context.globals.get("project", "ignite") home_dir = "%s-%s" % (self.project, str(self.version)) - self._home = os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir) + self.home = os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir) def module(self, module_name): """ @@ -48,11 +48,11 @@ def module(self, module_name): else: module_path = os.path.join("libs", "optional", module_name) - return os.path.join(self._home, module_path) + return os.path.join(self.home, module_path) def script(self, script_name): """ :param script_name: name of Ignite script :return: absolute path to the specified script """ - return os.path.join(self._home, "bin", script_name) + return os.path.join(self.home, "bin", script_name) diff --git a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py index d8366819e5afd..28fc4fb626280 100644 --- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py @@ -69,7 +69,7 @@ def test_add_node(self, version): IgniteApplicationService(self.test_context, java_class_name="org.apache.ignite.internal.ducktest.tests.DataGenerationApplication", version=ignite_version, - params="test-cache,%d" % self.DATA_AMOUNT, + params={"cacheName": "test-cache", "range": self.DATA_AMOUNT}, timeout_sec=self.PRELOAD_TIMEOUT).run() ignite = IgniteService(self.test_context, num_nodes=1, version=ignite_version) diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index f0cfd7777da91..cd9feecf27ec0 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -88,7 +88,7 @@ def test(self, version): self.test_context, java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.LongTxStreamerApplication", properties=self.properties(), - params="test-cache", + params={"cacheName": "test-cache"}, version=ignite_version) long_tx_streamer.start() @@ -100,7 +100,7 @@ def test(self, version): java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test." "SingleKeyTxStreamerApplication", properties=self.properties(), - params="test-cache,1000", + params={"cacheName": "test-cache", "warmup": 1000}, version=ignite_version) single_key_tx_streamer.start() diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index 94cb914c6b56f..e82bdfb6b8f52 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -15,6 +15,7 @@ import re import sys + from setuptools import find_packages, setup from setuptools.command.test import test as TestCommand From 36ed2c381811cc05d568cbfe49baf33087568539 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Mon, 10 Aug 2020 16:38:56 +0300 Subject: [PATCH 35/78] Ignite ducktape control sh (#8127) --- .gitignore | 1 + .travis.yml | 8 + bin/control.sh | 13 +- bin/include/build-classpath.sh | 7 +- modules/ducktests/tests/check_style.sh | 26 -- .../services/utils/config/ignite.xml.j2 | 4 +- .../services/utils/control_utility.py | 167 +++++++++++++ .../ignitetest/services/utils/ignite_aware.py | 21 +- .../ignitetest/tests/control_utility_test.py | 235 ++++++++++++++++++ .../ignitetest/tests/pme_free_switch_test.py | 4 +- .../ducktests/tests/{.pylintrc => tox.ini} | 23 ++ 11 files changed, 461 insertions(+), 48 deletions(-) delete mode 100755 modules/ducktests/tests/check_style.sh create mode 100644 modules/ducktests/tests/ignitetest/services/utils/control_utility.py create mode 100644 modules/ducktests/tests/ignitetest/tests/control_utility_test.py rename modules/ducktests/tests/{.pylintrc => tox.ini} (73%) diff --git a/.gitignore b/.gitignore index 1a8b46a217596..151ad0151bab5 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,4 @@ packages *.pyc /tests/venv modules/ducktests/tests/docker/build/** +modules/ducktests/tests/.tox diff --git a/.travis.yml b/.travis.yml index 73117e8f68d49..afb930fadb2f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,3 +50,11 @@ matrix: dotnet: 3.1.101 script: - dotnet build modules/platforms/dotnet/Apache.Ignite.DotNetCore.sln + + - language: python + python: + - "3.8" + install: pip install tox-travis + before_script: cd modules/ducktests/tests + script: + - tox diff --git a/bin/control.sh b/bin/control.sh index 5d615d48be55d..4493c6c6ea6f7 100755 --- a/bin/control.sh +++ b/bin/control.sh @@ -1,12 +1,9 @@ #!/usr/bin/env bash -if [ ! -z "${IGNITE_SCRIPT_STRICT_MODE:-}" ] -then - set -o nounset - set -o errexit - set -o pipefail - set -o errtrace - set -o functrace -fi +set -o nounset +set -o errexit +set -o pipefail +set -o errtrace +set -o functrace # # Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/bin/include/build-classpath.sh b/bin/include/build-classpath.sh index 9a43683c36f97..0625f41448624 100644 --- a/bin/include/build-classpath.sh +++ b/bin/include/build-classpath.sh @@ -47,14 +47,13 @@ includeToClassPath() { for file in $1/* do - if [[ -z "${EXCLUDE_MODULES}" ]] || [[ ${EXCLUDE_MODULES} != *"`basename $file`"* ]]; then - echo "$file included" + if [[ -z "${EXCLUDE_MODULES:-}" ]] || [[ ${EXCLUDE_MODULES:-} != *"`basename $file`"* ]]; then if [ -d ${file} ] && [ -d "${file}/target" ]; then if [ -d "${file}/target/classes" ]; then IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/classes fi - if [[ -z "${EXCLUDE_TEST_CLASSES}" ]]; then + if [[ -z "${EXCLUDE_TEST_CLASSES:-}" ]]; then if [ -d "${file}/target/test-classes" ]; then IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/target/test-classes fi @@ -68,7 +67,7 @@ includeToClassPath() { echo "$file excluded by EXCLUDE_MODULES settings" fi done - + IFS=$SAVEIFS } diff --git a/modules/ducktests/tests/check_style.sh b/modules/ducktests/tests/check_style.sh deleted file mode 100755 index 39405a7990037..0000000000000 --- a/modules/ducktests/tests/check_style.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if ! command -v pylint &> /dev/null -then - echo "Please, install pylint first" - exit 1 -fi - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -pylint --rcfile="$SCRIPT_DIR"/.pylintrc "$SCRIPT_DIR"/ignitetest diff --git a/modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 index 5ceb5b6c170a5..50d2c5cdc920c 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 @@ -30,7 +30,9 @@ - + {% if consistent_id %} + + {% endif %} {{ properties }} diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py new file mode 100644 index 0000000000000..36ff5bb796bca --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -0,0 +1,167 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains control utility wrapper. +""" +import random +import re +from collections import namedtuple + +from ducktape.cluster.remoteaccount import RemoteCommandError + + +class ControlUtility: + """ + Control utility (control.sh) wrapper. + """ + BASE_COMMAND = "control.sh" + + def __init__(self, cluster, text_context): + self._cluster = cluster + self.logger = text_context.logger + + def baseline(self): + """ + :return Baseline nodes. + """ + return self.cluster_state().baseline + + def cluster_state(self): + """ + :return: Cluster state. + """ + output = self.__run("--baseline") + + return self.__parse_cluster_state(output) + + def set_baseline(self, baseline): + """ + :param baseline: Baseline nodes or topology version to set as baseline. + """ + if isinstance(baseline, int): + result = self.__run("--baseline version %d --yes" % baseline) + else: + result = self.__run("--baseline set %s --yes" % + ",".join([node.account.externally_routable_ip for node in baseline])) + + return self.__parse_cluster_state(result) + + def add_to_baseline(self, nodes): + """ + :param nodes: Nodes that should be added to baseline. + """ + result = self.__run("--baseline add %s --yes" % + ",".join([node.account.externally_routable_ip for node in nodes])) + + return self.__parse_cluster_state(result) + + def remove_from_baseline(self, nodes): + """ + :param nodes: Nodes that should be removed to baseline. + """ + result = self.__run("--baseline remove %s --yes" % + ",".join([node.account.externally_routable_ip for node in nodes])) + + return self.__parse_cluster_state(result) + + def disable_baseline_auto_adjust(self): + """ + Disable baseline auto adjust. + """ + return self.__run("--baseline auto_adjust disable --yes") + + def enable_baseline_auto_adjust(self, timeout=None): + """ + Enable baseline auto adjust. + :param timeout: Auto adjust timeout in millis. + """ + timeout_str = "timeout %d" % timeout if timeout else "" + return self.__run("--baseline auto_adjust enable %s --yes" % timeout_str) + + def activate(self): + """ + Activate cluster. + """ + return self.__run("--activate --yes") + + def deactivate(self): + """ + Deactivate cluster. + """ + return self.__run("--deactivate --yes") + + @staticmethod + def __parse_cluster_state(output): + state_pattern = re.compile("Cluster state: ([^\\s]+)") + topology_pattern = re.compile("Current topology version: (\\d+)") + baseline_pattern = re.compile("Consistent(Id|ID)=([^\\s]+),\\sS(tate|TATE)=([^\\s]+),?(\\sOrder=(\\d+))?") + + match = state_pattern.search(output) + state = match.group(1) if match else None + + match = topology_pattern.search(output) + topology = int(match.group(1)) if match else None + + baseline = [BaselineNode(consistent_id=m[1], state=m[3], order=int(m[5]) if m[5] else None) + for m in baseline_pattern.findall(output)] + + return ClusterState(state=state, topology_version=topology, baseline=baseline) + + def __run(self, cmd): + node = random.choice(self.__alives()) + + self.logger.debug("Run command %s on node %s", cmd, node.name) + + raw_output = node.account.ssh_capture(self.__form_cmd(node, cmd), allow_fail=True) + code, output = self.__parse_output(raw_output) + + self.logger.debug("Output of command %s on node %s, exited with code %d, is %s", cmd, node.name, code, output) + + if code != 0: + raise ControlUtilityError(node.account, cmd, code, output) + + return output + + def __form_cmd(self, node, cmd): + return self._cluster.path.script("%s --host %s %s" % (self.BASE_COMMAND, node.account.externally_routable_ip, + cmd)) + + @staticmethod + def __parse_output(raw_output): + exit_code = raw_output.channel_file.channel.recv_exit_status() + output = "".join(raw_output) + + pattern = re.compile("Command \\[[^\\s]*\\] finished with code: (\\d+)") + match = pattern.search(output) + + if match: + return int(match.group(1)), output + return exit_code, output + + def __alives(self): + return [node for node in self._cluster.nodes if self._cluster.alive(node)] + + +BaselineNode = namedtuple("BaselineNode", ["consistent_id", "state", "order"]) +ClusterState = namedtuple("ClusterState", ["state", "topology_version", "baseline"]) + + +class ControlUtilityError(RemoteCommandError): + """ + Error is raised when control utility failed. + """ + def __init__(self, account, cmd, exit_status, output): + super(ControlUtilityError, self).__init__(account, cmd, exit_status, "".join(output)) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 7fe97a9a3532d..2e292a3e37b7b 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -18,10 +18,11 @@ """ import os -from abc import abstractmethod +from abc import abstractmethod, ABCMeta from ducktape.services.background_thread import BackgroundThreadService from ducktape.utils.util import wait_until +from six import add_metaclass from ignitetest.services.utils.ignite_config import IgniteLoggerConfig, IgniteServerConfig, IgniteClientConfig from ignitetest.services.utils.ignite_path import IgnitePath @@ -29,6 +30,7 @@ from ignitetest.tests.utils.version import IgniteVersion +@add_metaclass(ABCMeta) class IgniteAwareService(BackgroundThreadService): """ The base class to build services aware of Ignite. @@ -85,8 +87,14 @@ def init_persistent(self, node): logger_config = IgniteLoggerConfig().render(work_dir=self.WORK_DIR) node.account.mkdirs(self.PERSISTENT_ROOT) - node.account.create_file(self.CONFIG_FILE, self.config().render( - config_dir=self.PERSISTENT_ROOT, work_dir=self.WORK_DIR, properties=self.properties)) + + node_config = self.config().render(config_dir=self.PERSISTENT_ROOT, + work_dir=self.WORK_DIR, + properties=self.properties, + consistent_id=node.account.externally_routable_ip) + + setattr(node, "consistent_id", node.account.externally_routable_ip) + node.account.create_file(self.CONFIG_FILE, node_config) node.account.create_file(self.LOG4J_CONFIG_FILE, logger_config) @abstractmethod @@ -156,10 +164,9 @@ def await_event(self, evt_message, timeout_sec, from_the_beginning=False, backof :param backoff_sec: Number of seconds to back off between each failure to meet the condition before checking again. """ - assert len(self.nodes) == 1 - - self.await_event_on_node(evt_message, self.nodes[0], timeout_sec, from_the_beginning=from_the_beginning, - backoff_sec=backoff_sec) + for node in self.nodes: + self.await_event_on_node(evt_message, node, timeout_sec, from_the_beginning=from_the_beginning, + backoff_sec=backoff_sec) def execute(self, command): """ diff --git a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py new file mode 100644 index 0000000000000..027cc3c4436b5 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py @@ -0,0 +1,235 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains control.sh utility tests. +""" +from ducktape.mark import parametrize +from ducktape.mark.resource import cluster +from ducktape.utils.util import wait_until +from jinja2 import Template + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.utils.control_utility import ControlUtility, ControlUtilityError +from ignitetest.tests.utils.ignite_test import IgniteTest +from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_8, IgniteVersion, LATEST_2_7, V_2_8_0 + + +# pylint: disable=W0223 +class BaselineTests(IgniteTest): + """ + Tests baseline command + """ + NUM_NODES = 3 + + CONFIG_TEMPLATE = """ + {% if version > "2.9.0" %} + + {% else %} + + {% endif %} + + + + + + + + + + + """ + + @staticmethod + def properties(version): + """ + Render properties for ignite node configuration. + """ + return Template(BaselineTests.CONFIG_TEMPLATE) \ + .render(version=version) + + def __init__(self, test_context): + super(BaselineTests, self).__init__(test_context) + self.servers = None + + @cluster(num_nodes=NUM_NODES) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_8)) + @parametrize(version=str(LATEST_2_7)) + def test_baseline_set(self, version): + """ + Test baseline set. + """ + blt_size = self.NUM_NODES - 2 + self.servers = self.__start_ignite_nodes(version, blt_size) + + control_utility = ControlUtility(self.servers, self.test_context) + control_utility.activate() + + # Check baseline of activated cluster. + baseline = control_utility.baseline() + self.__check_baseline_size(baseline, blt_size) + self.__check_nodes_in_baseline(self.servers.nodes, baseline) + + # Set baseline using list of conststent ids. + new_node = self.__start_ignite_nodes(version, 1) + control_utility.set_baseline(self.servers.nodes + new_node.nodes) + blt_size += 1 + + baseline = control_utility.baseline() + self.__check_baseline_size(baseline, blt_size) + self.__check_nodes_in_baseline(new_node.nodes, baseline) + + # Set baseline using topology version. + new_node = self.__start_ignite_nodes(version, 1) + _, version, _ = control_utility.cluster_state() + control_utility.set_baseline(version) + blt_size += 1 + + baseline = control_utility.baseline() + self.__check_baseline_size(baseline, blt_size) + self.__check_nodes_in_baseline(new_node.nodes, baseline) + + @cluster(num_nodes=NUM_NODES) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_8)) + @parametrize(version=str(LATEST_2_7)) + def test_baseline_add_remove(self, version): + """ + Test add and remove nodes from baseline. + """ + blt_size = self.NUM_NODES - 1 + self.servers = self.__start_ignite_nodes(version, blt_size) + + control_utility = ControlUtility(self.servers, self.test_context) + + control_utility.activate() + + # Add node to baseline. + new_node = self.__start_ignite_nodes(version, 1) + control_utility.add_to_baseline(new_node.nodes) + blt_size += 1 + + baseline = control_utility.baseline() + self.__check_baseline_size(baseline, blt_size) + self.__check_nodes_in_baseline(new_node.nodes, baseline) + + # Expected failure (remove of online node is not allowed). + try: + control_utility.remove_from_baseline(new_node.nodes) + + assert False, "Remove of online node from baseline should fail!" + except ControlUtilityError: + pass + + # Remove of offline node from baseline. + new_node.stop() + + self.servers.await_event("Node left topology", timeout_sec=30, from_the_beginning=True) + + control_utility.remove_from_baseline(new_node.nodes) + blt_size -= 1 + + baseline = control_utility.baseline() + self.__check_baseline_size(baseline, blt_size) + self.__check_nodes_not_in_baseline(new_node.nodes, baseline) + + @cluster(num_nodes=NUM_NODES) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_8)) + @parametrize(version=str(LATEST_2_7)) + def test_activate_deactivate(self, version): + """ + Test activate and deactivate cluster. + """ + self.servers = self.__start_ignite_nodes(version, self.NUM_NODES) + + control_utility = ControlUtility(self.servers, self.test_context) + + control_utility.activate() + + state, _, _ = control_utility.cluster_state() + + assert state.lower() == 'active', 'Unexpected state %s' % state + + control_utility.deactivate() + + state, _, _ = control_utility.cluster_state() + + assert state.lower() == 'inactive', 'Unexpected state %s' % state + + @cluster(num_nodes=NUM_NODES) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_8)) + def test_baseline_autoadjust(self, version): + """ + Test activate and deactivate cluster. + """ + if version < V_2_8_0: + self.logger.info("Skipping test because this feature is not supported for version %s" % version) + return + + blt_size = self.NUM_NODES - 2 + self.servers = self.__start_ignite_nodes(version, blt_size) + + control_utility = ControlUtility(self.servers, self.test_context) + control_utility.activate() + + # Add node. + control_utility.enable_baseline_auto_adjust(2000) + new_node = self.__start_ignite_nodes(version, 1) + blt_size += 1 + + wait_until(lambda: len(control_utility.baseline()) == blt_size, timeout_sec=5) + + baseline = control_utility.baseline() + self.__check_nodes_in_baseline(new_node.nodes, baseline) + + # Add node when auto adjust disabled. + control_utility.disable_baseline_auto_adjust() + old_topology = control_utility.cluster_state().topology_version + new_node = self.__start_ignite_nodes(version, 1) + + wait_until(lambda: control_utility.cluster_state().topology_version != old_topology, timeout_sec=5) + baseline = control_utility.baseline() + self.__check_nodes_not_in_baseline(new_node.nodes, baseline) + + @staticmethod + def __check_nodes_in_baseline(nodes, baseline): + blset = set(node.consistent_id for node in baseline) + + for node in nodes: + assert node.consistent_id in blset + + @staticmethod + def __check_nodes_not_in_baseline(nodes, baseline): + blset = set(node.consistent_id for node in baseline) + + for node in nodes: + assert node.consistent_id not in blset + + @staticmethod + def __check_baseline_size(baseline, size): + assert len(baseline) == size, 'Unexpected size of baseline %d, %d expected' % (len(baseline), size) + + def __start_ignite_nodes(self, version, num_nodes, timeout_sec=180): + ignite_version = IgniteVersion(version) + + servers = IgniteService(self.test_context, num_nodes=num_nodes, version=ignite_version, + properties=self.properties(ignite_version)) + + servers.start(timeout_sec=timeout_sec) + + return servers diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index cd9feecf27ec0..108fa9d96fa95 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -24,6 +24,7 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility from ignitetest.tests.utils.ignite_test import IgniteTest from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion @@ -106,8 +107,7 @@ def test(self, version): single_key_tx_streamer.start() if ignite_version >= V_2_8_0: - long_tx_streamer.execute( - "control.sh --host %s --baseline auto_adjust disable --yes" % ignites.nodes[0].account.hostname) + ControlUtility(ignites, self.test_context).disable_baseline_auto_adjust() self.stage("Stopping server node") diff --git a/modules/ducktests/tests/.pylintrc b/modules/ducktests/tests/tox.ini similarity index 73% rename from modules/ducktests/tests/.pylintrc rename to modules/ducktests/tests/tox.ini index 73536f333a79b..8a7235b6fe11a 100644 --- a/modules/ducktests/tests/.pylintrc +++ b/modules/ducktests/tests/tox.ini @@ -12,6 +12,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +[tox] +envlist = linter +skipsdist = True + +[travis] +python = + 3.8: linter + +[testenv] +envdir = {homedir}/.virtualenvs/ignite-ducktests-{envname} +deps = + ducktape==0.7.8 + requests==2.20.0 + monotonic + mock + pytest + pylint + +[testenv:linter] +basepython = python3.8 +commands = + pylint --rcfile=tox.ini ./ignitetest + [BASIC] min-public-methods=0 # TODO: Remove after migrating to python 3 From 77c3ff45c5249cf024c2cfe9b11f60f283de2cd6 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Tue, 11 Aug 2020 11:50:06 +0300 Subject: [PATCH 36/78] Cellular affinity test (#8130) --- .../CellularAffinityBackupFilter.java | 59 ++++++++++ .../DistributionChecker.java | 65 ++++++++++ .../tests/ignitetest/services/ignite.py | 5 +- .../tests/ignitetest/services/ignite_app.py | 5 +- .../tests/ignitetest/services/spark.py | 2 +- .../ignitetest/services/utils/ignite_aware.py | 8 +- .../services/utils/ignite_aware_app.py | 6 +- .../tests/cellular_affinity_test.py | 111 ++++++++++++++++++ 8 files changed, 252 insertions(+), 9 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularAffinityBackupFilter.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java create mode 100644 modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularAffinityBackupFilter.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularAffinityBackupFilter.java new file mode 100644 index 0000000000000..6ecdd3fca6792 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularAffinityBackupFilter.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.cellular_affinity_test; + +import java.util.List; +import java.util.Objects; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.lang.IgniteBiPredicate; + +/** + * + */ +public class CellularAffinityBackupFilter implements IgniteBiPredicate> { + /** */ + private static final long serialVersionUID = 1L; + + /** Attribute name. */ + private final String attrName; + + /** + * @param attrName The attribute name for the attribute to compare. + */ + public CellularAffinityBackupFilter(String attrName) { + this.attrName = attrName; + } + + /** + * Defines a predicate which returns {@code true} if a node is acceptable for a backup + * or {@code false} otherwise. An acceptable node is one where its attribute value + * is exact match with previously selected nodes. If an attribute does not + * exist on exactly one node of a pair, then the attribute does not match. If the attribute + * does not exist both nodes of a pair, then the attribute matches. + * + * @param candidate A node that is a candidate for becoming a backup node for a partition. + * @param previouslySelected A list of primary/backup nodes already chosen for a partition. + * The primary is first. + */ + @Override public boolean apply(ClusterNode candidate, List previouslySelected) { + for (ClusterNode node : previouslySelected) + return Objects.equals(candidate.attribute(attrName), node.attribute(attrName)); + + return true; + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java new file mode 100644 index 0000000000000..38ec2a49e3a5a --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.cellular_affinity_test; + +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.Ignite; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * + */ +public class DistributionChecker extends IgniteAwareApplication { + /** + * @param ignite Ignite. + */ + public DistributionChecker(Ignite ignite) { + super(ignite); + } + + /** + * {@inheritDoc} + */ + @Override protected void run(JsonNode jsonNode) { + String cacheName = jsonNode.get("cacheName").asText(); + String attr = jsonNode.get("attr").asText(); + int nodesPerCell = jsonNode.get("nodesPerCell").intValue(); + + assert ignite.cluster().forServers().nodes().size() > nodesPerCell : "Cluster should contain more than one cell"; + + for (int i = 0; i < 10_000; i++) { + Collection nodes = ignite.affinity(cacheName).mapKeyToPrimaryAndBackups(i); + + Map stat = nodes.stream().collect( + Collectors.groupingBy(n -> n.attributes().get(attr), Collectors.counting())); + + log.info("Checking [key=" + i + ", stat=" + stat + "]"); + + assert 1 == stat.keySet().size() : "Partition should be located on nodes from only one cell [stat=" + stat + "]"; + + assert nodesPerCell == stat.values().iterator().next() : + "Partition should be located on all nodes of the cell [stat=" + stat + "]"; + } + + markSyncExecutionComplete(); + } +} diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index 08732c6a87830..4e2ea96455fb8 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -45,8 +45,9 @@ class IgniteService(IgniteAwareService): } # pylint: disable=R0913 - def __init__(self, context, num_nodes, modules=None, client_mode=False, version=DEV_BRANCH, properties=""): - super(IgniteService, self).__init__(context, num_nodes, modules, client_mode, version, properties) + def __init__(self, context, num_nodes, modules=None, client_mode=False, version=DEV_BRANCH, properties="", + jvm_opts=None): + super(IgniteService, self).__init__(context, num_nodes, modules, client_mode, version, properties, jvm_opts) # pylint: disable=W0221 def start(self, timeout_sec=180): diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index ec9007d2d3cb0..8e89911e1059e 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -29,6 +29,7 @@ class IgniteApplicationService(IgniteAwareApplicationService): # pylint: disable=R0913 def __init__(self, context, java_class_name, modules=None, client_mode=True, version=DEV_BRANCH, - properties="", params="", timeout_sec=60): + properties="", params="", jvm_options=None, timeout_sec=60): super(IgniteApplicationService, self).__init__(context, java_class_name, modules, client_mode, version, - properties, params, timeout_sec, self.service_java_class_name) + properties, params, jvm_options, timeout_sec, + self.service_java_class_name) diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py index 4b4b9cd01048d..72ce9bc9e3e40 100644 --- a/modules/ducktests/tests/ignitetest/services/spark.py +++ b/modules/ducktests/tests/ignitetest/services/spark.py @@ -44,7 +44,7 @@ def __init__(self, context, modules=None, version=DEV_BRANCH, num_nodes=3, prope modules = modules or [] modules.extend(["ignite-spark"]) - IgniteAwareService.__init__(self, context, num_nodes, modules, False, version, properties) + IgniteAwareService.__init__(self, context, num_nodes, modules, False, version, properties, None) self.log_level = "DEBUG" diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 2e292a3e37b7b..78300824ff4ba 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -49,10 +49,14 @@ class IgniteAwareService(BackgroundThreadService): } # pylint: disable=R0913 - def __init__(self, context, num_nodes, modules, client_mode, version, properties): + def __init__(self, context, num_nodes, modules, client_mode, version, properties, jvm_options): super(IgniteAwareService, self).__init__(context, num_nodes) - self.jvm_options = context.globals.get("jvm_opts", "") + global_jvm_options = context.globals.get("jvm_opts", "") + + service_jvm_options = " ".join(map(lambda x: '-J' + x, jvm_options)) if jvm_options else "" + + self.jvm_options = " ".join(filter(None, [global_jvm_options, service_jvm_options])) self.log_level = "DEBUG" self.properties = properties diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index eece2a6af65af..b376ff309e841 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -30,9 +30,11 @@ class IgniteAwareApplicationService(IgniteAwareService): """ # pylint: disable=R0913 - def __init__(self, context, java_class_name, modules, client_mode, version, properties, params, timeout_sec, + def __init__(self, context, java_class_name, modules, client_mode, version, properties, params, jvm_options, + timeout_sec, service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): - super(IgniteAwareApplicationService, self).__init__(context, 1, modules, client_mode, version, properties) + super(IgniteAwareApplicationService, self).__init__(context, 1, modules, client_mode, version, properties, + jvm_options) self.servicejava_class_name = service_java_class_name self.java_class_name = java_class_name diff --git a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py new file mode 100644 index 0000000000000..80870122b9a22 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains Cellular Affinity tests. +""" + +from ducktape.mark import parametrize +from ducktape.mark.resource import cluster +from jinja2 import Template + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.tests.utils.ignite_test import IgniteTest +from ignitetest.tests.utils.version import DEV_BRANCH + + +# pylint: disable=W0223 +class CellularAffinity(IgniteTest): + """ + Tests Cellular Affinity scenarios. + """ + NUM_NODES = 3 + + ATTRIBUTE = "CELL" + + CACHE_NAME = "test-cache" + + CONFIG_TEMPLATE = """ + + + + + + + + + + + + + + + + + + """ + + @staticmethod + def properties(): + """ + :return: Configuration properties. + """ + return Template(CellularAffinity.CONFIG_TEMPLATE) \ + .render(nodes=CellularAffinity.NUM_NODES, # bigger than cell capacity (to handle single cell useless test) + attr=CellularAffinity.ATTRIBUTE, + cacheName=CellularAffinity.CACHE_NAME) + + def __init__(self, test_context): + super(CellularAffinity, self).__init__(test_context=test_context) + + def setUp(self): + pass + + def teardown(self): + pass + + @cluster(num_nodes=NUM_NODES * 3 + 1) + @parametrize(version=str(DEV_BRANCH)) + def test(self, version): + """ + Test Cellular Affinity scenario (partition distribution). + """ + self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=1']) + self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=2']) + self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=XXX', '-DRANDOM=42']) + + checker = IgniteApplicationService( + self.test_context, + java_class_name="org.apache.ignite.internal.ducktest.tests.cellular_affinity_test.DistributionChecker", + params={"cacheName": CellularAffinity.CACHE_NAME, + "attr": CellularAffinity.ATTRIBUTE, + "nodesPerCell": self.NUM_NODES}, + version=version) + + checker.run() + + def start_cell(self, ignite_version, jvm_opts): + """ + Starts cell. + """ + ignites = IgniteService( + self.test_context, + num_nodes=CellularAffinity.NUM_NODES, + version=ignite_version, + properties=self.properties(), + jvm_opts=jvm_opts) + + ignites.start() From d750eeeabb9557fc4f2d12792bed8635ba27765b Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Tue, 11 Aug 2020 14:44:52 +0300 Subject: [PATCH 37/78] Ignite ducktape test decorators (#8137) --- .../services/utils/control_utility.py | 22 ++++--- .../ignitetest/tests/control_utility_test.py | 6 +- .../tests/ignitetest/tests/utils/__init__.py | 3 + .../tests/ignitetest/tests/utils/_mark.py | 58 +++++++++++++++++++ 4 files changed, 76 insertions(+), 13 deletions(-) create mode 100644 modules/ducktests/tests/ignitetest/tests/utils/_mark.py diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index 36ff5bb796bca..ed91d187987e4 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -43,9 +43,9 @@ def cluster_state(self): """ :return: Cluster state. """ - output = self.__run("--baseline") + result = self.__run("--baseline") - return self.__parse_cluster_state(output) + return self.__parse_cluster_state(result) def set_baseline(self, baseline): """ @@ -105,18 +105,22 @@ def deactivate(self): @staticmethod def __parse_cluster_state(output): - state_pattern = re.compile("Cluster state: ([^\\s]+)") - topology_pattern = re.compile("Current topology version: (\\d+)") - baseline_pattern = re.compile("Consistent(Id|ID)=([^\\s]+),\\sS(tate|TATE)=([^\\s]+),?(\\sOrder=(\\d+))?") + state_pattern = re.compile("Cluster state: (?P[^\\s]+)") + topology_pattern = re.compile("Current topology version: (?P\\d+)") + baseline_pattern = re.compile("Consistent(Id|ID)=(?P[^\\s]+),\\sS(tate|TATE)=(?P[^\\s]+)," + "?(\\sOrder=(?P\\d+))?") match = state_pattern.search(output) - state = match.group(1) if match else None + state = match.group("cluster_state") if match else None match = topology_pattern.search(output) - topology = int(match.group(1)) if match else None + topology = int(match.group("topology_version")) if match else None - baseline = [BaselineNode(consistent_id=m[1], state=m[3], order=int(m[5]) if m[5] else None) - for m in baseline_pattern.findall(output)] + baseline = [] + for match in baseline_pattern.finditer(output): + node = BaselineNode(consistent_id=match.group("consistent_id"), state=match.group("state"), + order=int(match.group("order")) if match.group("order") else None) + baseline.append(node) return ClusterState(state=state, topology_version=topology, baseline=baseline) diff --git a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py index 027cc3c4436b5..a1b00dee61757 100644 --- a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py +++ b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py @@ -23,6 +23,7 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.utils.control_utility import ControlUtility, ControlUtilityError +from ignitetest.tests.utils import version_if from ignitetest.tests.utils.ignite_test import IgniteTest from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_8, IgniteVersion, LATEST_2_7, V_2_8_0 @@ -171,16 +172,13 @@ def test_activate_deactivate(self, version): assert state.lower() == 'inactive', 'Unexpected state %s' % state @cluster(num_nodes=NUM_NODES) + @version_if(lambda version: version >= V_2_8_0) @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_8)) def test_baseline_autoadjust(self, version): """ Test activate and deactivate cluster. """ - if version < V_2_8_0: - self.logger.info("Skipping test because this feature is not supported for version %s" % version) - return - blt_size = self.NUM_NODES - 2 self.servers = self.__start_ignite_nodes(version, blt_size) diff --git a/modules/ducktests/tests/ignitetest/tests/utils/__init__.py b/modules/ducktests/tests/ignitetest/tests/utils/__init__.py index ec2014340d78f..777351b341818 100644 --- a/modules/ducktests/tests/ignitetest/tests/utils/__init__.py +++ b/modules/ducktests/tests/ignitetest/tests/utils/__init__.py @@ -12,3 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# pylint: disable=C0114 +from ._mark import version_if diff --git a/modules/ducktests/tests/ignitetest/tests/utils/_mark.py b/modules/ducktests/tests/ignitetest/tests/utils/_mark.py new file mode 100644 index 0000000000000..d5ea8cf8b2a73 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/utils/_mark.py @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module contains useful test decorators. +""" +import six +from ducktape.mark._mark import Ignore, Mark + +from ignitetest.tests.utils.version import IgniteVersion + + +class VersionIf(Ignore): + """ + Ignore test if version doesn't corresponds to condition. + """ + def __init__(self, condition): + super(VersionIf, self).__init__() + self.condition = condition + + def apply(self, seed_context, context_list): + assert len(context_list) > 0, "ignore_if decorator is not being applied to any test cases" + + for ctx in context_list: + assert 'version' in ctx.injected_args, "'version' in injected args not present" + version = ctx.injected_args['version'] + assert isinstance(version, six.string_types), "'version' in injected args must be a string" + ctx.ignore = ctx.ignore or not self.condition(IgniteVersion(version)) + + return context_list + + def __eq__(self, other): + return super(VersionIf, self).__eq__(other) and self.condition == other.condition + + +def version_if(condition): + """ + Mark decorated test method as IGNORE if version doesn't corresponds to condition. + + :param condition: function(IgniteVersion) -> bool + """ + def ignorer(func): + Mark.mark(func, VersionIf(condition)) + return func + + return ignorer From 9e34638ef22979f542f7a2c2304dce8eaba66a73 Mon Sep 17 00:00:00 2001 From: Vladsz83 Date: Tue, 11 Aug 2020 17:43:13 +0300 Subject: [PATCH 38/78] looks good (#8141) --- modules/ducktests/tests/docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index 6de634b068d39..a89ea744379ad 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -20,7 +20,8 @@ MAINTAINER Apache Ignite dev@ignite.apache.org VOLUME ["/opt/ignite-dev"] # Set the timezone. -ENV TZ="/usr/share/zoneinfo/Europe/Moscow" +ENV TZ=Europe/Moscow +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # Do not ask for confirmations when running apt-get, etc. ENV DEBIAN_FRONTEND noninteractive From 84d8016422cf2e2aaba4d4ad269a5f715d830a80 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Wed, 12 Aug 2020 13:31:27 +0300 Subject: [PATCH 39/78] API simplification (#8140) --- .../tests/DataGenerationApplication.java | 8 --- .../DistributionChecker.java | 8 --- .../LongTxStreamerApplication.java | 8 --- .../SingleKeyTxStreamerApplication.java | 8 --- .../tests/smoke_test/SimpleApplication.java | 8 --- .../utils/IgniteApplicationService.java | 68 ------------------- .../utils/IgniteAwareApplication.java | 16 ++--- .../utils/IgniteAwareApplicationService.java | 38 +++++++++-- .../tests/ignitetest/services/ignite_app.py | 4 +- .../services/utils/ignite_aware_app.py | 7 +- 10 files changed, 42 insertions(+), 131 deletions(-) delete mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java index f9732b2844aad..a65644aec24ec 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java @@ -18,7 +18,6 @@ package org.apache.ignite.internal.ducktest.tests; import com.fasterxml.jackson.databind.JsonNode; -import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; @@ -27,13 +26,6 @@ * */ public class DataGenerationApplication extends IgniteAwareApplication { - /** - * @param ignite Ignite. - */ - public DataGenerationApplication(Ignite ignite) { - super(ignite); - } - /** {@inheritDoc} */ @Override protected void run(JsonNode jsonNode) { log.info("Creating cache..."); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java index 38ec2a49e3a5a..2daf63eb1821b 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java @@ -21,7 +21,6 @@ import java.util.Map; import java.util.stream.Collectors; import com.fasterxml.jackson.databind.JsonNode; -import org.apache.ignite.Ignite; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; @@ -29,13 +28,6 @@ * */ public class DistributionChecker extends IgniteAwareApplication { - /** - * @param ignite Ignite. - */ - public DistributionChecker(Ignite ignite) { - super(ignite); - } - /** * {@inheritDoc} */ diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java index 82596f8d8e39d..7e3f146573ffd 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java @@ -20,7 +20,6 @@ import java.util.Collection; import java.util.concurrent.CountDownLatch; import com.fasterxml.jackson.databind.JsonNode; -import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInterruptedCheckedException; @@ -40,13 +39,6 @@ public class LongTxStreamerApplication extends IgniteAwareApplication { /** Started. */ private static final CountDownLatch started = new CountDownLatch(TX_CNT); - /** - * @param ignite Ignite. - */ - public LongTxStreamerApplication(Ignite ignite) { - super(ignite); - } - /** {@inheritDoc} */ @Override public void run(JsonNode jsonNode) throws InterruptedException { IgniteCache cache = ignite.getOrCreateCache(jsonNode.get("cacheName").asText()); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java index f38e8111bcb6c..8c8be15b53566 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java @@ -18,7 +18,6 @@ package org.apache.ignite.internal.ducktest.tests.pme_free_switch_test; import com.fasterxml.jackson.databind.JsonNode; -import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; @@ -26,13 +25,6 @@ * */ public class SingleKeyTxStreamerApplication extends IgniteAwareApplication { - /** - * @param ignite Ignite. - */ - public SingleKeyTxStreamerApplication(Ignite ignite) { - super(ignite); - } - /** {@inheritDoc} */ @Override public void run(JsonNode jsonNode) { IgniteCache cache = ignite.getOrCreateCache(jsonNode.get("cacheName").asText()); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java index 379f5b11b4842..5c243b4ebbdec 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/SimpleApplication.java @@ -19,7 +19,6 @@ import java.util.UUID; import com.fasterxml.jackson.databind.JsonNode; -import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; @@ -29,13 +28,6 @@ * Simple application that used in smoke tests */ public class SimpleApplication extends IgniteAwareApplication { - /** - * @param ignite Ignite. - */ - public SimpleApplication(Ignite ignite) { - super(ignite); - } - /** {@inheritDoc} */ @Override public void run(JsonNode jsonNode) { IgniteCache cache = ignite.getOrCreateCache(UUID.randomUUID().toString()); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java deleted file mode 100644 index b4f60b50226c2..0000000000000 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteApplicationService.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.ducktest.utils; - -import java.util.Base64; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.ignite.Ignite; -import org.apache.ignite.Ignition; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.IgnitionEx; -import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; -import org.apache.ignite.lang.IgniteBiTuple; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; - -/** - * - */ -public class IgniteApplicationService { - /** Logger. */ - private static final Logger log = LogManager.getLogger(IgniteApplicationService.class.getName()); - - /** - * @param args Args. - */ - public static void main(String[] args) throws Exception { - log.info("Starting Application... [params=" + args[0] + "]"); - - String[] params = args[0].split(","); - - Class clazz = Class.forName(params[0]); - - IgniteBiTuple cfgs = IgnitionEx.loadConfiguration(params[1]); - - IgniteConfiguration cfg = cfgs.get1(); - - assert cfg.isClientMode(); - - log.info("Starting Ignite node..."); - - try (Ignite ignite = Ignition.start(cfg)) { - IgniteAwareApplication app = (IgniteAwareApplication)clazz.getConstructor(Ignite.class).newInstance(ignite); - - ObjectMapper mapper = new ObjectMapper(); - - JsonNode jsonNode = params.length > 2 ? - mapper.readTree(Base64.getDecoder().decode(params[2])) : mapper.createObjectNode(); - - app.start(jsonNode); - } - } -} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java index 28c8d20d6ea4d..7a173a5689846 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java @@ -50,21 +50,15 @@ public abstract class IgniteAwareApplication { private static volatile boolean terminated; /** Ignite. */ - protected final Ignite ignite; + protected Ignite ignite; + + /** Cfg path. */ + protected String cfgPath; /** * Default constructor. */ protected IgniteAwareApplication() { - ignite = null; - } - - /** - * - */ - protected IgniteAwareApplication(Ignite ignite) { - this.ignite = ignite; - Runtime.getRuntime().addShutdownHook(new Thread(() -> { terminate(); @@ -170,6 +164,8 @@ public void start(JsonNode jsonNode) { try { log.info("Application params: " + jsonNode); + assert cfgPath != null; + run(jsonNode); assert inited : "Was not properly initialized."; diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java index de13ef1f46750..c9127655b72fa 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java @@ -20,7 +20,12 @@ import java.util.Base64; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.ignite.Ignite; +import org.apache.ignite.Ignition; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgnitionEx; +import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; +import org.apache.ignite.lang.IgniteBiTuple; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -39,17 +44,36 @@ public static void main(String[] args) throws Exception { String[] params = args[0].split(","); - Class clazz = Class.forName(params[0]); + boolean startIgnite = Boolean.parseBoolean(params[0]); - IgniteAwareApplication app = (IgniteAwareApplication)clazz.getConstructor().newInstance(); + Class clazz = Class.forName(params[1]); + + String cfgPath = params[2]; ObjectMapper mapper = new ObjectMapper(); - JsonNode jsonNode = params.length > 2 ? - mapper.readTree(Base64.getDecoder().decode(params[2])) : mapper.createObjectNode(); + JsonNode jsonNode = params.length > 3 ? + mapper.readTree(Base64.getDecoder().decode(params[3])) : mapper.createObjectNode(); + + IgniteAwareApplication app = + (IgniteAwareApplication)clazz.getConstructor().newInstance(); + + app.cfgPath = cfgPath; + + if (startIgnite) { + log.info("Starting Ignite node..."); + + IgniteBiTuple cfgs = IgnitionEx.loadConfiguration(cfgPath); + + IgniteConfiguration cfg = cfgs.get1(); - ((ObjectNode)jsonNode).put("cfgPath", params[1]); + try (Ignite ignite = Ignition.start(cfg)) { + app.ignite = ignite; - app.start(jsonNode); + app.start(jsonNode); + } + } + else + app.start(jsonNode); } } diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 8e89911e1059e..ef5120e9e3e3b 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -25,11 +25,9 @@ class IgniteApplicationService(IgniteAwareApplicationService): """ The Ignite application service allows to perform custom logic writen on java. """ - service_java_class_name = "org.apache.ignite.internal.ducktest.utils.IgniteApplicationService" # pylint: disable=R0913 def __init__(self, context, java_class_name, modules=None, client_mode=True, version=DEV_BRANCH, properties="", params="", jvm_options=None, timeout_sec=60): super(IgniteApplicationService, self).__init__(context, java_class_name, modules, client_mode, version, - properties, params, jvm_options, timeout_sec, - self.service_java_class_name) + properties, params, jvm_options, timeout_sec, start_ignite=True) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index b376ff309e841..b05f5a16b0838 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -31,7 +31,7 @@ class IgniteAwareApplicationService(IgniteAwareService): # pylint: disable=R0913 def __init__(self, context, java_class_name, modules, client_mode, version, properties, params, jvm_options, - timeout_sec, + timeout_sec, start_ignite=False, service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): super(IgniteAwareApplicationService, self).__init__(context, 1, modules, client_mode, version, properties, jvm_options) @@ -41,6 +41,7 @@ def __init__(self, context, java_class_name, modules, client_mode, version, prop self.timeout_sec = timeout_sec self.stop_timeout_sec = 10 self.params = params + self.start_ignite = start_ignite def start(self): super(IgniteAwareApplicationService, self).start() @@ -84,10 +85,10 @@ def app_args(self): """ :return: Application arguments. """ - args = self.java_class_name + "," + IgniteAwareApplicationService.CONFIG_FILE + args = ",".join([str(self.start_ignite), self.java_class_name, IgniteAwareApplicationService.CONFIG_FILE]) if self.params != "": - args += "," + str(base64.b64encode(json.dumps(self.params).encode("UTF-8"))) + args = ",".join([args, str(base64.b64encode(json.dumps(self.params).encode("UTF-8")))]) return args From 2e4b4addab42b02b78b03787a25795255ec5312e Mon Sep 17 00:00:00 2001 From: Vladsz83 Date: Wed, 12 Aug 2020 17:55:39 +0300 Subject: [PATCH 40/78] Fix measuring timers in discovery tests (#8142) --- .../tests/ignitetest/services/ignite.py | 64 +++++- .../ignitetest/services/utils/concurrent.py | 90 +++++++++ .../ignitetest/services/utils/ignite_aware.py | 2 +- .../ignitetest/services/utils/time_utils.py | 28 +++ .../tests/ignitetest/tests/discovery_test.py | 190 +++++++++++++----- 5 files changed, 318 insertions(+), 56 deletions(-) create mode 100644 modules/ducktests/tests/ignitetest/services/utils/concurrent.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/time_utils.py diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index 4e2ea96455fb8..1c5a476d7fd66 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -17,12 +17,19 @@ This module contains class to start ignite cluster node. """ -import os.path +import functools +import operator +import os import signal +import time +from datetime import datetime +from threading import Thread +import monotonic from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.utils.util import wait_until +from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue from ignitetest.services.utils.ignite_aware import IgniteAwareService from ignitetest.tests.utils.version import DEV_BRANCH @@ -91,7 +98,7 @@ def stop_node(self, node, clean_shutdown=True, timeout_sec=60): sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL for pid in pids: - node.account.signal(pid, sig, allow_fail=False) + self.__stop_node(node, pid, sig) try: wait_until(lambda: len(self.pids(node)) == 0, timeout_sec=timeout_sec, @@ -100,6 +107,59 @@ def stop_node(self, node, clean_shutdown=True, timeout_sec=60): self.thread_dump(node) raise + def stop_nodes_async(self, nodes, delay_ms=0, clean_shutdown=True, timeout_sec=20, wait_for_stop=False): + """ + Stops the nodes asynchronously. + """ + sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL + + sem = CountDownLatch(len(nodes)) + time_holder = AtomicValue() + + delay = 0 + threads = [] + + for node in nodes: + thread = Thread(target=self.__stop_node, + args=(node, next(iter(self.pids(node))), sig, sem, delay, time_holder)) + + threads.append(thread) + + thread.start() + + delay += delay_ms + + for thread in threads: + thread.join(timeout_sec) + + if wait_for_stop: + try: + wait_until(lambda: len(functools.reduce(operator.iconcat, (self.pids(n) for n in nodes), [])) == 0, + timeout_sec=timeout_sec, err_msg="Ignite node failed to stop in %d seconds" % timeout_sec) + except Exception: + for node in nodes: + self.thread_dump(node) + raise + + return time_holder.get() + + @staticmethod + def __stop_node(node, pid, sig, start_waiter=None, delay_ms=0, time_holder=None): + if start_waiter: + start_waiter.count_down() + start_waiter.wait() + + if delay_ms > 0: + time.sleep(delay_ms/1000.0) + + if time_holder: + mono = monotonic.monotonic() + timestamp = datetime.now() + + time_holder.compare_and_set(None, (mono, timestamp)) + + node.account.signal(pid, sig, False) + def clean_node(self, node): node.account.kill_java_processes(self.APP_SERVICE_CLASS, clean_shutdown=False, allow_fail=True) node.account.ssh("sudo rm -rf -- %s" % IgniteService.PERSISTENT_ROOT, allow_fail=False) diff --git a/modules/ducktests/tests/ignitetest/services/utils/concurrent.py b/modules/ducktests/tests/ignitetest/services/utils/concurrent.py new file mode 100644 index 0000000000000..57768bcc53371 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/concurrent.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains concurrent utils. +""" + +import threading + + +class CountDownLatch(object): + """ + A count-down latch. + """ + def __init__(self, count=1): + self.count = count + self.cond_var = threading.Condition() + + def count_down(self): + """ + Decreases the latch counter. + """ + with self.cond_var: + if self.count > 0: + self.count -= 1 + if self.count == 0: + self.cond_var.notifyAll() + + def wait(self): + """ + Blocks current thread if the latch is not free. + """ + with self.cond_var: + while self.count > 0: + self.cond_var.wait() + + +class AtomicValue: + """ + An atomic reference holder. + """ + def __init__(self, value=None): + self.value = value + self.lock = threading.Lock() + + def set(self, value): + """ + Sets new value to hold. + :param value: New value to hold. + """ + with self.lock: + self.value = value + + def get(self): + """ + Gives current value. + """ + with self.lock: + return self.value + + def compare_and_set(self, expected, value): + """ + Sets new value to hold if current one equals expected. + :param expected: The value to compare with. + :param value: New value to hold. + """ + return self.check_and_set(lambda: self.value == expected, value) + + def check_and_set(self, condition, value): + """ + Sets new value to hold by condition. + :param condition: The condition to check. + :param value: New value to hold. + """ + with self.lock: + if condition(): + self.value = value + return self.value diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 78300824ff4ba..0b5e18f9af196 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -148,7 +148,7 @@ def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning :param evt_message: Event message. :param node: Ignite service node. :param timeout_sec: Number of seconds to check the condition for before failing. - :param from_the_beginning: If True, search for message from the beggining of log file. + :param from_the_beginning: If True, search for message from the beginning of log file. :param backoff_sec: Number of seconds to back off between each failure to meet the condition before checking again. """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/time_utils.py b/modules/ducktests/tests/ignitetest/services/utils/time_utils.py new file mode 100644 index 0000000000000..01926bfed7129 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/time_utils.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains time utils. +""" + +import time + + +def epoch_mills(date_time): + """ + :param date_time: a datetime. + :return: milliseconds since epoch of passed datetime. + """ + return int(round((time.mktime(date_time.timetuple()) + date_time.microsecond / 1e6) * 1000)) diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 01f6bcbe76290..8d933966d6fd5 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -18,12 +18,16 @@ """ import random +import re +from datetime import datetime from ducktape.mark import parametrize from ducktape.mark.resource import cluster from jinja2 import Template from ignitetest.services.ignite import IgniteService +from ignitetest.services.utils.ignite_aware import IgniteAwareService +from ignitetest.services.utils.time_utils import epoch_mills from ignitetest.services.zk.zookeeper import ZookeeperService from ignitetest.tests.utils.ignite_test import IgniteTest from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7 @@ -32,26 +36,27 @@ # pylint: disable=W0223 class DiscoveryTest(IgniteTest): """ - Test basic discovery scenarious (TCP and Zookeeper). + Test various node failure scenarios (TCP and ZooKeeper). 1. Start of ignite cluster. 2. Kill random node. 3. Wait that survived node detects node failure. """ NUM_NODES = 7 + FAILURE_DETECTION_TIMEOUT = 2000 + CONFIG_TEMPLATE = """ - {% if zookeeper_settings %} + + {% if zookeeper_settings %} {% with zk = zookeeper_settings %} - - {% endwith %} - {% endif %} + {% endif %} """ def __init__(self, test_context): @@ -59,14 +64,28 @@ def __init__(self, test_context): self.zk_quorum = None self.servers = None + def __start_zk_quorum(self): + self.zk_quorum = ZookeeperService(self.test_context, 3) + + self.stage("Starting ZooKeeper quorum") + + self.zk_quorum.start() + + self.stage("ZooKeeper quorum started") + @staticmethod - def properties(zookeeper_settings=None): + def __properties(zookeeper_settings=None): """ - :param zookeeper_settings: ZookeperDiscoverySpi settings. If None, TcpDiscoverySpi will be used. + :param zookeeper_settings: ZooKeeperDiscoverySpi settings. If None, TcpDiscoverySpi will be used. :return: Rendered node's properties. """ return Template(DiscoveryTest.CONFIG_TEMPLATE) \ - .render(zookeeper_settings=zookeeper_settings) + .render(failure_detection_timeout=DiscoveryTest.FAILURE_DETECTION_TIMEOUT, + zookeeper_settings=zookeeper_settings) + + @staticmethod + def __zk_properties(connection_string): + return DiscoveryTest.__properties(zookeeper_settings={'connection_string': connection_string}) def setUp(self): pass @@ -81,31 +100,68 @@ def teardown(self): @cluster(num_nodes=NUM_NODES) @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_7)) - def test_tcp(self, version): + def test_tcp_not_coordinator_single(self, version): + """ + Test single-node-failure scenario (not the coordinator) with TcpDiscoverySpi. + """ + return self.__simulate_nodes_failure(version, self.__properties(), 1) + + @cluster(num_nodes=NUM_NODES) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_7)) + def test_tcp_not_coordinator_two(self, version): """ - Test basic discovery scenario with TcpDiscoverySpi. + Test two-node-failure scenario (not the coordinator) with TcpDiscoverySpi. """ - return self.__basic_test__(version, False) + return self.__simulate_nodes_failure(version, self.__properties(), 2) + + @cluster(num_nodes=NUM_NODES) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_7)) + def test_tcp_coordinator(self, version): + """ + Test coordinator-failure scenario with TcpDiscoverySpi. + """ + return self.__simulate_nodes_failure(version, self.__properties(), 0) @cluster(num_nodes=NUM_NODES + 3) @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_7)) - def test_zk(self, version): + def test_zk_not_coordinator_single(self, version): """ - Test basic discovery scenario with ZookeeperDiscoverySpi. + Test single node failure scenario (not the coordinator) with ZooKeeper. """ - return self.__basic_test__(version, True) + self.__start_zk_quorum() - def __basic_test__(self, version, with_zk=False): - if with_zk: - self.zk_quorum = ZookeeperService(self.test_context, 3) - self.stage("Starting Zookeper quorum") - self.zk_quorum.start() - properties = self.properties(zookeeper_settings={'connection_string': self.zk_quorum.connection_string()}) - self.stage("Zookeper quorum started") - else: - properties = self.properties() + return self.__simulate_nodes_failure(version, self.__zk_properties(self.zk_quorum.connection_string()), 1) + + @cluster(num_nodes=NUM_NODES + 3) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_7)) + def test_zk_not_coordinator_two(self, version): + """ + Test two-node-failure scenario (not the coordinator) with ZooKeeper. + """ + self.__start_zk_quorum() + + return self.__simulate_nodes_failure(version, self.__zk_properties(self.zk_quorum.connection_string()), 2) + + @cluster(num_nodes=NUM_NODES+3) + @parametrize(version=str(DEV_BRANCH)) + @parametrize(version=str(LATEST_2_7)) + def test_zk_coordinator(self, version): + """ + Test coordinator-failure scenario with ZooKeeper. + """ + self.__start_zk_quorum() + return self.__simulate_nodes_failure(version, self.__zk_properties(self.zk_quorum.connection_string()), 0) + + def __simulate_nodes_failure(self, version, properties, nodes_to_kill=1): + """ + :param nodes_to_kill: How many nodes to kill. If <1, the coordinator is the choice. Otherwise: not-coordinator + nodes of given number. + """ self.servers = IgniteService( self.test_context, num_nodes=self.NUM_NODES, @@ -115,48 +171,76 @@ def __basic_test__(self, version, with_zk=False): self.stage("Starting ignite cluster") - start = self.monotonic() + time_holder = self.monotonic() + self.servers.start() - data = {'Ignite cluster start time (s)': self.monotonic() - start} + + if nodes_to_kill > self.servers.num_nodes - 1: + raise Exception("Too many nodes to kill: " + str(nodes_to_kill)) + + data = {'Ignite cluster start time (s)': round(self.monotonic() - time_holder, 1)} self.stage("Topology is ready") - # Node failure detection - fail_node, survived_node = self.choose_random_node_to_kill(self.servers) + failed_nodes, survived_node = self.__choose_node_to_kill(nodes_to_kill) - data["nodes"] = [node.node_id() for node in self.servers.nodes] + ids_to_wait = [node.discovery_info().node_id for node in failed_nodes] - disco_infos = [] - for node in self.servers.nodes: - disco_info = node.discovery_info() - disco_infos.append({ - "id": disco_info.node_id, - "consistent_id": disco_info.consistent_id, - "coordinator": disco_info.coordinator, - "order": disco_info.order, - "int_order": disco_info.int_order, - "is_client": disco_info.is_client - }) + self.stage("Stopping " + str(len(failed_nodes)) + " nodes.") - data["node_disco_info"] = disco_infos + first_terminated = self.servers.stop_nodes_async(failed_nodes, clean_shutdown=False, wait_for_stop=False) - self.servers.stop_node(fail_node, clean_shutdown=False) + self.stage("Waiting for failure detection of " + str(len(failed_nodes)) + " nodes.") - start = self.monotonic() - self.servers.await_event_on_node("Node FAILED", random.choice(survived_node), 60, True) + # Keeps dates of logged node failures. + logged_timestamps = [] - data['Failure of node detected in time (s)'] = self.monotonic() - start + for failed_id in ids_to_wait: + self.servers.await_event_on_node(self.__failed_pattern(failed_id), survived_node, 10, + from_the_beginning=True, backoff_sec=0.01) + # Save mono of last detected failure. + time_holder = self.monotonic() + self.stage("Failure detection measured.") + + for failed_id in ids_to_wait: + _, stdout, _ = survived_node.account.ssh_client.exec_command( + "grep '%s' %s" % (self.__failed_pattern(failed_id), IgniteAwareService.STDOUT_STDERR_CAPTURE)) + + logged_timestamps.append( + datetime.strptime(re.match("^\\[[^\\[]+\\]", stdout.read()).group(), "[%Y-%m-%d %H:%M:%S,%f]")) + + logged_timestamps.sort(reverse=True) + + # Failure detection delay. + time_holder = int((time_holder - first_terminated[0]) * 1000) + # Failure detection delay by log. + by_log = epoch_mills(logged_timestamps[0]) - epoch_mills(first_terminated[1]) + + assert by_log > 0, "Negative node failure detection delay: " + by_log + ". Probably it is a timezone issue." + assert by_log <= time_holder, "Value of node failure detection delay taken from by the node log (" + \ + str(by_log) + "ms) must be lesser than measured value (" + str(time_holder) + \ + "ms) because watching this event consumes extra time." + + data['Detection of node(s) failure, measured (ms)'] = time_holder + data['Detection of node(s) failure, by the log (ms)'] = by_log + data['Nodes failed'] = len(failed_nodes) return data @staticmethod - def choose_random_node_to_kill(service): - """ - :param service: Service nodes to process. - :return: Tuple of random node to kill and survived nodes - """ - idx = random.randint(0, len(service.nodes) - 1) + def __failed_pattern(failed_node_id): + return "Node FAILED: .\\{1,\\}Node \\[id=" + failed_node_id + + def __choose_node_to_kill(self, nodes_to_kill): + nodes = self.servers.nodes + coordinator = nodes[0].discovery_info().coordinator + + if nodes_to_kill < 1: + to_kill = next(node for node in nodes if node.discovery_info().node_id == coordinator) + else: + to_kill = random.sample([n for n in nodes if n.discovery_info().node_id != coordinator], nodes_to_kill) + + to_kill = [to_kill] if not isinstance(to_kill, list) else to_kill - survive = [node for i, node in enumerate(service.nodes) if i != idx] - kill = service.nodes[idx] + survive = random.choice([node for node in self.servers.nodes if node not in to_kill]) - return kill, survive + return to_kill, survive From 879fa1f53d2f757059868cf21d9c9761ed49de19 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Thu, 13 Aug 2020 11:46:17 +0300 Subject: [PATCH 41/78] Fail fast (#8147) --- .../utils/IgniteAwareApplication.java | 55 +++++++++++++++---- .../services/utils/ignite_aware_app.py | 11 +++- .../ignitetest/tests/utils/ignite_test.py | 2 +- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java index 7a173a5689846..5e610f1eb583a 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java @@ -37,6 +37,9 @@ public abstract class IgniteAwareApplication { /** App finished. */ private static final String APP_FINISHED = "IGNITE_APPLICATION_FINISHED"; + /** App broken. */ + private static final String APP_BROKEN = "IGNITE_APPLICATION_BROKEN"; + /** App terminated. */ private static final String APP_TERMINATED = "IGNITE_APPLICATION_TERMINATED"; @@ -46,9 +49,15 @@ public abstract class IgniteAwareApplication { /** Finished. */ private static volatile boolean finished; + /** Broken. */ + private static volatile boolean broken; + /** Terminated. */ private static volatile boolean terminated; + /** Shutdown hook. */ + private static volatile Thread hook; + /** Ignite. */ protected Ignite ignite; @@ -59,10 +68,15 @@ public abstract class IgniteAwareApplication { * Default constructor. */ protected IgniteAwareApplication() { - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - terminate(); + Runtime.getRuntime().addShutdownHook(hook = new Thread(() -> { + log.info("SIGTERM recorded."); - while (!finished()) { + if (!finished && !broken) + terminate(); + else + log.info("Application already done [finished=" + finished + ", broken=" + broken + "]"); + + while (!finished && !broken) { log.info("Waiting for graceful termnation."); try { @@ -72,8 +86,6 @@ protected IgniteAwareApplication() { e.printStackTrace(); } } - - log.info("SIGTERM recorded."); })); log.info("ShutdownHook registered."); @@ -95,25 +107,44 @@ protected void markInitialized() { */ protected void markFinished() { assert !finished; + assert !broken; log.info(APP_FINISHED); + removeShutdownHook(); + finished = true; } /** * */ - protected void markSyncExecutionComplete() { - markInitialized(); - markFinished(); + private void markBroken() { + assert !finished; + assert !broken; + + log.info(APP_BROKEN); + + removeShutdownHook(); + + broken = true; } /** * */ - private boolean finished() { - return finished; + private void removeShutdownHook() { + Runtime.getRuntime().removeShutdownHook(hook); + + log.info("Shutdown hook removed."); + } + + /** + * + */ + protected void markSyncExecutionComplete() { + markInitialized(); + markFinished(); } /** @@ -173,6 +204,10 @@ public void start(JsonNode jsonNode) { } catch (Throwable th) { log.error("Unexpected Application failure... ", th); + + recordResult("ERROR", th.getMessage()); + + markBroken(); } finally { log.info("Application finished."); diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py index b05f5a16b0838..29e0a2cb55336 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py @@ -49,7 +49,13 @@ def start(self): self.logger.info("Waiting for Ignite aware Application (%s) to start..." % self.java_class_name) self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True) - self.await_event("IGNITE_APPLICATION_INITIALIZED", self.timeout_sec, from_the_beginning=True) + self.await_event("IGNITE_APPLICATION_INITIALIZED\\|IGNITE_APPLICATION_BROKEN", self.timeout_sec, + from_the_beginning=True) + + try: + self.await_event("IGNITE_APPLICATION_INITIALIZED", 1, from_the_beginning=True) + except Exception: + raise Exception("Java application execution failed. %s" % self.extract_result("ERROR")) def start_cmd(self, node): cmd = self.env() @@ -70,7 +76,8 @@ def stop_node(self, node, clean_shutdown=True, timeout_sec=20): assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ (str(node.account), str(self.stop_timeout_sec)) - self.await_event("IGNITE_APPLICATION_FINISHED", from_the_beginning=True, timeout_sec=timeout_sec) + self.await_event("IGNITE_APPLICATION_FINISHED\\|IGNITE_APPLICATION_BROKEN", from_the_beginning=True, + timeout_sec=timeout_sec) def clean_node(self, node): if self.alive(node): diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py index df0d30eceda77..c0c35c7924da4 100644 --- a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py +++ b/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py @@ -34,7 +34,7 @@ def stage(self, msg): Print stage mark. :param msg: Stage mark message. """ - self.logger.info("[TEST_STAGE] " + msg + "...") + self.logger.info("[TEST_STAGE] " + msg) @staticmethod def monotonic(): From 5f3b0f18fa5aa6fa45b8cfb546414cf4c31dd3d0 Mon Sep 17 00:00:00 2001 From: Maksim Timonin Date: Mon, 17 Aug 2020 15:15:54 +0300 Subject: [PATCH 42/78] Ducktape Service Specs (#8154) --- modules/ducktests/tests/MANIFEST.in | 2 +- .../tests/ignitetest/services/ignite.py | 28 +-- .../tests/ignitetest/services/ignite_app.py | 88 +++++++- .../tests/ignitetest/services/spark.py | 30 +-- .../services/utils/control_utility.py | 4 +- .../ignitetest/services/utils/ignite_aware.py | 90 ++------ .../services/utils/ignite_aware_app.py | 143 ------------ .../services/utils/ignite_config.py | 10 +- .../ignitetest/services/utils/ignite_path.py | 8 +- .../services/utils/ignite_persistence.py | 64 ++++++ .../ignitetest/services/utils/ignite_spec.py | 213 ++++++++++++++++++ .../tests/add_node_rebalance_test.py | 4 +- .../tests/cellular_affinity_test.py | 4 +- .../ignitetest/tests/control_utility_test.py | 6 +- .../tests/ignitetest/tests/discovery_test.py | 6 +- .../ignitetest/tests/pme_free_switch_test.py | 4 +- .../tests/ignitetest/tests/smoke_test.py | 21 +- .../ignitetest/{tests => }/utils/__init__.py | 0 .../ignitetest/{tests => }/utils/_mark.py | 2 +- .../{tests => }/utils/ignite_test.py | 0 .../ignitetest/{tests => }/utils/version.py | 0 modules/ducktests/tests/setup.cfg | 30 --- modules/ducktests/tests/setup.py | 31 +-- 23 files changed, 434 insertions(+), 354 deletions(-) delete mode 100644 modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py rename modules/ducktests/tests/ignitetest/{tests => }/utils/__init__.py (100%) rename modules/ducktests/tests/ignitetest/{tests => }/utils/_mark.py (97%) rename modules/ducktests/tests/ignitetest/{tests => }/utils/ignite_test.py (100%) rename modules/ducktests/tests/ignitetest/{tests => }/utils/version.py (100%) delete mode 100644 modules/ducktests/tests/setup.cfg diff --git a/modules/ducktests/tests/MANIFEST.in b/modules/ducktests/tests/MANIFEST.in index edef4ab7becbe..6fcceb37144dd 100644 --- a/modules/ducktests/tests/MANIFEST.in +++ b/modules/ducktests/tests/MANIFEST.in @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -recursive-include ignitetest */templates/* +recursive-include ignitetest **.j2 diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index 1c5a476d7fd66..d0bb6d4005206 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -31,7 +31,7 @@ from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.tests.utils.version import DEV_BRANCH +from ignitetest.utils.version import DEV_BRANCH class IgniteService(IgniteAwareService): @@ -52,9 +52,11 @@ class IgniteService(IgniteAwareService): } # pylint: disable=R0913 - def __init__(self, context, num_nodes, modules=None, client_mode=False, version=DEV_BRANCH, properties="", - jvm_opts=None): - super(IgniteService, self).__init__(context, num_nodes, modules, client_mode, version, properties, jvm_opts) + def __init__(self, context, num_nodes, jvm_opts=None, properties="", client_mode=False, modules=None, + version=DEV_BRANCH): + super(IgniteService, self).__init__(context, num_nodes, properties, + client_mode=client_mode, modules=modules, version=version, + jvm_opts=jvm_opts) # pylint: disable=W0221 def start(self, timeout_sec=180): @@ -65,22 +67,6 @@ def start(self, timeout_sec=180): for node in self.nodes: self.await_node_started(node, timeout_sec) - def start_cmd(self, node): - jvm_opts = self.jvm_options + " " - jvm_opts += "-J-DIGNITE_SUCCESS_FILE=" + IgniteService.PERSISTENT_ROOT + "/success_file " - jvm_opts += "-J-Dlog4j.configDebug=true " - - cmd = "export EXCLUDE_TEST_CLASSES=true; " - cmd += "export IGNITE_LOG_DIR=" + IgniteService.PERSISTENT_ROOT + "; " - cmd += "export USER_LIBS=%s; " % ":".join(self.user_libs) - cmd += "%s %s %s 1>> %s 2>> %s &" % \ - (self.path.script("ignite.sh"), - jvm_opts, - IgniteService.CONFIG_FILE, - IgniteService.STDOUT_STDERR_CAPTURE, - IgniteService.STDOUT_STDERR_CAPTURE) - return cmd - def await_node_started(self, node, timeout_sec): """ Await topology ready event on node start. @@ -162,7 +148,7 @@ def __stop_node(node, pid, sig, start_waiter=None, delay_ms=0, time_holder=None) def clean_node(self, node): node.account.kill_java_processes(self.APP_SERVICE_CLASS, clean_shutdown=False, allow_fail=True) - node.account.ssh("sudo rm -rf -- %s" % IgniteService.PERSISTENT_ROOT, allow_fail=False) + node.account.ssh("sudo rm -rf -- %s" % self.PERSISTENT_ROOT, allow_fail=False) def thread_dump(self, node): """ diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index ef5120e9e3e3b..8d60dee4837ad 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -14,20 +14,90 @@ # limitations under the License. """ -This module contains the ignite application service allows to perform custom logic writen on java. +This module contains the base class to build Ignite aware application written on java. """ -from ignitetest.services.utils.ignite_aware_app import IgniteAwareApplicationService -from ignitetest.tests.utils.version import DEV_BRANCH +import re +from ignitetest.services.utils.ignite_aware import IgniteAwareService +from ignitetest.utils.version import DEV_BRANCH -class IgniteApplicationService(IgniteAwareApplicationService): + +class IgniteApplicationService(IgniteAwareService): """ - The Ignite application service allows to perform custom logic writen on java. + The base class to build Ignite aware application written on java. """ + SERVICE_JAVA_CLASS_NAME = "org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService" + # pylint: disable=R0913 - def __init__(self, context, java_class_name, modules=None, client_mode=True, version=DEV_BRANCH, - properties="", params="", jvm_options=None, timeout_sec=60): - super(IgniteApplicationService, self).__init__(context, java_class_name, modules, client_mode, version, - properties, params, jvm_options, timeout_sec, start_ignite=True) + def __init__(self, context, java_class_name, params="", properties="", timeout_sec=60, modules=None, + client_mode=True, version=DEV_BRANCH, servicejava_class_name=SERVICE_JAVA_CLASS_NAME, + jvm_opts=None, start_ignite=True): + super(IgniteApplicationService, self).__init__(context, 1, properties, + client_mode=client_mode, + version=version, + modules=modules, + servicejava_class_name=servicejava_class_name, + java_class_name=java_class_name, + params=params, + jvm_opts=jvm_opts, + start_ignite=start_ignite) + + self.servicejava_class_name = servicejava_class_name + self.java_class_name = java_class_name + self.timeout_sec = timeout_sec + self.stop_timeout_sec = 10 + + def start(self): + super(IgniteApplicationService, self).start() + + self.logger.info("Waiting for Ignite aware Application (%s) to start..." % self.java_class_name) + + self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True) + self.await_event("IGNITE_APPLICATION_INITIALIZED\\|IGNITE_APPLICATION_BROKEN", self.timeout_sec, + from_the_beginning=True) + + try: + self.await_event("IGNITE_APPLICATION_INITIALIZED", 1, from_the_beginning=True) + except Exception: + raise Exception("Java application execution failed. %s" % self.extract_result("ERROR")) + + # pylint: disable=W0221 + def stop_node(self, node, clean_shutdown=True, timeout_sec=20): + self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) + node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=clean_shutdown, allow_fail=True) + + stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec) + assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ + (str(node.account), str(self.stop_timeout_sec)) + + self.await_event("IGNITE_APPLICATION_FINISHED\\|IGNITE_APPLICATION_BROKEN", from_the_beginning=True, + timeout_sec=timeout_sec) + + def clean_node(self, node): + if self.alive(node): + self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." % + (self.__class__.__name__, node.account)) + + node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=False, allow_fail=True) + + node.account.ssh("rm -rf %s" % self.PERSISTENT_ROOT, allow_fail=False) + + def pids(self, node): + return node.account.java_pids(self.servicejava_class_name) + + def extract_result(self, name): + """ + :param name: Result parameter's name. + :return: Extracted result of application run. + """ + res = "" + + output = self.nodes[0].account.ssh_capture( + "grep '%s' %s" % (name + "->", self.STDOUT_STDERR_CAPTURE), allow_fail=False) + + for line in output: + res = re.search("%s(.*)%s" % (name + "->", "<-"), line).group(1) + + return res diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py index 72ce9bc9e3e40..a7cb65eabaf3f 100644 --- a/modules/ducktests/tests/ignitetest/services/spark.py +++ b/modules/ducktests/tests/ignitetest/services/spark.py @@ -20,13 +20,11 @@ import os.path from ducktape.cluster.remoteaccount import RemoteCommandError -from ducktape.services.service import Service +from ducktape.services.background_thread import BackgroundThreadService +from ignitetest.services.utils.ignite_persistence import PersistenceAware -from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.tests.utils.version import DEV_BRANCH - -class SparkService(IgniteAwareService): +class SparkService(BackgroundThreadService, PersistenceAware): """ Start a spark node. """ @@ -36,15 +34,12 @@ class SparkService(IgniteAwareService): logs = {} # pylint: disable=R0913 - def __init__(self, context, modules=None, version=DEV_BRANCH, num_nodes=3, properties=""): + def __init__(self, context, num_nodes=3): """ :param context: test context :param num_nodes: number of Ignite nodes. """ - modules = modules or [] - modules.extend(["ignite-spark"]) - - IgniteAwareService.__init__(self, context, num_nodes, modules, False, version, properties, None) + super(SparkService, self).__init__(context, num_nodes) self.log_level = "DEBUG" @@ -59,11 +54,14 @@ def __init__(self, context, modules=None, version=DEV_BRANCH, num_nodes=3, prope } def start(self): - Service.start(self) + BackgroundThreadService.start(self) self.logger.info("Waiting for Spark to start...") def start_cmd(self, node): + """ + Prepare command to start Spark nodes + """ if node == self.nodes[0]: script = "start-master.sh" else: @@ -77,8 +75,7 @@ def start_cmd(self, node): return cmd - # pylint: disable=W0221 - def start_node(self, node, timeout_sec=30): + def start_node(self, node): self.init_persistent(node) cmd = self.start_cmd(node) @@ -93,6 +90,7 @@ def start_node(self, node, timeout_sec=30): self.logger.debug("Monitoring - %s" % log_file) + timeout_sec = 30 with node.account.monitor_log(log_file) as monitor: node.account.ssh(cmd) monitor.wait_until(log_msg, timeout_sec=timeout_sec, backoff_sec=5, @@ -108,11 +106,17 @@ def stop_node(self, node): node.account.ssh(os.path.join(SparkService.INSTALL_DIR, "sbin", "stop-slave.sh")) def clean_node(self, node): + """ + Clean spark persistence files + """ node.account.kill_java_processes(self.java_class_name(node), clean_shutdown=False, allow_fail=True) node.account.ssh("sudo rm -rf -- %s" % SparkService.SPARK_PERSISTENT_ROOT, allow_fail=False) def pids(self, node): + """ + :return: list of service pids on specific node + """ try: cmd = "jcmd | grep -e %s | awk '{print $1}'" % self.java_class_name(node) return list(node.account.ssh_capture(cmd, allow_fail=True, callback=int)) diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index ed91d187987e4..2c1330131eaed 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -140,8 +140,8 @@ def __run(self, cmd): return output def __form_cmd(self, node, cmd): - return self._cluster.path.script("%s --host %s %s" % (self.BASE_COMMAND, node.account.externally_routable_ip, - cmd)) + return self._cluster.spec.path.script("%s --host %s %s" % + (self.BASE_COMMAND, node.account.externally_routable_ip, cmd)) @staticmethod def __parse_output(raw_output): diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 0b5e18f9af196..77e48d700a943 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -17,62 +17,34 @@ This module contains the base class to build services aware of Ignite. """ -import os from abc import abstractmethod, ABCMeta from ducktape.services.background_thread import BackgroundThreadService from ducktape.utils.util import wait_until from six import add_metaclass -from ignitetest.services.utils.ignite_config import IgniteLoggerConfig, IgniteServerConfig, IgniteClientConfig -from ignitetest.services.utils.ignite_path import IgnitePath +from ignitetest.services.utils.ignite_spec import resolve_spec from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin -from ignitetest.tests.utils.version import IgniteVersion +from ignitetest.services.utils.ignite_persistence import IgnitePersistenceAware @add_metaclass(ABCMeta) -class IgniteAwareService(BackgroundThreadService): +class IgniteAwareService(BackgroundThreadService, IgnitePersistenceAware): """ The base class to build services aware of Ignite. """ - # Root directory for persistent output - PERSISTENT_ROOT = "/mnt/service" - STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") - WORK_DIR = os.path.join(PERSISTENT_ROOT, "work") - CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-config.xml") - LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "ignite-log4j.xml") - - logs = { - "console_log": { - "path": STDOUT_STDERR_CAPTURE, - "collect_default": True} - } # pylint: disable=R0913 - def __init__(self, context, num_nodes, modules, client_mode, version, properties, jvm_options): + def __init__(self, context, num_nodes, properties, **kwargs): + """ + **kwargs are params that passed to IgniteSpec + """ super(IgniteAwareService, self).__init__(context, num_nodes) - global_jvm_options = context.globals.get("jvm_opts", "") - - service_jvm_options = " ".join(map(lambda x: '-J' + x, jvm_options)) if jvm_options else "" - - self.jvm_options = " ".join(filter(None, [global_jvm_options, service_jvm_options])) - self.log_level = "DEBUG" self.properties = properties - if isinstance(version, IgniteVersion): - self.version = version - else: - self.version = IgniteVersion(version) - - self.path = IgnitePath(self.version, context) - self.client_mode = client_mode - - libs = modules or [] - libs.extend(["ignite-log4j"]) - - self.user_libs = list(map(lambda m: self.path.module(m) + "/*", libs)) + self.spec = resolve_spec(self, context, **kwargs) def start_node(self, node): self.init_persistent(node) @@ -88,26 +60,15 @@ def init_persistent(self, node): Init persistent directory. :param node: Ignite service node. """ - logger_config = IgniteLoggerConfig().render(work_dir=self.WORK_DIR) - - node.account.mkdirs(self.PERSISTENT_ROOT) + super(IgniteAwareService, self).init_persistent(node) - node_config = self.config().render(config_dir=self.PERSISTENT_ROOT, - work_dir=self.WORK_DIR, - properties=self.properties, - consistent_id=node.account.externally_routable_ip) + node_config = self.spec.config().render(config_dir=self.PERSISTENT_ROOT, + work_dir=self.WORK_DIR, + properties=self.properties, + consistent_id=node.account.externally_routable_ip) setattr(node, "consistent_id", node.account.externally_routable_ip) node.account.create_file(self.CONFIG_FILE, node_config) - node.account.create_file(self.LOG4J_CONFIG_FILE, logger_config) - - @abstractmethod - def start_cmd(self, node): - """ - Start up command for service. - :param node: Ignite service node. - """ - raise NotImplementedError @abstractmethod def pids(self, node): @@ -117,18 +78,9 @@ def pids(self, node): """ raise NotImplementedError - def config(self): - """ - :return: Ignite node configuration. - """ - if self.client_mode: - return IgniteClientConfig(self.context) - - return IgniteServerConfig(self.context) - # pylint: disable=W0613 def _worker(self, idx, node): - cmd = self.start_cmd(node) + cmd = self.spec.command() self.logger.debug("Attempting to start Application Service on %s with command: %s" % (str(node.account), cmd)) @@ -141,7 +93,6 @@ def alive(self, node): """ return len(self.pids(node)) > 0 - # pylint: disable=R0913 def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning=False, backoff_sec=5): """ Await for specific event message in a node's log file. @@ -171,16 +122,3 @@ def await_event(self, evt_message, timeout_sec, from_the_beginning=False, backof for node in self.nodes: self.await_event_on_node(evt_message, node, timeout_sec, from_the_beginning=from_the_beginning, backoff_sec=backoff_sec) - - def execute(self, command): - """ - Execute command on all nodes. - :param command: Command to execute. - """ - for node in self.nodes: - cmd = "%s 1>> %s 2>> %s" % \ - (self.path.script(command), - self.STDOUT_STDERR_CAPTURE, - self.STDOUT_STDERR_CAPTURE) - - node.account.ssh(cmd) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py deleted file mode 100644 index 29e0a2cb55336..0000000000000 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware_app.py +++ /dev/null @@ -1,143 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This module contains the base class to build Ignite aware application written on java. -""" - -import base64 -import json -import re - -from ignitetest.services.utils.ignite_aware import IgniteAwareService - - -class IgniteAwareApplicationService(IgniteAwareService): - """ - The base class to build Ignite aware application written on java. - """ - - # pylint: disable=R0913 - def __init__(self, context, java_class_name, modules, client_mode, version, properties, params, jvm_options, - timeout_sec, start_ignite=False, - service_java_class_name="org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService"): - super(IgniteAwareApplicationService, self).__init__(context, 1, modules, client_mode, version, properties, - jvm_options) - - self.servicejava_class_name = service_java_class_name - self.java_class_name = java_class_name - self.timeout_sec = timeout_sec - self.stop_timeout_sec = 10 - self.params = params - self.start_ignite = start_ignite - - def start(self): - super(IgniteAwareApplicationService, self).start() - - self.logger.info("Waiting for Ignite aware Application (%s) to start..." % self.java_class_name) - - self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True) - self.await_event("IGNITE_APPLICATION_INITIALIZED\\|IGNITE_APPLICATION_BROKEN", self.timeout_sec, - from_the_beginning=True) - - try: - self.await_event("IGNITE_APPLICATION_INITIALIZED", 1, from_the_beginning=True) - except Exception: - raise Exception("Java application execution failed. %s" % self.extract_result("ERROR")) - - def start_cmd(self, node): - cmd = self.env() - cmd += "%s %s %s 1>> %s 2>> %s &" % \ - (self.path.script("ignite.sh"), - self.jvm_opts(), - self.app_args(), - self.STDOUT_STDERR_CAPTURE, - self.STDOUT_STDERR_CAPTURE) - return cmd - - # pylint: disable=W0221 - def stop_node(self, node, clean_shutdown=True, timeout_sec=20): - self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) - node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=clean_shutdown, allow_fail=True) - - stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec) - assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ - (str(node.account), str(self.stop_timeout_sec)) - - self.await_event("IGNITE_APPLICATION_FINISHED\\|IGNITE_APPLICATION_BROKEN", from_the_beginning=True, - timeout_sec=timeout_sec) - - def clean_node(self, node): - if self.alive(node): - self.logger.warn("%s %s was still alive at cleanup time. Killing forcefully..." % - (self.__class__.__name__, node.account)) - - node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=False, allow_fail=True) - - node.account.ssh("rm -rf %s" % self.PERSISTENT_ROOT, allow_fail=False) - - def app_args(self): - """ - :return: Application arguments. - """ - args = ",".join([str(self.start_ignite), self.java_class_name, IgniteAwareApplicationService.CONFIG_FILE]) - - if self.params != "": - args = ",".join([args, str(base64.b64encode(json.dumps(self.params).encode("UTF-8")))]) - - return args - - def pids(self, node): - return node.account.java_pids(self.servicejava_class_name) - - def jvm_opts(self): - """ - :return: Application JVM options. - """ - return "-J-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file " + \ - "-J-Dlog4j.configDebug=true " \ - "-J-Xmx1G " \ - "-J-ea " \ - "-J-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false " + self.jvm_options - - def env(self): - """ - :return: Export string of additional environment variables. - """ - if not self.version.is_dev: - # Jackson requred to parse application params at java side. Release's version should be used. - for line in self.nodes[0].account.ssh_capture( - "ls -d %s/libs/optional/ignite-aws/* | grep jackson | tr '\n' ':' | sed 's/.$//'" % self.path.home): - self.user_libs.extend([line]) - - return "export MAIN_CLASS={main_class}; ".format(main_class=self.servicejava_class_name) + \ - "export EXCLUDE_TEST_CLASSES=true; " + \ - "export IGNITE_LOG_DIR={log_dir}; ".format(log_dir=self.PERSISTENT_ROOT) + \ - "export USER_LIBS=%s:/opt/ignite-dev/modules/ducktests/target/*; " % (":".join(self.user_libs)) - - def extract_result(self, name): - """ - :param name: Result parameter's name. - :return: Extracted result of application run. - """ - res = "" - - output = self.nodes[0].account.ssh_capture( - "grep '%s' %s" % (name + "->", self.STDOUT_STDERR_CAPTURE), allow_fail=False) - - for line in output: - res = re.search("%s(.*)%s" % (name + "->", "<-"), line).group(1) - - return res diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py index 90b3bfa970754..a2bfe3dc026d2 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py @@ -51,10 +51,7 @@ class IgniteServerConfig(Config): """ Ignite server node configuration. """ - def __init__(self, context): - path = DEFAULT_IGNITE_CONF - if 'ignite_server_config_path' in context.globals: - path = context.globals['ignite_server_config_path'] + def __init__(self, path=DEFAULT_IGNITE_CONF): super(IgniteServerConfig, self).__init__(path) @@ -62,10 +59,7 @@ class IgniteClientConfig(Config): """ Ignite client node configuration. """ - def __init__(self, context): - path = DEFAULT_IGNITE_CONF - if 'ignite_client_config_path' in context.globals: - path = context.globals['ignite_client_config_path'] + def __init__(self, path=DEFAULT_IGNITE_CONF): super(IgniteClientConfig, self).__init__(path) self.default_params.update(client_mode=True) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py index 2177e9bd785de..4089112533f98 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_path.py @@ -31,11 +31,9 @@ class IgnitePath: SCRATCH_ROOT = "/mnt" IGNITE_INSTALL_ROOT = "/opt" - def __init__(self, version, context): + def __init__(self, version, project="ignite"): self.version = version - self.project = context.globals.get("project", "ignite") - - home_dir = "%s-%s" % (self.project, str(self.version)) + home_dir = "%s-%s" % (project, str(self.version)) self.home = os.path.join(IgnitePath.IGNITE_INSTALL_ROOT, home_dir) def module(self, module_name): @@ -46,7 +44,7 @@ def module(self, module_name): if self.version.is_dev: module_path = os.path.join("modules", module_name, "target") else: - module_path = os.path.join("libs", "optional", module_name) + module_path = os.path.join("libs", "optional", "ignite-%s" % module_name) return os.path.join(self.home, module_path) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py new file mode 100644 index 0000000000000..9c79b2e5b8cd4 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains classes that represent persistent artifacts of tests +""" + +import os + +from ignitetest.services.utils.ignite_config import IgniteLoggerConfig + + +class PersistenceAware(object): + """ + This class contains basic persistence artifacts + """ + # Root directory for persistent output + PERSISTENT_ROOT = "/mnt/service" + STDOUT_STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "console.log") + + logs = { + "console_log": { + "path": STDOUT_STDERR_CAPTURE, + "collect_default": True + } + } + + def init_persistent(self, node): + """ + Init persistent directory. + :param node: Service node. + """ + node.account.mkdirs(self.PERSISTENT_ROOT) + + +class IgnitePersistenceAware(PersistenceAware): + """ + This class contains Ignite persistence artifacts + """ + WORK_DIR = os.path.join(PersistenceAware.PERSISTENT_ROOT, "work") + CONFIG_FILE = os.path.join(PersistenceAware.PERSISTENT_ROOT, "ignite-config.xml") + LOG4J_CONFIG_FILE = os.path.join(PersistenceAware.PERSISTENT_ROOT, "ignite-log4j.xml") + + def init_persistent(self, node): + """ + Init persistent directory. + :param node: Ignite service node. + """ + super(IgnitePersistenceAware, self).init_persistent(node) + + logger_config = IgniteLoggerConfig().render(work_dir=self.WORK_DIR) + node.account.create_file(self.LOG4J_CONFIG_FILE, logger_config) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py new file mode 100644 index 0000000000000..32468a93460b0 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -0,0 +1,213 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains Spec classes that describes config and command line to start Ignite services +""" + +import base64 +import importlib +import json + +from ignitetest.services.utils.ignite_path import IgnitePath +from ignitetest.services.utils.ignite_config import IgniteClientConfig, IgniteServerConfig +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + +from ignitetest.services.utils.ignite_persistence import IgnitePersistenceAware + + +# pylint: disable=no-else-return +def resolve_spec(service, context, **kwargs): + """ + Resolve Spec classes for IgniteService and IgniteApplicationService + """ + def _resolve_spec(name, default): + if name in context.globals: + fqdn = context.globals[name] + (module, clazz) = fqdn.rsplit('.', 1) + module = importlib.import_module(module) + return getattr(module, clazz) + return default + + def is_impl(impl): + classes = map(lambda s: s.__name__, service.__class__.mro()) + impl_filter = list(filter(lambda c: c == impl, classes)) + return len(impl_filter) > 0 + + if is_impl("IgniteService"): + return _resolve_spec("NodeSpec", ApacheIgniteNodeSpec)(**kwargs) + elif is_impl("IgniteApplicationService"): + return _resolve_spec("AppSpec", ApacheIgniteApplicationSpec)(context=context, **kwargs) + else: + raise Exception("There is no specification for class %s" % type(service)) + + +class IgniteSpec(object): + """ + This class is a basic Spec + """ + def __init__(self, version, project, client_mode, jvm_opts): + if isinstance(version, IgniteVersion): + self.version = version + else: + self.version = IgniteVersion(version) + + self.path = IgnitePath(self.version, project) + self.envs = {} + self.jvm_opts = jvm_opts or [] + self.client_mode = client_mode + + def config(self): + """ + :return: config that service will use to start on a node + """ + if self.client_mode: + return IgniteClientConfig() + return IgniteServerConfig() + + def command(self): + """ + :return: string that represents command to run service on a node + """ + raise NotImplementedError() + + def _envs(self): + """ + :return: line with exports env variables: export A=B; export C=D; + """ + exports = ["export %s=%s" % (key, self.envs[key]) for key in self.envs] + return "; ".join(exports) + ";" + + def _jvm_opts(self): + """ + :return: line with extra JVM params for ignite.sh script: -J-Dparam=value -J-ea + """ + opts = ["-J%s" % o for o in self.jvm_opts] + return " ".join(opts) + + +class IgniteNodeSpec(IgniteSpec, IgnitePersistenceAware): + """ + Spec to run ignite node + """ + def __init__(self, **kwargs): + IgniteSpec.__init__(self, **kwargs) + + def command(self): + cmd = "%s %s %s %s 1>> %s 2>> %s &" % \ + (self._envs(), + self.path.script("ignite.sh"), + self._jvm_opts(), + self.CONFIG_FILE, + self.STDOUT_STDERR_CAPTURE, + self.STDOUT_STDERR_CAPTURE) + + return cmd + + +class IgniteApplicationSpec(IgniteSpec, IgnitePersistenceAware): + """ + Spec to run ignite application + """ + def __init__(self, **kwargs): + super(IgniteApplicationSpec, self).__init__(**kwargs) + self.args = "" + + def _app_args(self): + return ",".join(self.args) + + def command(self): + cmd = "%s %s %s %s 1>> %s 2>> %s &" % \ + (self._envs(), + self.path.script("ignite.sh"), + self._jvm_opts(), + self._app_args(), + self.STDOUT_STDERR_CAPTURE, + self.STDOUT_STDERR_CAPTURE) + + return cmd + + +### + + +class ApacheIgniteNodeSpec(IgniteNodeSpec, IgnitePersistenceAware): + """ + Implementation IgniteNodeSpec for Apache Ignite project + """ + def __init__(self, modules, **kwargs): + super(ApacheIgniteNodeSpec, self).__init__(project="ignite", **kwargs) + + libs = (modules or []) + libs.append("log4j") + libs = list(map(lambda m: self.path.module(m) + "/*", libs)) + + self.envs = { + 'EXCLUDE_TEST_CLASSES': 'true', + 'IGNITE_LOG_DIR': self.PERSISTENT_ROOT, + 'USER_LIBS': ":".join(libs) + } + + self.jvm_opts.extend([ + "-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file", + "-Dlog4j.configDebug=true" + ]) + + +class ApacheIgniteApplicationSpec(IgniteApplicationSpec, IgnitePersistenceAware): + """ + Implementation IgniteApplicationSpec for Apache Ignite project + """ + # pylint: disable=too-many-arguments + def __init__(self, context, modules, servicejava_class_name, java_class_name, params, start_ignite, **kwargs): + super(ApacheIgniteApplicationSpec, self).__init__(project="ignite", **kwargs) + self.context = context + + libs = modules or [] + libs.extend(["log4j"]) + + libs = [self.path.module(m) + "/*" for m in libs] + libs.append(IgnitePath(DEV_BRANCH).module("ducktests") + "/*") + libs.extend(self.__jackson()) + + self.envs = { + "MAIN_CLASS": servicejava_class_name, + "EXCLUDE_TEST_CLASSES": "true", + "IGNITE_LOG_DIR": self.PERSISTENT_ROOT, + "USER_LIBS": ":".join(libs) + } + + self.jvm_opts.extend([ + "-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file ", + "-Dlog4j.configDebug=true", + "-Xmx1G", + "-ea", + "-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false" + ]) + + self.args = [ + str(start_ignite), + java_class_name, + self.CONFIG_FILE, + str(base64.b64encode(json.dumps(params).encode("UTF-8"))) + ] + + def __jackson(self): + if not self.version.is_dev: + aws = self.path.module("aws") + return self.context.cluster.nodes[0].account.ssh_capture( + "ls -d %s/* | grep jackson | tr '\n' ':' | sed 's/.$//'" % aws) + else: + return [] diff --git a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py index 28fc4fb626280..0eebea8f3d0b8 100644 --- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py @@ -22,8 +22,8 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService -from ignitetest.tests.utils.ignite_test import IgniteTest -from ignitetest.tests.utils.version import DEV_BRANCH, IgniteVersion, LATEST +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion, LATEST # pylint: disable=W0223 diff --git a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py index 80870122b9a22..718d5daedc036 100644 --- a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py +++ b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py @@ -23,8 +23,8 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService -from ignitetest.tests.utils.ignite_test import IgniteTest -from ignitetest.tests.utils.version import DEV_BRANCH +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH # pylint: disable=W0223 diff --git a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py index a1b00dee61757..f2c1a4c430465 100644 --- a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py +++ b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py @@ -23,9 +23,9 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.utils.control_utility import ControlUtility, ControlUtilityError -from ignitetest.tests.utils import version_if -from ignitetest.tests.utils.ignite_test import IgniteTest -from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_8, IgniteVersion, LATEST_2_7, V_2_8_0 +from ignitetest.utils import version_if +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, LATEST_2_8, IgniteVersion, LATEST_2_7, V_2_8_0 # pylint: disable=W0223 diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 8d933966d6fd5..ee94d2c177ede 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -29,8 +29,8 @@ from ignitetest.services.utils.ignite_aware import IgniteAwareService from ignitetest.services.utils.time_utils import epoch_mills from ignitetest.services.zk.zookeeper import ZookeeperService -from ignitetest.tests.utils.ignite_test import IgniteTest -from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7 +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, LATEST_2_7 # pylint: disable=W0223 @@ -165,7 +165,7 @@ def __simulate_nodes_failure(self, version, properties, nodes_to_kill=1): self.servers = IgniteService( self.test_context, num_nodes=self.NUM_NODES, - modules=["ignite-zookeeper"], + modules=["zookeeper"], properties=properties, version=version) diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index 108fa9d96fa95..774ab41edde5e 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -25,8 +25,8 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.utils.control_utility import ControlUtility -from ignitetest.tests.utils.ignite_test import IgniteTest -from ignitetest.tests.utils.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion # pylint: disable=W0223 diff --git a/modules/ducktests/tests/ignitetest/tests/smoke_test.py b/modules/ducktests/tests/ignitetest/tests/smoke_test.py index a52f38246791f..a8dfc87b73340 100644 --- a/modules/ducktests/tests/ignitetest/tests/smoke_test.py +++ b/modules/ducktests/tests/ignitetest/tests/smoke_test.py @@ -23,8 +23,8 @@ from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.spark import SparkService from ignitetest.services.zk.zookeeper import ZookeeperService -from ignitetest.tests.utils.ignite_test import IgniteTest -from ignitetest.tests.utils.version import DEV_BRANCH +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH # pylint: disable=W0223 @@ -41,6 +41,18 @@ def setUp(self): def teardown(self): pass + @parametrize(version=str(DEV_BRANCH)) + def test_ignite_start_stop(self, version): + """ + Test that IgniteService correctly start and stop + """ + ignite = IgniteService( + self.test_context, + num_nodes=1, + version=version) + ignite.start() + ignite.stop() + @parametrize(version=str(DEV_BRANCH)) def test_ignite_app_start_stop(self, version): """ @@ -61,12 +73,11 @@ def test_ignite_app_start_stop(self, version): app.stop() ignite.stop() - @parametrize(version=str(DEV_BRANCH)) - def test_spark_start_stop(self, version): + def test_spark_start_stop(self): """ Test that SparkService correctly start and stop """ - spark = SparkService(self.test_context, version=version, num_nodes=2) + spark = SparkService(self.test_context, num_nodes=2) spark.start() spark.stop() diff --git a/modules/ducktests/tests/ignitetest/tests/utils/__init__.py b/modules/ducktests/tests/ignitetest/utils/__init__.py similarity index 100% rename from modules/ducktests/tests/ignitetest/tests/utils/__init__.py rename to modules/ducktests/tests/ignitetest/utils/__init__.py diff --git a/modules/ducktests/tests/ignitetest/tests/utils/_mark.py b/modules/ducktests/tests/ignitetest/utils/_mark.py similarity index 97% rename from modules/ducktests/tests/ignitetest/tests/utils/_mark.py rename to modules/ducktests/tests/ignitetest/utils/_mark.py index d5ea8cf8b2a73..c4b2b35eecab2 100644 --- a/modules/ducktests/tests/ignitetest/tests/utils/_mark.py +++ b/modules/ducktests/tests/ignitetest/utils/_mark.py @@ -19,7 +19,7 @@ import six from ducktape.mark._mark import Ignore, Mark -from ignitetest.tests.utils.version import IgniteVersion +from ignitetest.utils.version import IgniteVersion class VersionIf(Ignore): diff --git a/modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/utils/ignite_test.py similarity index 100% rename from modules/ducktests/tests/ignitetest/tests/utils/ignite_test.py rename to modules/ducktests/tests/ignitetest/utils/ignite_test.py diff --git a/modules/ducktests/tests/ignitetest/tests/utils/version.py b/modules/ducktests/tests/ignitetest/utils/version.py similarity index 100% rename from modules/ducktests/tests/ignitetest/tests/utils/version.py rename to modules/ducktests/tests/ignitetest/utils/version.py diff --git a/modules/ducktests/tests/setup.cfg b/modules/ducktests/tests/setup.cfg deleted file mode 100644 index 974d5bb9a972f..0000000000000 --- a/modules/ducktests/tests/setup.cfg +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pytest configuration (can also be defined in tox.ini or pytest.ini file) -# -# This file defines naming convention and root search directory for autodiscovery of -# pytest unit tests for the system test service classes. -# -# To ease possible confusion, 'check' instead of 'test' as a prefix for unit tests, since -# many system test files, classes, and methods have 'test' somewhere in the name -[pytest] -testpaths=unit -python_files=check_*.py -python_classes=Check -python_functions=check_* - -# don't search inside any resources directory for unit tests -norecursedirs = resources diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index e82bdfb6b8f52..8ad70b02b78ce 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -14,36 +14,13 @@ # limitations under the License. import re -import sys - from setuptools import find_packages, setup -from setuptools.command.test import test as TestCommand -version = '' + with open('ignitetest/__init__.py', 'r') as fd: version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) -class PyTest(TestCommand): - user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] - - def initialize_options(self): - TestCommand.initialize_options(self) - self.pytest_args = [] - - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True - - def run_tests(self): - # import here, cause outside the eggs aren't loaded - import pytest - print(self.pytest_args) - errno = pytest.main(self.pytest_args) - sys.exit(errno) - - # Note: when changing the version of ducktape, also revise tests/docker/Dockerfile setup(name="ignitetest", version=version, @@ -51,9 +28,7 @@ def run_tests(self): author="Apache Ignite", platforms=["any"], license="apache2.0", - packages=find_packages(), + packages=find_packages(exclude=["ignitetest.tests", "ignitetest.tests.*"]), include_package_data=True, - install_requires=["ducktape==0.7.8", "requests==2.20.0"], - tests_require=["pytest", "mock", "monotonic"], - cmdclass={'test': PyTest} + install_requires=["ducktape==0.7.8", "requests==2.22.0", "monotonic==1.5"] ) From f839a29aaf105b9742002a16de85eeb42244acae Mon Sep 17 00:00:00 2001 From: Maksim Timonin Date: Wed, 19 Aug 2020 17:22:10 +0300 Subject: [PATCH 43/78] fix: collect logs for IgniteAwareService (#8171) --- modules/ducktests/tests/ignitetest/services/ignite.py | 10 ---------- .../tests/ignitetest/services/utils/ignite_aware.py | 4 ++++ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index d0bb6d4005206..2c85528499196 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -41,16 +41,6 @@ class IgniteService(IgniteAwareService): APP_SERVICE_CLASS = "org.apache.ignite.startup.cmdline.CommandLineStartup" HEAP_DUMP_FILE = os.path.join(IgniteAwareService.PERSISTENT_ROOT, "ignite-heap.bin") - logs = { - "console_log": { - "path": IgniteAwareService.STDOUT_STDERR_CAPTURE, - "collect_default": True}, - - "heap_dump": { - "path": HEAP_DUMP_FILE, - "collect_default": False} - } - # pylint: disable=R0913 def __init__(self, context, num_nodes, jvm_opts=None, properties="", client_mode=False, modules=None, version=DEV_BRANCH): diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 77e48d700a943..7a608bc56c56d 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -41,7 +41,11 @@ def __init__(self, context, num_nodes, properties, **kwargs): """ super(IgniteAwareService, self).__init__(context, num_nodes) + # Ducktape checks a Service implementation attribute 'logs' to get config for logging. + # IgniteAwareService uses IgnitePersistenceAware mixin to override default Service 'log' definition. self.log_level = "DEBUG" + self.logs = IgnitePersistenceAware.logs + self.properties = properties self.spec = resolve_spec(self, context, **kwargs) From 3b9c6ac0067f8f6ca2b7e92fb7c362ff66f65b7e Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Mon, 24 Aug 2020 11:43:26 +0300 Subject: [PATCH 44/78] Set version of pylint (#8181) --- modules/ducktests/tests/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ducktests/tests/tox.ini b/modules/ducktests/tests/tox.ini index 8a7235b6fe11a..60e7012ac43d4 100644 --- a/modules/ducktests/tests/tox.ini +++ b/modules/ducktests/tests/tox.ini @@ -28,7 +28,7 @@ deps = monotonic mock pytest - pylint + pylint==2.5.3 [testenv:linter] basepython = python3.8 From f707e4398a677b88e17bacaa8922ba084f7f3481 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Mon, 24 Aug 2020 20:41:56 +0300 Subject: [PATCH 45/78] Ignite ducktape py3 (#8155) * Migrate to python 3 * small fixes * Remove unnecessary python2 compatibility stuff * Remove object inheritance * python3 fix * Migrate to modern super() and refactor some more. * Fix some minors. Set pylint specific version (2.6.0) --- modules/ducktests/tests/docker/Dockerfile | 7 +++--- .../ducktests/tests/docker/requirements.txt | 18 +++++++++++++++ .../tests/ignitetest/services/ignite.py | 11 +++++----- .../tests/ignitetest/services/ignite_app.py | 22 +++++++++---------- .../tests/ignitetest/services/spark.py | 2 +- .../ignitetest/services/utils/concurrent.py | 2 +- .../services/utils/config/ignite.xml.j2 | 4 +++- .../services/utils/control_utility.py | 2 +- .../ignitetest/services/utils/ignite_aware.py | 11 ++++------ .../services/utils/ignite_config.py | 8 +++---- .../services/utils/ignite_persistence.py | 10 +++++++-- .../ignitetest/services/utils/ignite_spec.py | 12 +++++----- .../ignitetest/services/utils/jmx_utils.py | 8 +++---- .../tests/ignitetest/services/zk/zookeeper.py | 4 ++-- .../tests/add_node_rebalance_test.py | 3 --- .../tests/cellular_affinity_test.py | 3 --- .../ignitetest/tests/control_utility_test.py | 2 +- .../tests/ignitetest/tests/discovery_test.py | 5 +++-- .../ignitetest/tests/pme_free_switch_test.py | 3 --- .../tests/ignitetest/tests/smoke_test.py | 2 -- .../ducktests/tests/ignitetest/utils/_mark.py | 7 +++--- .../tests/ignitetest/utils/ignite_test.py | 4 ++-- .../tests/ignitetest/utils/version.py | 6 ++--- modules/ducktests/tests/setup.py | 5 ++++- modules/ducktests/tests/tox.ini | 10 ++------- 25 files changed, 88 insertions(+), 83 deletions(-) create mode 100644 modules/ducktests/tests/docker/requirements.txt diff --git a/modules/ducktests/tests/docker/Dockerfile b/modules/ducktests/tests/docker/Dockerfile index a89ea744379ad..9467da73e9654 100644 --- a/modules/ducktests/tests/docker/Dockerfile +++ b/modules/ducktests/tests/docker/Dockerfile @@ -34,9 +34,10 @@ LABEL ducker.creator=$ducker_creator # Update Linux and install necessary utilities. RUN cat /etc/apt/sources.list | sed 's/http:\/\/deb.debian.org/https:\/\/deb.debian.org/g' > /etc/apt/sources.list.2 && mv /etc/apt/sources.list.2 /etc/apt/sources.list -RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python-pip python-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute mc && apt-get -y clean -RUN python -m pip install -U pip==9.0.3; -RUN pip install --upgrade cffi virtualenv pyasn1 boto3 pycrypto pywinrm ipaddress enum34 monotonic && pip install --upgrade ducktape==0.7.8 +RUN apt update && apt install -y sudo netcat iptables rsync unzip wget curl jq coreutils openssh-server net-tools vim python3-pip python3-dev libffi-dev libssl-dev cmake pkg-config libfuse-dev iperf traceroute mc && apt-get -y clean +RUN python3 -m pip install -U pip==20.2.2; +COPY ./requirements.txt /root/requirements.txt +RUN pip3 install -r /root/requirements.txt # Set up ssh COPY ./ssh-config /root/.ssh/config diff --git a/modules/ducktests/tests/docker/requirements.txt b/modules/ducktests/tests/docker/requirements.txt new file mode 100644 index 0000000000000..137f8040c9115 --- /dev/null +++ b/modules/ducktests/tests/docker/requirements.txt @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pytest==6.0.1 +mock==4.0.2 +git+https://github.com/confluentinc/ducktape diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index 2c85528499196..2fb64b3b43578 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -25,7 +25,7 @@ from datetime import datetime from threading import Thread -import monotonic +from time import monotonic from ducktape.cluster.remoteaccount import RemoteCommandError from ducktape.utils.util import wait_until @@ -44,13 +44,12 @@ class IgniteService(IgniteAwareService): # pylint: disable=R0913 def __init__(self, context, num_nodes, jvm_opts=None, properties="", client_mode=False, modules=None, version=DEV_BRANCH): - super(IgniteService, self).__init__(context, num_nodes, properties, - client_mode=client_mode, modules=modules, version=version, - jvm_opts=jvm_opts) + super().__init__(context, num_nodes, properties, client_mode=client_mode, modules=modules, version=version, + jvm_opts=jvm_opts) # pylint: disable=W0221 def start(self, timeout_sec=180): - super(IgniteService, self).start() + super().start() self.logger.info("Waiting for Ignite(s) to start...") @@ -129,7 +128,7 @@ def __stop_node(node, pid, sig, start_waiter=None, delay_ms=0, time_holder=None) time.sleep(delay_ms/1000.0) if time_holder: - mono = monotonic.monotonic() + mono = monotonic() timestamp = datetime.now() time_holder.compare_and_set(None, (mono, timestamp)) diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 8d60dee4837ad..84871d80fa768 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -34,15 +34,15 @@ class IgniteApplicationService(IgniteAwareService): def __init__(self, context, java_class_name, params="", properties="", timeout_sec=60, modules=None, client_mode=True, version=DEV_BRANCH, servicejava_class_name=SERVICE_JAVA_CLASS_NAME, jvm_opts=None, start_ignite=True): - super(IgniteApplicationService, self).__init__(context, 1, properties, - client_mode=client_mode, - version=version, - modules=modules, - servicejava_class_name=servicejava_class_name, - java_class_name=java_class_name, - params=params, - jvm_opts=jvm_opts, - start_ignite=start_ignite) + super().__init__(context, 1, properties, + client_mode=client_mode, + version=version, + modules=modules, + servicejava_class_name=servicejava_class_name, + java_class_name=java_class_name, + params=params, + jvm_opts=jvm_opts, + start_ignite=start_ignite) self.servicejava_class_name = servicejava_class_name self.java_class_name = java_class_name @@ -50,7 +50,7 @@ def __init__(self, context, java_class_name, params="", properties="", timeout_s self.stop_timeout_sec = 10 def start(self): - super(IgniteApplicationService, self).start() + super().start() self.logger.info("Waiting for Ignite aware Application (%s) to start..." % self.java_class_name) @@ -61,7 +61,7 @@ def start(self): try: self.await_event("IGNITE_APPLICATION_INITIALIZED", 1, from_the_beginning=True) except Exception: - raise Exception("Java application execution failed. %s" % self.extract_result("ERROR")) + raise Exception("Java application execution failed. %s" % self.extract_result("ERROR")) from None # pylint: disable=W0221 def stop_node(self, node, clean_shutdown=True, timeout_sec=20): diff --git a/modules/ducktests/tests/ignitetest/services/spark.py b/modules/ducktests/tests/ignitetest/services/spark.py index a7cb65eabaf3f..16d90e1dbfff4 100644 --- a/modules/ducktests/tests/ignitetest/services/spark.py +++ b/modules/ducktests/tests/ignitetest/services/spark.py @@ -39,7 +39,7 @@ def __init__(self, context, num_nodes=3): :param context: test context :param num_nodes: number of Ignite nodes. """ - super(SparkService, self).__init__(context, num_nodes) + super().__init__(context, num_nodes) self.log_level = "DEBUG" diff --git a/modules/ducktests/tests/ignitetest/services/utils/concurrent.py b/modules/ducktests/tests/ignitetest/services/utils/concurrent.py index 57768bcc53371..99292fdfdaddb 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/concurrent.py +++ b/modules/ducktests/tests/ignitetest/services/utils/concurrent.py @@ -20,7 +20,7 @@ import threading -class CountDownLatch(object): +class CountDownLatch: """ A count-down latch. """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 index 50d2c5cdc920c..f03ba1769d215 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 @@ -33,6 +33,8 @@ {% if consistent_id %} {% endif %} - {{ properties }} + {% if properties %} + {{ properties }} + {% endif %} diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index 2c1330131eaed..4b194e68bda68 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -168,4 +168,4 @@ class ControlUtilityError(RemoteCommandError): Error is raised when control utility failed. """ def __init__(self, account, cmd, exit_status, output): - super(ControlUtilityError, self).__init__(account, cmd, exit_status, "".join(output)) + super().__init__(account, cmd, exit_status, "".join(output)) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 7a608bc56c56d..9f374bb5fd6f1 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -21,15 +21,13 @@ from ducktape.services.background_thread import BackgroundThreadService from ducktape.utils.util import wait_until -from six import add_metaclass from ignitetest.services.utils.ignite_spec import resolve_spec from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin from ignitetest.services.utils.ignite_persistence import IgnitePersistenceAware -@add_metaclass(ABCMeta) -class IgniteAwareService(BackgroundThreadService, IgnitePersistenceAware): +class IgniteAwareService(BackgroundThreadService, IgnitePersistenceAware, metaclass=ABCMeta): """ The base class to build services aware of Ignite. """ @@ -39,12 +37,11 @@ def __init__(self, context, num_nodes, properties, **kwargs): """ **kwargs are params that passed to IgniteSpec """ - super(IgniteAwareService, self).__init__(context, num_nodes) + super().__init__(context, num_nodes) # Ducktape checks a Service implementation attribute 'logs' to get config for logging. # IgniteAwareService uses IgnitePersistenceAware mixin to override default Service 'log' definition. self.log_level = "DEBUG" - self.logs = IgnitePersistenceAware.logs self.properties = properties @@ -53,7 +50,7 @@ def __init__(self, context, num_nodes, properties, **kwargs): def start_node(self, node): self.init_persistent(node) - super(IgniteAwareService, self).start_node(node) + super().start_node(node) wait_until(lambda: len(self.pids(node)) > 0, timeout_sec=10) @@ -64,7 +61,7 @@ def init_persistent(self, node): Init persistent directory. :param node: Ignite service node. """ - super(IgniteAwareService, self).init_persistent(node) + super().init_persistent(node) node_config = self.spec.config().render(config_dir=self.PERSISTENT_ROOT, work_dir=self.WORK_DIR, diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py index a2bfe3dc026d2..61231989e9a40 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py @@ -24,7 +24,7 @@ DEFAULT_IGNITE_CONF = DEFAULT_CONFIG_PATH + "/ignite.xml.j2" -class Config(object): +class Config: """ Basic configuration. """ @@ -52,7 +52,7 @@ class IgniteServerConfig(Config): Ignite server node configuration. """ def __init__(self, path=DEFAULT_IGNITE_CONF): - super(IgniteServerConfig, self).__init__(path) + super().__init__(path) class IgniteClientConfig(Config): @@ -60,7 +60,7 @@ class IgniteClientConfig(Config): Ignite client node configuration. """ def __init__(self, path=DEFAULT_IGNITE_CONF): - super(IgniteClientConfig, self).__init__(path) + super().__init__(path) self.default_params.update(client_mode=True) @@ -69,4 +69,4 @@ class IgniteLoggerConfig(Config): Ignite logger configuration. """ def __init__(self): - super(IgniteLoggerConfig, self).__init__(DEFAULT_CONFIG_PATH + "/log4j.xml.j2") + super().__init__(DEFAULT_CONFIG_PATH + "/log4j.xml.j2") diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py index 9c79b2e5b8cd4..27c3161302fae 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py @@ -22,7 +22,7 @@ from ignitetest.services.utils.ignite_config import IgniteLoggerConfig -class PersistenceAware(object): +class PersistenceAware: """ This class contains basic persistence artifacts """ @@ -53,12 +53,18 @@ class IgnitePersistenceAware(PersistenceAware): CONFIG_FILE = os.path.join(PersistenceAware.PERSISTENT_ROOT, "ignite-config.xml") LOG4J_CONFIG_FILE = os.path.join(PersistenceAware.PERSISTENT_ROOT, "ignite-log4j.xml") + def __getattribute__(self, item): + if item == 'logs': + return PersistenceAware.logs + + return super().__getattribute__(item) + def init_persistent(self, node): """ Init persistent directory. :param node: Ignite service node. """ - super(IgnitePersistenceAware, self).init_persistent(node) + super().init_persistent(node) logger_config = IgniteLoggerConfig().render(work_dir=self.WORK_DIR) node.account.create_file(self.LOG4J_CONFIG_FILE, logger_config) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index 32468a93460b0..f992e3e7de3ab 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -54,7 +54,7 @@ def is_impl(impl): raise Exception("There is no specification for class %s" % type(service)) -class IgniteSpec(object): +class IgniteSpec: """ This class is a basic Spec """ @@ -102,8 +102,6 @@ class IgniteNodeSpec(IgniteSpec, IgnitePersistenceAware): """ Spec to run ignite node """ - def __init__(self, **kwargs): - IgniteSpec.__init__(self, **kwargs) def command(self): cmd = "%s %s %s %s 1>> %s 2>> %s &" % \ @@ -122,7 +120,7 @@ class IgniteApplicationSpec(IgniteSpec, IgnitePersistenceAware): Spec to run ignite application """ def __init__(self, **kwargs): - super(IgniteApplicationSpec, self).__init__(**kwargs) + super().__init__(**kwargs) self.args = "" def _app_args(self): @@ -148,7 +146,7 @@ class ApacheIgniteNodeSpec(IgniteNodeSpec, IgnitePersistenceAware): Implementation IgniteNodeSpec for Apache Ignite project """ def __init__(self, modules, **kwargs): - super(ApacheIgniteNodeSpec, self).__init__(project="ignite", **kwargs) + super().__init__(project="ignite", **kwargs) libs = (modules or []) libs.append("log4j") @@ -172,7 +170,7 @@ class ApacheIgniteApplicationSpec(IgniteApplicationSpec, IgnitePersistenceAware) """ # pylint: disable=too-many-arguments def __init__(self, context, modules, servicejava_class_name, java_class_name, params, start_ignite, **kwargs): - super(ApacheIgniteApplicationSpec, self).__init__(project="ignite", **kwargs) + super().__init__(project="ignite", **kwargs) self.context = context libs = modules or [] @@ -201,7 +199,7 @@ def __init__(self, context, modules, servicejava_class_name, java_class_name, pa str(start_ignite), java_class_name, self.CONFIG_FILE, - str(base64.b64encode(json.dumps(params).encode("UTF-8"))) + str(base64.b64encode(json.dumps(params).encode('utf-8')), 'utf-8') ] def __jackson(self): diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py index bfbab25f9d00a..925034b90fbe4 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_utils.py @@ -35,7 +35,7 @@ def ignite_jmx_mixin(node, pids): node.__class__ = type(base_cls_name, (base_cls, IgniteJmxMixin), {}) -class JmxMBean(object): +class JmxMBean: """ Dynamically exposes JMX MBean attributes. """ @@ -52,7 +52,7 @@ def __getattr__(self, attr): return self.client.mbean_attribute(self.name, attr) -class JmxClient(object): +class JmxClient: """JMX client, invokes jmxterm on node locally. """ jmx_util_cmd = 'java -jar /opt/jmxterm.jar -v silent -n' @@ -92,7 +92,7 @@ def __run_cmd(self, cmd): return self.node.account.ssh_capture(cmd, allow_fail=False, callback=str) -class DiscoveryInfo(object): +class DiscoveryInfo: """ Ignite service node discovery info, obtained from DiscoverySpi mbean. """ def __init__(self, coordinator, local_raw): @@ -147,7 +147,7 @@ def __find__(self, pattern): return res.group(1) if res else None -class IgniteJmxMixin(object): +class IgniteJmxMixin: """ Mixin to IgniteService node, exposing useful properties, obtained from JMX. """ diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py index f4a0158cc8f9c..b3da642aedd14 100644 --- a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py +++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py @@ -54,12 +54,12 @@ class ZookeeperService(Service): } def __init__(self, context, num_nodes, settings=ZookeeperSettings(), start_timeout_sec=60): - super(ZookeeperService, self).__init__(context, num_nodes) + super().__init__(context, num_nodes) self.settings = settings self.start_timeout_sec = start_timeout_sec def start(self): - super(ZookeeperService, self).start() + super().start() self.logger.info("Waiting for Zookeeper quorum...") for node in self.nodes: diff --git a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py index 0eebea8f3d0b8..a1d90b8a27c2c 100644 --- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py @@ -36,9 +36,6 @@ class AddNodeRebalanceTest(IgniteTest): DATA_AMOUNT = 1000000 REBALANCE_TIMEOUT = 60 - def __init__(self, test_context): - super(AddNodeRebalanceTest, self).__init__(test_context=test_context) - def setUp(self): pass diff --git a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py index 718d5daedc036..78d3eba04518f 100644 --- a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py +++ b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py @@ -68,9 +68,6 @@ def properties(): attr=CellularAffinity.ATTRIBUTE, cacheName=CellularAffinity.CACHE_NAME) - def __init__(self, test_context): - super(CellularAffinity, self).__init__(test_context=test_context) - def setUp(self): pass diff --git a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py index f2c1a4c430465..63c87ac0919bc 100644 --- a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py +++ b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py @@ -62,7 +62,7 @@ def properties(version): .render(version=version) def __init__(self, test_context): - super(BaselineTests, self).__init__(test_context) + super().__init__(test_context) self.servers = None @cluster(num_nodes=NUM_NODES) diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index ee94d2c177ede..8aec38fb2d56e 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -60,7 +60,7 @@ class DiscoveryTest(IgniteTest): """ def __init__(self, test_context): - super(DiscoveryTest, self).__init__(test_context=test_context) + super().__init__(test_context=test_context) self.zk_quorum = None self.servers = None @@ -206,7 +206,8 @@ def __simulate_nodes_failure(self, version, properties, nodes_to_kill=1): "grep '%s' %s" % (self.__failed_pattern(failed_id), IgniteAwareService.STDOUT_STDERR_CAPTURE)) logged_timestamps.append( - datetime.strptime(re.match("^\\[[^\\[]+\\]", stdout.read()).group(), "[%Y-%m-%d %H:%M:%S,%f]")) + datetime.strptime(re.match("^\\[[^\\[]+\\]", stdout.read().decode("utf-8")).group(), + "[%Y-%m-%d %H:%M:%S,%f]")) logged_timestamps.sort(reverse=True) diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index 774ab41edde5e..ea266c3e2baba 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -53,9 +53,6 @@ def properties(): """ - def __init__(self, test_context): - super(PmeFreeSwitchTest, self).__init__(test_context=test_context) - def setUp(self): pass diff --git a/modules/ducktests/tests/ignitetest/tests/smoke_test.py b/modules/ducktests/tests/ignitetest/tests/smoke_test.py index a8dfc87b73340..48965e366e57f 100644 --- a/modules/ducktests/tests/ignitetest/tests/smoke_test.py +++ b/modules/ducktests/tests/ignitetest/tests/smoke_test.py @@ -32,8 +32,6 @@ class SmokeServicesTest(IgniteTest): """ Tests services implementations """ - def __init__(self, test_context): - super(SmokeServicesTest, self).__init__(test_context=test_context) def setUp(self): pass diff --git a/modules/ducktests/tests/ignitetest/utils/_mark.py b/modules/ducktests/tests/ignitetest/utils/_mark.py index c4b2b35eecab2..7387d50978e67 100644 --- a/modules/ducktests/tests/ignitetest/utils/_mark.py +++ b/modules/ducktests/tests/ignitetest/utils/_mark.py @@ -16,7 +16,6 @@ """ Module contains useful test decorators. """ -import six from ducktape.mark._mark import Ignore, Mark from ignitetest.utils.version import IgniteVersion @@ -27,7 +26,7 @@ class VersionIf(Ignore): Ignore test if version doesn't corresponds to condition. """ def __init__(self, condition): - super(VersionIf, self).__init__() + super().__init__() self.condition = condition def apply(self, seed_context, context_list): @@ -36,13 +35,13 @@ def apply(self, seed_context, context_list): for ctx in context_list: assert 'version' in ctx.injected_args, "'version' in injected args not present" version = ctx.injected_args['version'] - assert isinstance(version, six.string_types), "'version' in injected args must be a string" + assert isinstance(version, str), "'version' in injected args must be a string" ctx.ignore = ctx.ignore or not self.condition(IgniteVersion(version)) return context_list def __eq__(self, other): - return super(VersionIf, self).__eq__(other) and self.condition == other.condition + return super().__eq__(other) and self.condition == other.condition def version_if(condition): diff --git a/modules/ducktests/tests/ignitetest/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/utils/ignite_test.py index c0c35c7924da4..a4bfe7da81f87 100644 --- a/modules/ducktests/tests/ignitetest/utils/ignite_test.py +++ b/modules/ducktests/tests/ignitetest/utils/ignite_test.py @@ -16,9 +16,9 @@ """ This module contains basic ignite test. """ +from time import monotonic from ducktape.tests.test import Test -from monotonic import monotonic # pylint: disable=W0223 @@ -27,7 +27,7 @@ class IgniteTest(Test): Basic ignite test. """ def __init__(self, test_context): - super(IgniteTest, self).__init__(test_context=test_context) + super().__init__(test_context=test_context) def stage(self, msg): """ diff --git a/modules/ducktests/tests/ignitetest/utils/version.py b/modules/ducktests/tests/ignitetest/utils/version.py index 5131b47404b4a..0454597bcd383 100644 --- a/modules/ducktests/tests/ignitetest/utils/version.py +++ b/modules/ducktests/tests/ignitetest/utils/version.py @@ -44,15 +44,13 @@ def __init__(self, version_string): if dev_suffix_index >= 0: version_string = version_string[:dev_suffix_index] - # Don't use the form super.(...).__init__(...) because - # LooseVersion is an "old style" python class - LooseVersion.__init__(self, version_string) + super().__init__(version_string) def __str__(self): if self.is_dev: return "dev" - return LooseVersion.__str__(self) + return super().__str__() def __repr__(self): return "IgniteVersion ('%s')" % str(self) diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index 8ad70b02b78ce..76dd8155bc6f0 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -30,5 +30,8 @@ license="apache2.0", packages=find_packages(exclude=["ignitetest.tests", "ignitetest.tests.*"]), include_package_data=True, - install_requires=["ducktape==0.7.8", "requests==2.22.0", "monotonic==1.5"] + install_requires=['ducktape==0.8.0'], + dependency_links = [ + 'https://github.com/confluentinc/ducktape/tarball/master#egg=ducktape-0.8.0' + ] ) diff --git a/modules/ducktests/tests/tox.ini b/modules/ducktests/tests/tox.ini index 60e7012ac43d4..45107be71cd3d 100644 --- a/modules/ducktests/tests/tox.ini +++ b/modules/ducktests/tests/tox.ini @@ -23,12 +23,8 @@ python = [testenv] envdir = {homedir}/.virtualenvs/ignite-ducktests-{envname} deps = - ducktape==0.7.8 - requests==2.20.0 - monotonic - mock - pytest - pylint==2.5.3 + -r ./docker/requirements.txt + pylint==2.6.0 [testenv:linter] basepython = python3.8 @@ -37,8 +33,6 @@ commands = [BASIC] min-public-methods=0 -# TODO: Remove after migrating to python 3 -disable=R0205 [SIMILARITIES] ignore-imports=yes From c6f55041ad4033d94cda759995c4245192d8bad4 Mon Sep 17 00:00:00 2001 From: Vladsz83 Date: Wed, 26 Aug 2020 12:36:58 +0300 Subject: [PATCH 46/78] Loading in discovery tests. (#8159) --- .../tests/ContinuousDataLoadApplication.java | 74 ++++++ .../utils/IgniteAwareApplication.java | 25 +- .../ignitetest/services/utils/ignite_spec.py | 1 + .../tests/ignitetest/tests/discovery_test.py | 225 ++++++++---------- 4 files changed, 203 insertions(+), 122 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java new file mode 100644 index 0000000000000..46a4b76f130a6 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.Random; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +/** + * Keeps data load until stopped. + */ +public class ContinuousDataLoadApplication extends IgniteAwareApplication { + /** Logger. */ + private static final Logger log = LogManager.getLogger(ContinuousDataLoadApplication.class.getName()); + + /** {@inheritDoc} */ + @Override protected void run(JsonNode jsonNode) { + String cacheName = jsonNode.get("cacheName").asText(); + int range = jsonNode.get("range").asInt(); + + IgniteCache cache = ignite.getOrCreateCache(cacheName); + + int warmUpCnt = (int)Math.max(1, 0.1f * range); + + Random rnd = new Random(); + + long streamed = 0; + + log.info("Generating data in background..."); + + long notifyTime = System.nanoTime(); + + while (active()) { + cache.put(rnd.nextInt(range), rnd.nextInt(range)); + + streamed++; + + if (notifyTime + U.millisToNanos(1500) < System.nanoTime()) { + notifyTime = System.nanoTime(); + + if (log.isDebugEnabled()) + log.debug("Streamed " + streamed + " entries."); + } + + // Delayed notify of the initialization to make sure the data load has completelly began and + // has produced some valuable amount of data. + if (!inited() && warmUpCnt == streamed) + markInitialized(); + } + + log.info("Background data generation finished."); + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java index 5e610f1eb583a..107787b91de07 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java @@ -76,8 +76,11 @@ protected IgniteAwareApplication() { else log.info("Application already done [finished=" + finished + ", broken=" + broken + "]"); + if (log.isDebugEnabled()) + log.debug("Waiting for graceful termination..."); + while (!finished && !broken) { - log.info("Waiting for graceful termnation."); + log.info("Waiting for graceful termination cycle..."); try { U.sleep(100); @@ -86,6 +89,9 @@ protected IgniteAwareApplication() { e.printStackTrace(); } } + + if (log.isDebugEnabled()) + log.debug("Graceful termination done."); })); log.info("ShutdownHook registered."); @@ -111,7 +117,8 @@ protected void markFinished() { log.info(APP_FINISHED); - removeShutdownHook(); + if (!terminated()) + removeShutdownHook(); finished = true; } @@ -165,6 +172,20 @@ protected boolean terminated() { return terminated; } + /** + * + */ + protected boolean inited() { + return inited; + } + + /** + * + */ + protected boolean active() { + return !(terminated || broken || finished); + } + /** * @param name Name. * @param val Value. diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index f992e3e7de3ab..9cbfcc00a2006 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -190,6 +190,7 @@ def __init__(self, context, modules, servicejava_class_name, java_class_name, pa self.jvm_opts.extend([ "-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file ", "-Dlog4j.configDebug=true", + "-DIGNITE_NO_SHUTDOWN_HOOK=true", # allows to perform operations on app termination. "-Xmx1G", "-ea", "-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false" diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 8aec38fb2d56e..63f6ef889a6ff 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -21,16 +21,17 @@ import re from datetime import datetime -from ducktape.mark import parametrize +from ducktape.mark import matrix from ducktape.mark.resource import cluster from jinja2 import Template +from ignitetest.services.ignite import IgniteAwareService from ignitetest.services.ignite import IgniteService -from ignitetest.services.utils.ignite_aware import IgniteAwareService +from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.utils.time_utils import epoch_mills from ignitetest.services.zk.zookeeper import ZookeeperService from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH, LATEST_2_7 +from ignitetest.utils.version import DEV_BRANCH, LATEST_2_8 # pylint: disable=W0223 @@ -41,10 +42,21 @@ class DiscoveryTest(IgniteTest): 2. Kill random node. 3. Wait that survived node detects node failure. """ + class Config: + """ + Configuration for DiscoveryTest. + """ + def __init__(self, nodes_to_kill=1, kill_coordinator=False, with_load=False): + self.nodes_to_kill = nodes_to_kill + self.kill_coordinator = kill_coordinator + self.with_load = with_load + NUM_NODES = 7 FAILURE_DETECTION_TIMEOUT = 2000 + DATA_AMOUNT = 100000 + CONFIG_TEMPLATE = """ {% if zookeeper_settings %} @@ -63,145 +75,85 @@ def __init__(self, test_context): super().__init__(test_context=test_context) self.zk_quorum = None self.servers = None + self.loader = None - def __start_zk_quorum(self): - self.zk_quorum = ZookeeperService(self.test_context, 3) - - self.stage("Starting ZooKeeper quorum") - - self.zk_quorum.start() + @cluster(num_nodes=NUM_NODES) + @matrix(ignite_version=[str(DEV_BRANCH), str(LATEST_2_8)], + kill_coordinator=[False, True], + nodes_to_kill=[1, 2], + with_load=[False, True]) + def test_tcp(self, ignite_version, kill_coordinator, nodes_to_kill, with_load): + """ + Test nodes failure scenario with TcpDiscoverySpi. + """ + config = DiscoveryTest.Config(nodes_to_kill, kill_coordinator, with_load) - self.stage("ZooKeeper quorum started") + return self.__simulate_nodes_failure(ignite_version, self.__properties(), None, config) - @staticmethod - def __properties(zookeeper_settings=None): + @cluster(num_nodes=NUM_NODES + 3) + @matrix(ignite_version=[str(DEV_BRANCH), str(LATEST_2_8)], + kill_coordinator=[False, True], + nodes_to_kill=[1, 2], + with_load=[False, True]) + def test_zk(self, ignite_version, kill_coordinator, nodes_to_kill, with_load): """ - :param zookeeper_settings: ZooKeeperDiscoverySpi settings. If None, TcpDiscoverySpi will be used. - :return: Rendered node's properties. + Test node failure scenario with ZooKeeperSpi. """ - return Template(DiscoveryTest.CONFIG_TEMPLATE) \ - .render(failure_detection_timeout=DiscoveryTest.FAILURE_DETECTION_TIMEOUT, - zookeeper_settings=zookeeper_settings) + config = DiscoveryTest.Config(nodes_to_kill, kill_coordinator, with_load) - @staticmethod - def __zk_properties(connection_string): - return DiscoveryTest.__properties(zookeeper_settings={'connection_string': connection_string}) + self.__start_zk_quorum() + + properties = self.__zk_properties(self.zk_quorum.connection_string()) + modules = ["zookeeper"] + + return self.__simulate_nodes_failure(ignite_version, properties, modules, config) def setUp(self): pass def teardown(self): - if self.zk_quorum: - self.zk_quorum.stop() + if self.loader: + self.loader.stop() if self.servers: self.servers.stop() - @cluster(num_nodes=NUM_NODES) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_7)) - def test_tcp_not_coordinator_single(self, version): - """ - Test single-node-failure scenario (not the coordinator) with TcpDiscoverySpi. - """ - return self.__simulate_nodes_failure(version, self.__properties(), 1) - - @cluster(num_nodes=NUM_NODES) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_7)) - def test_tcp_not_coordinator_two(self, version): - """ - Test two-node-failure scenario (not the coordinator) with TcpDiscoverySpi. - """ - return self.__simulate_nodes_failure(version, self.__properties(), 2) - - @cluster(num_nodes=NUM_NODES) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_7)) - def test_tcp_coordinator(self, version): - """ - Test coordinator-failure scenario with TcpDiscoverySpi. - """ - return self.__simulate_nodes_failure(version, self.__properties(), 0) - - @cluster(num_nodes=NUM_NODES + 3) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_7)) - def test_zk_not_coordinator_single(self, version): - """ - Test single node failure scenario (not the coordinator) with ZooKeeper. - """ - self.__start_zk_quorum() - - return self.__simulate_nodes_failure(version, self.__zk_properties(self.zk_quorum.connection_string()), 1) - - @cluster(num_nodes=NUM_NODES + 3) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_7)) - def test_zk_not_coordinator_two(self, version): - """ - Test two-node-failure scenario (not the coordinator) with ZooKeeper. - """ - self.__start_zk_quorum() - - return self.__simulate_nodes_failure(version, self.__zk_properties(self.zk_quorum.connection_string()), 2) - - @cluster(num_nodes=NUM_NODES+3) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_7)) - def test_zk_coordinator(self, version): - """ - Test coordinator-failure scenario with ZooKeeper. - """ - self.__start_zk_quorum() + if self.zk_quorum: + self.zk_quorum.stop() - return self.__simulate_nodes_failure(version, self.__zk_properties(self.zk_quorum.connection_string()), 0) + def __simulate_nodes_failure(self, version, properties, modules, config): + if config.nodes_to_kill < 1: + return {"No nodes to kill": "Nothing to do"} - def __simulate_nodes_failure(self, version, properties, nodes_to_kill=1): - """ - :param nodes_to_kill: How many nodes to kill. If <1, the coordinator is the choice. Otherwise: not-coordinator - nodes of given number. - """ self.servers = IgniteService( self.test_context, - num_nodes=self.NUM_NODES, - modules=["zookeeper"], + num_nodes=self.NUM_NODES - 1, + modules=modules, properties=properties, version=version) - self.stage("Starting ignite cluster") - time_holder = self.monotonic() self.servers.start() - if nodes_to_kill > self.servers.num_nodes - 1: - raise Exception("Too many nodes to kill: " + str(nodes_to_kill)) - data = {'Ignite cluster start time (s)': round(self.monotonic() - time_holder, 1)} - self.stage("Topology is ready") - failed_nodes, survived_node = self.__choose_node_to_kill(nodes_to_kill) + failed_nodes, survived_node = self.__choose_node_to_kill(config.kill_coordinator, config.nodes_to_kill) ids_to_wait = [node.discovery_info().node_id for node in failed_nodes] - self.stage("Stopping " + str(len(failed_nodes)) + " nodes.") + if config.with_load: + self.__start_loading(version, properties, modules) first_terminated = self.servers.stop_nodes_async(failed_nodes, clean_shutdown=False, wait_for_stop=False) - self.stage("Waiting for failure detection of " + str(len(failed_nodes)) + " nodes.") - # Keeps dates of logged node failures. logged_timestamps = [] for failed_id in ids_to_wait: - self.servers.await_event_on_node(self.__failed_pattern(failed_id), survived_node, 10, - from_the_beginning=True, backoff_sec=0.01) - # Save mono of last detected failure. - time_holder = self.monotonic() - self.stage("Failure detection measured.") + self.servers.await_event_on_node(self.__failed_pattern(failed_id), survived_node, 20, + from_the_beginning=True, backoff_sec=0.1) - for failed_id in ids_to_wait: _, stdout, _ = survived_node.account.ssh_client.exec_command( "grep '%s' %s" % (self.__failed_pattern(failed_id), IgniteAwareService.STDOUT_STDERR_CAPTURE)) @@ -211,37 +163,70 @@ def __simulate_nodes_failure(self, version, properties, nodes_to_kill=1): logged_timestamps.sort(reverse=True) - # Failure detection delay. - time_holder = int((time_holder - first_terminated[0]) * 1000) - # Failure detection delay by log. - by_log = epoch_mills(logged_timestamps[0]) - epoch_mills(first_terminated[1]) - - assert by_log > 0, "Negative node failure detection delay: " + by_log + ". Probably it is a timezone issue." - assert by_log <= time_holder, "Value of node failure detection delay taken from by the node log (" + \ - str(by_log) + "ms) must be lesser than measured value (" + str(time_holder) + \ - "ms) because watching this event consumes extra time." + self.__store_results(data, logged_timestamps, first_terminated[1]) - data['Detection of node(s) failure, measured (ms)'] = time_holder - data['Detection of node(s) failure, by the log (ms)'] = by_log data['Nodes failed'] = len(failed_nodes) return data + @staticmethod + def __store_results(data, logged_timestamps, first_kill_time): + first_kill_time = epoch_mills(first_kill_time) + + detection_delay = epoch_mills(logged_timestamps[0]) - first_kill_time + + data['Detection of node(s) failure (ms)'] = detection_delay + data['All detection delays (ms):'] = str([epoch_mills(ts) - first_kill_time for ts in logged_timestamps]) + @staticmethod def __failed_pattern(failed_node_id): return "Node FAILED: .\\{1,\\}Node \\[id=" + failed_node_id - def __choose_node_to_kill(self, nodes_to_kill): + def __choose_node_to_kill(self, kill_coordinator, nodes_to_kill): + assert nodes_to_kill > 0, "No nodes to kill passed. Check the parameters." + nodes = self.servers.nodes coordinator = nodes[0].discovery_info().coordinator + to_kill = [] - if nodes_to_kill < 1: - to_kill = next(node for node in nodes if node.discovery_info().node_id == coordinator) - else: - to_kill = random.sample([n for n in nodes if n.discovery_info().node_id != coordinator], nodes_to_kill) + if kill_coordinator: + to_kill.append(next(node for node in nodes if node.discovery_info().node_id == coordinator)) + nodes_to_kill -= 1 - to_kill = [to_kill] if not isinstance(to_kill, list) else to_kill + if nodes_to_kill > 0: + choice = random.sample([n for n in nodes if n.discovery_info().node_id != coordinator], nodes_to_kill) + to_kill.extend([choice] if not isinstance(choice, list) else choice) survive = random.choice([node for node in self.servers.nodes if node not in to_kill]) return to_kill, survive + + def __start_loading(self, ignite_version, properties, modules): + self.loader = IgniteApplicationService( + self.test_context, + java_class_name="org.apache.ignite.internal.ducktest.tests.ContinuousDataLoadApplication", + version=ignite_version, + modules=modules, + properties=properties, + params={"cacheName": "test-cache", "range": self.DATA_AMOUNT}) + + self.loader.start() + + def __start_zk_quorum(self): + self.zk_quorum = ZookeeperService(self.test_context, 3) + + self.zk_quorum.start() + + @staticmethod + def __properties(zookeeper_settings=None): + """ + :param zookeeper_settings: ZooKeeperDiscoverySpi settings. If None, TcpDiscoverySpi will be used. + :return: Rendered node's properties. + """ + return Template(DiscoveryTest.CONFIG_TEMPLATE) \ + .render(failure_detection_timeout=DiscoveryTest.FAILURE_DETECTION_TIMEOUT, + zookeeper_settings=zookeeper_settings) + + @staticmethod + def __zk_properties(connection_string): + return DiscoveryTest.__properties(zookeeper_settings={'connection_string': connection_string}) From bb5453ffcc2098ff389a1054e0926f4616670987 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Mon, 31 Aug 2020 17:25:11 +0300 Subject: [PATCH 47/78] Ducktape parallel (#8192) --- modules/ducktests/tests/docker/run_tests.sh | 18 +- .../tests/ignitetest/services/ignite.py | 9 +- .../tests/ignitetest/services/ignite_app.py | 17 +- .../{ignite_config.py => config_template.py} | 10 +- .../ignitetest/services/utils/ignite_aware.py | 31 ++- .../utils/ignite_configuration/__init__.py | 46 ++++ .../utils/ignite_configuration/cache.py | 29 ++ .../ignite_configuration/data_storage.py | 38 +++ .../utils/ignite_configuration/discovery.py | 141 ++++++++++ .../services/utils/ignite_persistence.py | 4 +- .../ignitetest/services/utils/ignite_spec.py | 59 ++-- .../services/utils/templates/cache_macro.j2 | 34 +++ .../utils/templates/datastorage_macro.j2 | 46 ++++ .../utils/templates/discovery_macro.j2 | 57 ++++ .../utils/{config => templates}/ignite.xml.j2 | 29 +- .../utils/{config => templates}/log4j.xml.j2 | 0 .../services/utils/templates/misc_macro.j2 | 24 ++ .../tests/add_node_rebalance_test.py | 18 +- .../tests/cellular_affinity_test.py | 35 ++- .../ignitetest/tests/control_utility_test.py | 85 +++--- .../tests/ignitetest/tests/discovery_test.py | 251 +++++++++--------- .../ignitetest/tests/pme_free_switch_test.py | 52 ++-- .../tests/ignitetest/tests/smoke_test.py | 31 +-- .../ignitetest/tests/suites/fast_suite.yml | 29 ++ .../ignitetest/tests/suites/slow_suite.yml | 17 ++ modules/ducktests/tests/setup.py | 5 +- 26 files changed, 772 insertions(+), 343 deletions(-) rename modules/ducktests/tests/ignitetest/services/utils/{ignite_config.py => config_template.py} (91%) create mode 100644 modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 create mode 100644 modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 create mode 100644 modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 rename modules/ducktests/tests/ignitetest/services/utils/{config => templates}/ignite.xml.j2 (65%) rename modules/ducktests/tests/ignitetest/services/utils/{config => templates}/log4j.xml.j2 (100%) create mode 100644 modules/ducktests/tests/ignitetest/services/utils/templates/misc_macro.j2 create mode 100644 modules/ducktests/tests/ignitetest/tests/suites/fast_suite.yml create mode 100644 modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml diff --git a/modules/ducktests/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh index e5dc56113f357..b9ee4887b87bd 100755 --- a/modules/ducktests/tests/docker/run_tests.sh +++ b/modules/ducktests/tests/docker/run_tests.sh @@ -52,13 +52,22 @@ Usage: ${0} [options] The options are as follows: -h|--help - Display this help message + Display this help message. + +-n|--num-nodes + Specify how many nodes to start. Default number of nodes to start: 11. + +-j|--max-parallel + Specify max number of tests that can be run in parallel. -p|--param Use specified param to inject in tests. Could be used multiple times. ./run_tests.sh --param version=2.8.1 +-pj|--params-json + Use specified json as parameters to inject in tests. Can be extended with -p|--param. + -g|--global Use specified global param to pass to test context. Could be used multiple times. @@ -108,8 +117,11 @@ while [[ $# -ge 1 ]]; do case "$1" in -h|--help) usage;; -p|--param) duck_add_param "$2"; shift 2;; + -pj|--params-json) PARAMETERS="$2"; shift 2;; -g|--global) duck_add_global "$2"; shift 2;; -t|--tc-paths) TC_PATHS="$2"; shift 2;; + -n|--num-nodes) IGNITE_NUM_CONTAINERS="$2"; shift 2;; + -j|--max-parallel) MAX_PARALLEL="$2"; shift 2;; -f|--force) FORCE=$1; shift;; *) break;; esac @@ -139,5 +151,9 @@ if [[ "$PARAMETERS" != "{}" ]]; then DUCKTAPE_OPTIONS="$DUCKTAPE_OPTIONS --parameters '$PARAMETERS'" fi +if [[ -n "$MAX_PARALLEL" ]]; then + DUCKTAPE_OPTIONS="$DUCKTAPE_OPTIONS --max-parallel $MAX_PARALLEL" +fi + "$SCRIPT_DIR"/ducker-ignite test "$TC_PATHS" "$DUCKTAPE_OPTIONS" \ || die "ducker-ignite test failed" diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index 2fb64b3b43578..f50365d7c156e 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -31,7 +31,6 @@ from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.utils.version import DEV_BRANCH class IgniteService(IgniteAwareService): @@ -42,10 +41,8 @@ class IgniteService(IgniteAwareService): HEAP_DUMP_FILE = os.path.join(IgniteAwareService.PERSISTENT_ROOT, "ignite-heap.bin") # pylint: disable=R0913 - def __init__(self, context, num_nodes, jvm_opts=None, properties="", client_mode=False, modules=None, - version=DEV_BRANCH): - super().__init__(context, num_nodes, properties, client_mode=client_mode, modules=modules, version=version, - jvm_opts=jvm_opts) + def __init__(self, context, config, num_nodes, jvm_opts=None, modules=None): + super().__init__(context, config, num_nodes, modules=modules, jvm_opts=jvm_opts) # pylint: disable=W0221 def start(self, timeout_sec=180): @@ -125,7 +122,7 @@ def __stop_node(node, pid, sig, start_waiter=None, delay_ms=0, time_holder=None) start_waiter.wait() if delay_ms > 0: - time.sleep(delay_ms/1000.0) + time.sleep(delay_ms / 1000.0) if time_holder: mono = monotonic() diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 84871d80fa768..b40d01ada55f2 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -20,7 +20,6 @@ import re from ignitetest.services.utils.ignite_aware import IgniteAwareService -from ignitetest.utils.version import DEV_BRANCH class IgniteApplicationService(IgniteAwareService): @@ -31,18 +30,10 @@ class IgniteApplicationService(IgniteAwareService): SERVICE_JAVA_CLASS_NAME = "org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService" # pylint: disable=R0913 - def __init__(self, context, java_class_name, params="", properties="", timeout_sec=60, modules=None, - client_mode=True, version=DEV_BRANCH, servicejava_class_name=SERVICE_JAVA_CLASS_NAME, - jvm_opts=None, start_ignite=True): - super().__init__(context, 1, properties, - client_mode=client_mode, - version=version, - modules=modules, - servicejava_class_name=servicejava_class_name, - java_class_name=java_class_name, - params=params, - jvm_opts=jvm_opts, - start_ignite=start_ignite) + def __init__(self, context, config, java_class_name, params="", timeout_sec=60, modules=None, + servicejava_class_name=SERVICE_JAVA_CLASS_NAME, jvm_opts=None, start_ignite=True): + super().__init__(context, config, 1, modules=modules, servicejava_class_name=servicejava_class_name, + java_class_name=java_class_name, params=params, jvm_opts=jvm_opts, start_ignite=start_ignite) self.servicejava_class_name = servicejava_class_name self.java_class_name = java_class_name diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py b/modules/ducktests/tests/ignitetest/services/utils/config_template.py similarity index 91% rename from modules/ducktests/tests/ignitetest/services/utils/ignite_config.py rename to modules/ducktests/tests/ignitetest/services/utils/config_template.py index 61231989e9a40..5170c4d37c1a0 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_config.py +++ b/modules/ducktests/tests/ignitetest/services/utils/config_template.py @@ -20,11 +20,11 @@ from jinja2 import FileSystemLoader, Environment -DEFAULT_CONFIG_PATH = os.path.dirname(os.path.abspath(__file__)) + "/config" +DEFAULT_CONFIG_PATH = os.path.dirname(os.path.abspath(__file__)) + "/templates" DEFAULT_IGNITE_CONF = DEFAULT_CONFIG_PATH + "/ignite.xml.j2" -class Config: +class ConfigTemplate: """ Basic configuration. """ @@ -47,7 +47,7 @@ def render(self, **kwargs): return res -class IgniteServerConfig(Config): +class IgniteServerConfigTemplate(ConfigTemplate): """ Ignite server node configuration. """ @@ -55,7 +55,7 @@ def __init__(self, path=DEFAULT_IGNITE_CONF): super().__init__(path) -class IgniteClientConfig(Config): +class IgniteClientConfigTemplate(ConfigTemplate): """ Ignite client node configuration. """ @@ -64,7 +64,7 @@ def __init__(self, path=DEFAULT_IGNITE_CONF): self.default_params.update(client_mode=True) -class IgniteLoggerConfig(Config): +class IgniteLoggerConfigTemplate(ConfigTemplate): """ Ignite logger configuration. """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 9f374bb5fd6f1..fadbbe6d77d55 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -33,7 +33,7 @@ class IgniteAwareService(BackgroundThreadService, IgnitePersistenceAware, metacl """ # pylint: disable=R0913 - def __init__(self, context, num_nodes, properties, **kwargs): + def __init__(self, context, config, num_nodes, **kwargs): """ **kwargs are params that passed to IgniteSpec """ @@ -43,9 +43,9 @@ def __init__(self, context, num_nodes, properties, **kwargs): # IgniteAwareService uses IgnitePersistenceAware mixin to override default Service 'log' definition. self.log_level = "DEBUG" - self.properties = properties + self.config = config - self.spec = resolve_spec(self, context, **kwargs) + self.spec = resolve_spec(self, context, config, **kwargs) def start_node(self, node): self.init_persistent(node) @@ -63,14 +63,27 @@ def init_persistent(self, node): """ super().init_persistent(node) - node_config = self.spec.config().render(config_dir=self.PERSISTENT_ROOT, - work_dir=self.WORK_DIR, - properties=self.properties, - consistent_id=node.account.externally_routable_ip) + node_config = self._prepare_config(node) - setattr(node, "consistent_id", node.account.externally_routable_ip) node.account.create_file(self.CONFIG_FILE, node_config) + def _prepare_config(self, node): + if not self.config.consistent_id: + config = self.config._replace(consistent_id=node.account.externally_routable_ip) + else: + config = self.config + + config.discovery_spi.prepare_on_start(cluster=self) + + node_config = self.spec.config_template.render(config_dir=self.PERSISTENT_ROOT, work_dir=self.WORK_DIR, + config=config) + + setattr(node, "consistent_id", node.account.externally_routable_ip) + + self.logger.debug("Config for node %s: %s" % (node.account.hostname, node_config)) + + return node_config + @abstractmethod def pids(self, node): """ @@ -81,7 +94,7 @@ def pids(self, node): # pylint: disable=W0613 def _worker(self, idx, node): - cmd = self.spec.command() + cmd = self.spec.command self.logger.debug("Attempting to start Application Service on %s with command: %s" % (str(node.account), cmd)) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py new file mode 100644 index 0000000000000..b9602e46339a1 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains IgniteConfiguration classes and utilities. +""" + +from typing import NamedTuple + +from ignitetest.services.utils.ignite_configuration.data_storage import DataStorageConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import DiscoverySpi, TcpDiscoverySpi +from ignitetest.utils.version import IgniteVersion, DEV_BRANCH + + +class IgniteConfiguration(NamedTuple): + """ + Ignite configuration. + """ + discovery_spi: DiscoverySpi = TcpDiscoverySpi() + version: IgniteVersion = DEV_BRANCH + cluster_state: str = 'ACTIVE' + client_mode: bool = False + consistent_id: str = None + failure_detection_timeout: int = 10000 + properties: str = None + data_storage: DataStorageConfiguration = None + caches: list = [] + + +class IgniteClientConfiguration(IgniteConfiguration): + """ + Ignite client configuration. + """ + client_mode = True diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py new file mode 100644 index 0000000000000..cc58a69bb3670 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +""" +This module contains classes and utilities for Ignite Cache configuration. +""" +from typing import NamedTuple + + +class CacheConfiguration(NamedTuple): + """ + Ignite Cache configuration. + """ + name: str + cache_mode: str = 'PARTITIONED' + atomicity_mode: str = 'ATOMIC' + backups: int = 0 diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py new file mode 100644 index 0000000000000..7b2999d82e4b8 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +""" +This module contains classes and utilities for Ignite DataStorage configuration. +""" + +from typing import NamedTuple + + +class DataRegionConfiguration(NamedTuple): + """ + Ignite DataRegion Configuration + """ + name: str = "default" + persistent: bool = False + init_size: int = 100 * 1024 * 1024 + max_size: int = 512 * 1024 * 1024 + + +class DataStorageConfiguration(NamedTuple): + """ + Ignite DataStorage configuration + """ + default: DataRegionConfiguration = DataRegionConfiguration() + regions: list = [] diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py new file mode 100644 index 0000000000000..9f38da5f9de99 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +""" +Module contains classes and utility methods to create discovery configuration for ignite nodes. +""" + +from abc import ABCMeta, abstractmethod + +from ignitetest.services.utils.ignite_aware import IgniteAwareService +from ignitetest.services.zk.zookeeper import ZookeeperService + + +class DiscoverySpi(metaclass=ABCMeta): + """ + Abstract class for DiscoverySpi. + """ + @property + @abstractmethod + def type(self): + """ + Type of DiscoverySPI. + """ + + @abstractmethod + def prepare_on_start(self, **kwargs): + """ + Call if update before start is needed. + """ + + +class ZookeeperDiscoverySpi(DiscoverySpi): + """ + ZookeeperDiscoverySpi. + """ + def __init__(self, connection_string, root_path): + self.connection_string = connection_string + self.root_path = root_path + + @property + def type(self): + return "ZOOKEEPER" + + def prepare_on_start(self, **kwargs): + pass + + +class TcpDiscoveryIpFinder(metaclass=ABCMeta): + """ + Abstract class for TcpDiscoveryIpFinder. + """ + @property + @abstractmethod + def type(self): + """ + Type of TcpDiscoveryIpFinder. + """ + + @abstractmethod + def prepare_on_start(self, **kwargs): + """ + Call if update before start is needed. + """ + + +class TcpDiscoveryVmIpFinder(TcpDiscoveryIpFinder): + """ + IpFinder with static ips, obtained from cluster nodes. + """ + def __init__(self, nodes=None): + self.addresses = TcpDiscoveryVmIpFinder.__get_addresses(nodes) if nodes else None + + @property + def type(self): + return 'VM' + + def prepare_on_start(self, **kwargs): + if not self.addresses: + cluster = kwargs.get('cluster') + self.addresses = TcpDiscoveryVmIpFinder.__get_addresses(cluster.nodes) + + @staticmethod + def __get_addresses(nodes): + return [node.account.externally_routable_ip for node in nodes] + + +class TcpDiscoverySpi(DiscoverySpi): + """ + TcpDiscoverySpi. + """ + def __init__(self, ip_finder=TcpDiscoveryVmIpFinder()): + self.ip_finder = ip_finder + + @property + def type(self): + return 'TCP' + + def prepare_on_start(self, **kwargs): + self.ip_finder.prepare_on_start(**kwargs) + + +def from_ignite_cluster(cluster, subset=None): + """ + Form TcpDiscoverySpi from cluster or its subset. + :param cluster: IgniteService cluster + :param subset: slice object (optional). + :return: TcpDiscoverySpi with static ip addresses. + """ + assert isinstance(cluster, IgniteAwareService) + + if subset: + assert isinstance(subset, slice) + nodes = cluster.nodes[subset] + else: + nodes = cluster.nodes + + return TcpDiscoverySpi(ip_finder=TcpDiscoveryVmIpFinder(nodes)) + + +def from_zookeeper_cluster(cluster, root_path="/apacheIgnite"): + """ + Form ZookeeperDiscoverySpi from zookeeper service cluster. + :param cluster: ZookeeperService cluster. + :param root_path: root ZNode path. + :return: ZookeeperDiscoverySpi. + """ + assert isinstance(cluster, ZookeeperService) + + return ZookeeperDiscoverySpi(cluster.connection_string(), root_path=root_path) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py index 27c3161302fae..589b5355be03f 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_persistence.py @@ -19,7 +19,7 @@ import os -from ignitetest.services.utils.ignite_config import IgniteLoggerConfig +from ignitetest.services.utils.config_template import IgniteLoggerConfigTemplate class PersistenceAware: @@ -66,5 +66,5 @@ def init_persistent(self, node): """ super().init_persistent(node) - logger_config = IgniteLoggerConfig().render(work_dir=self.WORK_DIR) + logger_config = IgniteLoggerConfigTemplate().render(work_dir=self.WORK_DIR) node.account.create_file(self.LOG4J_CONFIG_FILE, logger_config) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index 9cbfcc00a2006..124fefb530f57 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -20,16 +20,16 @@ import base64 import importlib import json +from abc import ABCMeta, abstractmethod from ignitetest.services.utils.ignite_path import IgnitePath -from ignitetest.services.utils.ignite_config import IgniteClientConfig, IgniteServerConfig -from ignitetest.utils.version import DEV_BRANCH, IgniteVersion +from ignitetest.services.utils.config_template import IgniteClientConfigTemplate, IgniteServerConfigTemplate +from ignitetest.utils.version import DEV_BRANCH from ignitetest.services.utils.ignite_persistence import IgnitePersistenceAware -# pylint: disable=no-else-return -def resolve_spec(service, context, **kwargs): +def resolve_spec(service, context, config, **kwargs): """ Resolve Spec classes for IgniteService and IgniteApplicationService """ @@ -47,41 +47,40 @@ def is_impl(impl): return len(impl_filter) > 0 if is_impl("IgniteService"): - return _resolve_spec("NodeSpec", ApacheIgniteNodeSpec)(**kwargs) - elif is_impl("IgniteApplicationService"): - return _resolve_spec("AppSpec", ApacheIgniteApplicationSpec)(context=context, **kwargs) - else: - raise Exception("There is no specification for class %s" % type(service)) + return _resolve_spec("NodeSpec", ApacheIgniteNodeSpec)(config=config, **kwargs) + if is_impl("IgniteApplicationService"): + return _resolve_spec("AppSpec", ApacheIgniteApplicationSpec)(context=context, config=config, **kwargs) -class IgniteSpec: + raise Exception("There is no specification for class %s" % type(service)) + + +class IgniteSpec(metaclass=ABCMeta): """ This class is a basic Spec """ - def __init__(self, version, project, client_mode, jvm_opts): - if isinstance(version, IgniteVersion): - self.version = version - else: - self.version = IgniteVersion(version) - + def __init__(self, config, project, jvm_opts): + self.version = config.version self.path = IgnitePath(self.version, project) self.envs = {} self.jvm_opts = jvm_opts or [] - self.client_mode = client_mode + self.config = config - def config(self): + @property + def config_template(self): """ :return: config that service will use to start on a node """ - if self.client_mode: - return IgniteClientConfig() - return IgniteServerConfig() + if self.config.client_mode: + return IgniteClientConfigTemplate() + return IgniteServerConfigTemplate() + @property + @abstractmethod def command(self): """ :return: string that represents command to run service on a node """ - raise NotImplementedError() def _envs(self): """ @@ -102,14 +101,13 @@ class IgniteNodeSpec(IgniteSpec, IgnitePersistenceAware): """ Spec to run ignite node """ - + @property def command(self): - cmd = "%s %s %s %s 1>> %s 2>> %s &" % \ + cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \ (self._envs(), self.path.script("ignite.sh"), self._jvm_opts(), self.CONFIG_FILE, - self.STDOUT_STDERR_CAPTURE, self.STDOUT_STDERR_CAPTURE) return cmd @@ -126,21 +124,18 @@ def __init__(self, **kwargs): def _app_args(self): return ",".join(self.args) + @property def command(self): - cmd = "%s %s %s %s 1>> %s 2>> %s &" % \ + cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \ (self._envs(), self.path.script("ignite.sh"), self._jvm_opts(), self._app_args(), - self.STDOUT_STDERR_CAPTURE, self.STDOUT_STDERR_CAPTURE) return cmd -### - - class ApacheIgniteNodeSpec(IgniteNodeSpec, IgnitePersistenceAware): """ Implementation IgniteNodeSpec for Apache Ignite project @@ -208,5 +203,5 @@ def __jackson(self): aws = self.path.module("aws") return self.context.cluster.nodes[0].account.ssh_capture( "ls -d %s/* | grep jackson | tr '\n' ':' | sed 's/.$//'" % aws) - else: - return [] + + return [] diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 new file mode 100644 index 0000000000000..d227369abe3d7 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 @@ -0,0 +1,34 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} + +{% macro cache_configs(caches) %} + {% if caches %} + + + {% for cache in caches %} + + + {% if cache.cache_mode == 'PARTITIONED' %} + + {% endif %} + + + {% endfor %} + + + {% endif %} +{% endmacro %} diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 new file mode 100644 index 0000000000000..1c2b4627b10e4 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 @@ -0,0 +1,46 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} + +{% macro data_storage(config) %} + {% if config %} + + + + {{ data_region(config.default) }} + + {% if config.regions %} + + + {% for region in config.regions %} + {{ data_region(region) }} + {% endfor %} + + + {% endif %} + + + {% endif %} +{% endmacro %} + +{% macro data_region(config) %} + + + + + + +{% endmacro %} diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 new file mode 100644 index 0000000000000..fdd632a46b5fd --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 @@ -0,0 +1,57 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} + +{% macro ip_finder(finder) %} + {% if finder %} + + {% if finder.type == 'VM' %} + + + + {% for address in finder.addresses %} + {{ address }} + {% endfor %} + + + + {% endif %} + + {% endif %} +{% endmacro %} + +{% macro zookeeper_discovery_spi(spi) %} + + + + +{% endmacro %} + +{% macro tcp_discovery_spi(spi) %} + + {{ ip_finder(spi.ip_finder) }} + +{% endmacro %} + +{% macro discovery_spi(spi) %} + + {% if spi.type == 'TCP' %} + {{ tcp_discovery_spi(spi) }} + {% elif spi.type == 'ZOOKEEPER' %} + {{ zookeeper_discovery_spi(spi) }} + {% endif %} + +{% endmacro %} diff --git a/modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2 similarity index 65% rename from modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 rename to modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2 index f03ba1769d215..86abd3529587e 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/config/ignite.xml.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2 @@ -1,6 +1,6 @@ - +#} + +{% import 'discovery_macro.j2' as disco_utils %} +{% import 'cache_macro.j2' as cache_utils %} +{% import 'datastorage_macro.j2' as datastorage_utils %} +{% import 'misc_macro.j2' as misc_utils %} - - {% if consistent_id %} - - {% endif %} - {% if properties %} - {{ properties }} + + + + + {{ misc_utils.cluster_state(config.cluster_state, config.version) }} + + {{ disco_utils.discovery_spi(config.discovery_spi) }} + + {{ datastorage_utils.data_storage(config.data_storage) }} + + {{ cache_utils.cache_configs(config.caches) }} + + {% if config.properties %} + {{ config.properties }} {% endif %} diff --git a/modules/ducktests/tests/ignitetest/services/utils/config/log4j.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 similarity index 100% rename from modules/ducktests/tests/ignitetest/services/utils/config/log4j.xml.j2 rename to modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/misc_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/misc_macro.j2 new file mode 100644 index 0000000000000..892453b74fb2e --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/misc_macro.j2 @@ -0,0 +1,24 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} + +{% macro cluster_state(state, version) %} + {% if version > "2.9.0" %} + + {% else %} + + {% endif %} +{% endmacro %} diff --git a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py index a1d90b8a27c2c..b162d2e79203c 100644 --- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py @@ -22,6 +22,8 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, IgniteVersion, LATEST @@ -36,12 +38,6 @@ class AddNodeRebalanceTest(IgniteTest): DATA_AMOUNT = 1000000 REBALANCE_TIMEOUT = 60 - def setUp(self): - pass - - def teardown(self): - pass - @cluster(num_nodes=NUM_NODES + 1) @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST)) @@ -56,20 +52,22 @@ def test_add_node(self, version): self.stage("Start Ignite nodes") - ignites = IgniteService(self.test_context, num_nodes=AddNodeRebalanceTest.NUM_NODES - 1, version=ignite_version) + node_config = IgniteConfiguration(version=ignite_version) + ignites = IgniteService(self.test_context, config=node_config, num_nodes=self.NUM_NODES - 1) ignites.start() self.stage("Starting DataGenerationApplication") # This client just put some data to the cache. - IgniteApplicationService(self.test_context, + app_config = node_config._replace(client_mode=True, discovery_spi=from_ignite_cluster(ignites)) + IgniteApplicationService(self.test_context, config=app_config, java_class_name="org.apache.ignite.internal.ducktest.tests.DataGenerationApplication", - version=ignite_version, params={"cacheName": "test-cache", "range": self.DATA_AMOUNT}, timeout_sec=self.PRELOAD_TIMEOUT).run() - ignite = IgniteService(self.test_context, num_nodes=1, version=ignite_version) + ignite = IgniteService(self.test_context, node_config._replace(discovery_spi=from_ignite_cluster(ignites)), + num_nodes=1) self.stage("Starting Ignite node") diff --git a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py index 78d3eba04518f..6b1e6647c0f23 100644 --- a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py +++ b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py @@ -23,8 +23,10 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteClientConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion # pylint: disable=W0223 @@ -68,41 +70,36 @@ def properties(): attr=CellularAffinity.ATTRIBUTE, cacheName=CellularAffinity.CACHE_NAME) - def setUp(self): - pass - - def teardown(self): - pass - @cluster(num_nodes=NUM_NODES * 3 + 1) @parametrize(version=str(DEV_BRANCH)) def test(self, version): """ Test Cellular Affinity scenario (partition distribution). """ - self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=1']) - self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=2']) - self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=XXX', '-DRANDOM=42']) + cell1 = self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=1']) + self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=2'], joined_cluster=cell1) + self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=XXX', '-DRANDOM=42'], joined_cluster=cell1) checker = IgniteApplicationService( self.test_context, + IgniteClientConfiguration(version=IgniteVersion(version), discovery_spi=from_ignite_cluster(cell1)), java_class_name="org.apache.ignite.internal.ducktest.tests.cellular_affinity_test.DistributionChecker", params={"cacheName": CellularAffinity.CACHE_NAME, "attr": CellularAffinity.ATTRIBUTE, - "nodesPerCell": self.NUM_NODES}, - version=version) + "nodesPerCell": self.NUM_NODES}) checker.run() - def start_cell(self, ignite_version, jvm_opts): + def start_cell(self, version, jvm_opts, joined_cluster=None): """ Starts cell. """ - ignites = IgniteService( - self.test_context, - num_nodes=CellularAffinity.NUM_NODES, - version=ignite_version, - properties=self.properties(), - jvm_opts=jvm_opts) + config = IgniteConfiguration(version=IgniteVersion(version), properties=self.properties()) + if joined_cluster: + config = config._replace(discovery_spi=from_ignite_cluster(joined_cluster)) + + ignites = IgniteService(self.test_context, config, num_nodes=CellularAffinity.NUM_NODES, jvm_opts=jvm_opts) ignites.start() + + return ignites diff --git a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py index 63c87ac0919bc..a4afd092ec82c 100644 --- a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py +++ b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py @@ -19,10 +19,12 @@ from ducktape.mark import parametrize from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until -from jinja2 import Template from ignitetest.services.ignite import IgniteService from ignitetest.services.utils.control_utility import ControlUtility, ControlUtilityError +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration +from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.utils import version_if from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, LATEST_2_8, IgniteVersion, LATEST_2_7, V_2_8_0 @@ -35,36 +37,6 @@ class BaselineTests(IgniteTest): """ NUM_NODES = 3 - CONFIG_TEMPLATE = """ - {% if version > "2.9.0" %} - - {% else %} - - {% endif %} - - - - - - - - - - - """ - - @staticmethod - def properties(version): - """ - Render properties for ignite node configuration. - """ - return Template(BaselineTests.CONFIG_TEMPLATE) \ - .render(version=version) - - def __init__(self, test_context): - super().__init__(test_context) - self.servers = None - @cluster(num_nodes=NUM_NODES) @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_8)) @@ -74,19 +46,19 @@ def test_baseline_set(self, version): Test baseline set. """ blt_size = self.NUM_NODES - 2 - self.servers = self.__start_ignite_nodes(version, blt_size) + servers = self.__start_ignite_nodes(version, blt_size) - control_utility = ControlUtility(self.servers, self.test_context) + control_utility = ControlUtility(servers, self.test_context) control_utility.activate() # Check baseline of activated cluster. baseline = control_utility.baseline() self.__check_baseline_size(baseline, blt_size) - self.__check_nodes_in_baseline(self.servers.nodes, baseline) + self.__check_nodes_in_baseline(servers.nodes, baseline) - # Set baseline using list of conststent ids. - new_node = self.__start_ignite_nodes(version, 1) - control_utility.set_baseline(self.servers.nodes + new_node.nodes) + # Set baseline using list of consisttent ids. + new_node = self.__start_ignite_nodes(version, 1, join_cluster=servers) + control_utility.set_baseline(servers.nodes + new_node.nodes) blt_size += 1 baseline = control_utility.baseline() @@ -94,7 +66,7 @@ def test_baseline_set(self, version): self.__check_nodes_in_baseline(new_node.nodes, baseline) # Set baseline using topology version. - new_node = self.__start_ignite_nodes(version, 1) + new_node = self.__start_ignite_nodes(version, 1, join_cluster=servers) _, version, _ = control_utility.cluster_state() control_utility.set_baseline(version) blt_size += 1 @@ -112,14 +84,14 @@ def test_baseline_add_remove(self, version): Test add and remove nodes from baseline. """ blt_size = self.NUM_NODES - 1 - self.servers = self.__start_ignite_nodes(version, blt_size) + servers = self.__start_ignite_nodes(version, blt_size) - control_utility = ControlUtility(self.servers, self.test_context) + control_utility = ControlUtility(servers, self.test_context) control_utility.activate() # Add node to baseline. - new_node = self.__start_ignite_nodes(version, 1) + new_node = self.__start_ignite_nodes(version, 1, join_cluster=servers) control_utility.add_to_baseline(new_node.nodes) blt_size += 1 @@ -138,7 +110,7 @@ def test_baseline_add_remove(self, version): # Remove of offline node from baseline. new_node.stop() - self.servers.await_event("Node left topology", timeout_sec=30, from_the_beginning=True) + servers.await_event("Node left topology", timeout_sec=30, from_the_beginning=True) control_utility.remove_from_baseline(new_node.nodes) blt_size -= 1 @@ -155,9 +127,9 @@ def test_activate_deactivate(self, version): """ Test activate and deactivate cluster. """ - self.servers = self.__start_ignite_nodes(version, self.NUM_NODES) + servers = self.__start_ignite_nodes(version, self.NUM_NODES) - control_utility = ControlUtility(self.servers, self.test_context) + control_utility = ControlUtility(servers, self.test_context) control_utility.activate() @@ -180,14 +152,14 @@ def test_baseline_autoadjust(self, version): Test activate and deactivate cluster. """ blt_size = self.NUM_NODES - 2 - self.servers = self.__start_ignite_nodes(version, blt_size) + servers = self.__start_ignite_nodes(version, blt_size) - control_utility = ControlUtility(self.servers, self.test_context) + control_utility = ControlUtility(servers, self.test_context) control_utility.activate() # Add node. control_utility.enable_baseline_auto_adjust(2000) - new_node = self.__start_ignite_nodes(version, 1) + new_node = self.__start_ignite_nodes(version, 1, join_cluster=servers) blt_size += 1 wait_until(lambda: len(control_utility.baseline()) == blt_size, timeout_sec=5) @@ -198,7 +170,7 @@ def test_baseline_autoadjust(self, version): # Add node when auto adjust disabled. control_utility.disable_baseline_auto_adjust() old_topology = control_utility.cluster_state().topology_version - new_node = self.__start_ignite_nodes(version, 1) + new_node = self.__start_ignite_nodes(version, 1, join_cluster=servers) wait_until(lambda: control_utility.cluster_state().topology_version != old_topology, timeout_sec=5) baseline = control_utility.baseline() @@ -222,11 +194,20 @@ def __check_nodes_not_in_baseline(nodes, baseline): def __check_baseline_size(baseline, size): assert len(baseline) == size, 'Unexpected size of baseline %d, %d expected' % (len(baseline), size) - def __start_ignite_nodes(self, version, num_nodes, timeout_sec=180): - ignite_version = IgniteVersion(version) + def __start_ignite_nodes(self, version, num_nodes, timeout_sec=60, join_cluster=None): + config = IgniteConfiguration( + cluster_state="INACTIVE", + version=IgniteVersion(version), + data_storage=DataStorageConfiguration( + default=DataRegionConfiguration(name='persistent', persistent=True), + regions=[DataRegionConfiguration(name='in-memory', persistent=False, max_size=100 * 1024 * 1024)] + ) + ) + + if join_cluster: + config._replace(discovery_spi=from_ignite_cluster(join_cluster)) - servers = IgniteService(self.test_context, num_nodes=num_nodes, version=ignite_version, - properties=self.properties(ignite_version)) + servers = IgniteService(self.test_context, config=config, num_nodes=num_nodes) servers.start(timeout_sec=timeout_sec) diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 63f6ef889a6ff..6daaa2db363e2 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -20,18 +20,33 @@ import random import re from datetime import datetime +from time import monotonic +from typing import NamedTuple from ducktape.mark import matrix from ducktape.mark.resource import cluster -from jinja2 import Template from ignitetest.services.ignite import IgniteAwareService from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_zookeeper_cluster, from_ignite_cluster, \ + TcpDiscoverySpi from ignitetest.services.utils.time_utils import epoch_mills from ignitetest.services.zk.zookeeper import ZookeeperService from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH, LATEST_2_8 +from ignitetest.utils.version import DEV_BRANCH, LATEST_2_8, IgniteVersion + + +class DiscoveryTestConfig(NamedTuple): + """ + Configuration for DiscoveryTest. + """ + version: IgniteVersion + nodes_to_kill: int = 1 + kill_coordinator: bool = False + with_load: bool = False + with_zk: bool = False # pylint: disable=W0223 @@ -42,191 +57,167 @@ class DiscoveryTest(IgniteTest): 2. Kill random node. 3. Wait that survived node detects node failure. """ - class Config: - """ - Configuration for DiscoveryTest. - """ - def __init__(self, nodes_to_kill=1, kill_coordinator=False, with_load=False): - self.nodes_to_kill = nodes_to_kill - self.kill_coordinator = kill_coordinator - self.with_load = with_load - NUM_NODES = 7 FAILURE_DETECTION_TIMEOUT = 2000 DATA_AMOUNT = 100000 - CONFIG_TEMPLATE = """ - - {% if zookeeper_settings %} - {% with zk = zookeeper_settings %} - - - - - - - {% endwith %} - {% endif %} - """ - - def __init__(self, test_context): - super().__init__(test_context=test_context) - self.zk_quorum = None - self.servers = None - self.loader = None - @cluster(num_nodes=NUM_NODES) - @matrix(ignite_version=[str(DEV_BRANCH), str(LATEST_2_8)], + @matrix(version=[str(DEV_BRANCH), str(LATEST_2_8)], kill_coordinator=[False, True], nodes_to_kill=[1, 2], with_load=[False, True]) - def test_tcp(self, ignite_version, kill_coordinator, nodes_to_kill, with_load): + def test_node_fail_tcp(self, version, kill_coordinator, nodes_to_kill, with_load): """ Test nodes failure scenario with TcpDiscoverySpi. """ - config = DiscoveryTest.Config(nodes_to_kill, kill_coordinator, with_load) + test_config = DiscoveryTestConfig(version=IgniteVersion(version), kill_coordinator=kill_coordinator, + nodes_to_kill=nodes_to_kill, with_load=with_load, with_zk=False) - return self.__simulate_nodes_failure(ignite_version, self.__properties(), None, config) + return self._perform_node_fail_scenario(test_config) @cluster(num_nodes=NUM_NODES + 3) - @matrix(ignite_version=[str(DEV_BRANCH), str(LATEST_2_8)], + @matrix(version=[str(DEV_BRANCH), str(LATEST_2_8)], kill_coordinator=[False, True], nodes_to_kill=[1, 2], with_load=[False, True]) - def test_zk(self, ignite_version, kill_coordinator, nodes_to_kill, with_load): + def test_node_fail_zk(self, version, kill_coordinator, nodes_to_kill, with_load): """ Test node failure scenario with ZooKeeperSpi. """ - config = DiscoveryTest.Config(nodes_to_kill, kill_coordinator, with_load) + test_config = DiscoveryTestConfig(version=IgniteVersion(version), kill_coordinator=kill_coordinator, + nodes_to_kill=nodes_to_kill, with_load=with_load, with_zk=True) - self.__start_zk_quorum() + return self._perform_node_fail_scenario(test_config) - properties = self.__zk_properties(self.zk_quorum.connection_string()) - modules = ["zookeeper"] + def _perform_node_fail_scenario(self, test_config): + modules = ['zookeeper'] if test_config.with_zk else None - return self.__simulate_nodes_failure(ignite_version, properties, modules, config) + if test_config.with_zk: + zk_quorum = start_zookeeper(self.test_context, 3) - def setUp(self): - pass + discovery_spi = from_zookeeper_cluster(zk_quorum) + else: + discovery_spi = TcpDiscoverySpi() - def teardown(self): - if self.loader: - self.loader.stop() + ignite_config = IgniteConfiguration( + version=test_config.version, + discovery_spi=discovery_spi, + failure_detection_timeout=self.FAILURE_DETECTION_TIMEOUT + ) - if self.servers: - self.servers.stop() + servers, start_servers_sec = start_servers(self.test_context, self.NUM_NODES - 1, ignite_config, modules) - if self.zk_quorum: - self.zk_quorum.stop() + if test_config.with_load: + load_config = ignite_config._replace(client_mode=True) if test_config.with_zk else \ + ignite_config._replace(client_mode=True, discovery_spi=from_ignite_cluster(servers)) - def __simulate_nodes_failure(self, version, properties, modules, config): - if config.nodes_to_kill < 1: - return {"No nodes to kill": "Nothing to do"} + start_load_app(self.test_context, ignite_config=load_config, data_amount=self.DATA_AMOUNT, modules=modules) - self.servers = IgniteService( - self.test_context, - num_nodes=self.NUM_NODES - 1, - modules=modules, - properties=properties, - version=version) - - time_holder = self.monotonic() - - self.servers.start() + data = simulate_nodes_failure(servers, test_config.kill_coordinator, test_config.nodes_to_kill) + data['Ignite cluster start time (s)'] = start_servers_sec + return data - data = {'Ignite cluster start time (s)': round(self.monotonic() - time_holder, 1)} - failed_nodes, survived_node = self.__choose_node_to_kill(config.kill_coordinator, config.nodes_to_kill) +def start_zookeeper(test_context, num_nodes): + """ + Start zookeeper cluster. + """ + zk_quorum = ZookeeperService(test_context, num_nodes) + zk_quorum.start() + return zk_quorum - ids_to_wait = [node.discovery_info().node_id for node in failed_nodes] - if config.with_load: - self.__start_loading(version, properties, modules) +def start_servers(test_context, num_nodes, ignite_config, modules=None): + """ + Start ignite servers. + """ + servers = IgniteService(test_context, config=ignite_config, num_nodes=num_nodes, modules=modules, + # mute spam in log. + jvm_opts=["-DIGNITE_DUMP_THREADS_ON_FAILURE=false"]) - first_terminated = self.servers.stop_nodes_async(failed_nodes, clean_shutdown=False, wait_for_stop=False) + start = monotonic() + servers.start() + return servers, round(monotonic() - start, 1) - # Keeps dates of logged node failures. - logged_timestamps = [] - for failed_id in ids_to_wait: - self.servers.await_event_on_node(self.__failed_pattern(failed_id), survived_node, 20, - from_the_beginning=True, backoff_sec=0.1) +def start_load_app(test_context, ignite_config, data_amount, modules=None): + """ + Start loader application. + """ + loader = IgniteApplicationService( + test_context, + config=ignite_config, + java_class_name="org.apache.ignite.internal.ducktest.tests.ContinuousDataLoadApplication", + modules=modules, + # mute spam in log. + jvm_opts=["-DIGNITE_DUMP_THREADS_ON_FAILURE=false"], + params={"cacheName": "test-cache", "range": data_amount}) - _, stdout, _ = survived_node.account.ssh_client.exec_command( - "grep '%s' %s" % (self.__failed_pattern(failed_id), IgniteAwareService.STDOUT_STDERR_CAPTURE)) + loader.start() - logged_timestamps.append( - datetime.strptime(re.match("^\\[[^\\[]+\\]", stdout.read().decode("utf-8")).group(), - "[%Y-%m-%d %H:%M:%S,%f]")) - logged_timestamps.sort(reverse=True) +def failed_pattern(failed_node_id): + """ + Failed node pattern in log + """ + return "Node FAILED: .\\{1,\\}Node \\[id=" + failed_node_id - self.__store_results(data, logged_timestamps, first_terminated[1]) - data['Nodes failed'] = len(failed_nodes) +def choose_node_to_kill(servers, kill_coordinator, nodes_to_kill): + """Choose node to kill during test""" + assert nodes_to_kill > 0, " No nodes to kill passed. Check the parameters." - return data + nodes = servers.nodes + coordinator = nodes[0].discovery_info().coordinator + to_kill = [] - @staticmethod - def __store_results(data, logged_timestamps, first_kill_time): - first_kill_time = epoch_mills(first_kill_time) + if kill_coordinator: + to_kill.append(next(node for node in nodes if node.discovery_info().node_id == coordinator)) + nodes_to_kill -= 1 - detection_delay = epoch_mills(logged_timestamps[0]) - first_kill_time + if nodes_to_kill > 0: + choice = random.sample([n for n in nodes if n.discovery_info().node_id != coordinator], nodes_to_kill) + to_kill.extend([choice] if not isinstance(choice, list) else choice) - data['Detection of node(s) failure (ms)'] = detection_delay - data['All detection delays (ms):'] = str([epoch_mills(ts) - first_kill_time for ts in logged_timestamps]) + survive = random.choice([node for node in servers.nodes if node not in to_kill]) - @staticmethod - def __failed_pattern(failed_node_id): - return "Node FAILED: .\\{1,\\}Node \\[id=" + failed_node_id + return to_kill, survive - def __choose_node_to_kill(self, kill_coordinator, nodes_to_kill): - assert nodes_to_kill > 0, "No nodes to kill passed. Check the parameters." - nodes = self.servers.nodes - coordinator = nodes[0].discovery_info().coordinator - to_kill = [] +def simulate_nodes_failure(servers, kill_coordinator, nodes_to_kill): + """ + Perform node failure scenario + """ + failed_nodes, survived_node = choose_node_to_kill(servers, kill_coordinator, nodes_to_kill) - if kill_coordinator: - to_kill.append(next(node for node in nodes if node.discovery_info().node_id == coordinator)) - nodes_to_kill -= 1 + ids_to_wait = [node.discovery_info().node_id for node in failed_nodes] - if nodes_to_kill > 0: - choice = random.sample([n for n in nodes if n.discovery_info().node_id != coordinator], nodes_to_kill) - to_kill.extend([choice] if not isinstance(choice, list) else choice) + _, first_terminated = servers.stop_nodes_async(failed_nodes, clean_shutdown=False, wait_for_stop=False) - survive = random.choice([node for node in self.servers.nodes if node not in to_kill]) + # Keeps dates of logged node failures. + logged_timestamps = [] + data = {} - return to_kill, survive + for failed_id in ids_to_wait: + servers.await_event_on_node(failed_pattern(failed_id), survived_node, 20, + from_the_beginning=True, backoff_sec=0.1) - def __start_loading(self, ignite_version, properties, modules): - self.loader = IgniteApplicationService( - self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.tests.ContinuousDataLoadApplication", - version=ignite_version, - modules=modules, - properties=properties, - params={"cacheName": "test-cache", "range": self.DATA_AMOUNT}) + _, stdout, _ = survived_node.account.ssh_client.exec_command( + "grep '%s' %s" % (failed_pattern(failed_id), IgniteAwareService.STDOUT_STDERR_CAPTURE)) - self.loader.start() + logged_timestamps.append( + datetime.strptime(re.match("^\\[[^\\[]+\\]", stdout.read().decode("utf-8")).group(), + "[%Y-%m-%d %H:%M:%S,%f]")) - def __start_zk_quorum(self): - self.zk_quorum = ZookeeperService(self.test_context, 3) + logged_timestamps.sort(reverse=True) - self.zk_quorum.start() + first_kill_time = epoch_mills(first_terminated) + detection_delay = epoch_mills(logged_timestamps[0]) - first_kill_time - @staticmethod - def __properties(zookeeper_settings=None): - """ - :param zookeeper_settings: ZooKeeperDiscoverySpi settings. If None, TcpDiscoverySpi will be used. - :return: Rendered node's properties. - """ - return Template(DiscoveryTest.CONFIG_TEMPLATE) \ - .render(failure_detection_timeout=DiscoveryTest.FAILURE_DETECTION_TIMEOUT, - zookeeper_settings=zookeeper_settings) + data['Detection of node(s) failure (ms)'] = detection_delay + data['All detection delays (ms):'] = str([epoch_mills(ts) - first_kill_time for ts in logged_timestamps]) + data['Nodes failed'] = len(failed_nodes) - @staticmethod - def __zk_properties(connection_string): - return DiscoveryTest.__properties(zookeeper_settings={'connection_string': connection_string}) + return data diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index ea266c3e2baba..8ec2daa749ad5 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -25,6 +25,9 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration +from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion @@ -36,29 +39,6 @@ class PmeFreeSwitchTest(IgniteTest): """ NUM_NODES = 3 - @staticmethod - def properties(): - """ - :return: Rendered configuration properties. - """ - return """ - - - - - - - - - - """ - - def setUp(self): - pass - - def teardown(self): - pass - @cluster(num_nodes=NUM_NODES + 2) @parametrize(version=str(DEV_BRANCH)) @parametrize(version=str(LATEST_2_7)) @@ -72,22 +52,25 @@ def test(self, version): ignite_version = IgniteVersion(version) - ignites = IgniteService( - self.test_context, - num_nodes=self.NUM_NODES, - properties=self.properties(), - version=ignite_version) + config = IgniteConfiguration( + version=ignite_version, + caches=[CacheConfiguration(name='test-cache', backups=2, atomicity_mode='TRANSACTIONAL')] + ) + + ignites = IgniteService(self.test_context, config, num_nodes=self.NUM_NODES) ignites.start() self.stage("Starting long_tx_streamer") + client_config = config._replace(client_mode=True, + discovery_spi=from_ignite_cluster(ignites, slice(0, self.NUM_NODES - 1))) + long_tx_streamer = IgniteApplicationService( self.test_context, + client_config, java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.LongTxStreamerApplication", - properties=self.properties(), - params={"cacheName": "test-cache"}, - version=ignite_version) + params={"cacheName": "test-cache"}) long_tx_streamer.start() @@ -95,11 +78,10 @@ def test(self, version): single_key_tx_streamer = IgniteApplicationService( self.test_context, + client_config, java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test." "SingleKeyTxStreamerApplication", - properties=self.properties(), - params={"cacheName": "test-cache", "warmup": 1000}, - version=ignite_version) + params={"cacheName": "test-cache", "warmup": 1000}) single_key_tx_streamer.start() @@ -108,7 +90,7 @@ def test(self, version): self.stage("Stopping server node") - ignites.stop_node(ignites.nodes[1]) + ignites.stop_node(ignites.nodes[self.NUM_NODES - 1]) long_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True) diff --git a/modules/ducktests/tests/ignitetest/tests/smoke_test.py b/modules/ducktests/tests/ignitetest/tests/smoke_test.py index 48965e366e57f..d7207a23940e9 100644 --- a/modules/ducktests/tests/ignitetest/tests/smoke_test.py +++ b/modules/ducktests/tests/ignitetest/tests/smoke_test.py @@ -18,13 +18,16 @@ """ from ducktape.mark import parametrize +from ducktape.mark.resource import cluster from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.spark import SparkService +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteClientConfiguration from ignitetest.services.zk.zookeeper import ZookeeperService from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion # pylint: disable=W0223 @@ -33,44 +36,35 @@ class SmokeServicesTest(IgniteTest): Tests services implementations """ - def setUp(self): - pass - - def teardown(self): - pass - + @cluster(num_nodes=1) @parametrize(version=str(DEV_BRANCH)) def test_ignite_start_stop(self, version): """ Test that IgniteService correctly start and stop """ - ignite = IgniteService( - self.test_context, - num_nodes=1, - version=version) + ignite = IgniteService(self.test_context, IgniteConfiguration(version=IgniteVersion(version)), num_nodes=1) ignite.start() ignite.stop() + @cluster(num_nodes=2) @parametrize(version=str(DEV_BRANCH)) def test_ignite_app_start_stop(self, version): """ Test that IgniteService and IgniteApplicationService correctly start and stop """ - ignite = IgniteService( - self.test_context, - num_nodes=1, - version=version) + ignite = IgniteService(self.test_context, IgniteConfiguration(version=IgniteVersion(version)), num_nodes=1) app = IgniteApplicationService( self.test_context, - java_class_name="org.apache.ignite.internal.ducktest.tests.smoke_test.SimpleApplication", - version=version) + IgniteClientConfiguration(version=IgniteVersion(version), discovery_spi=from_ignite_cluster(ignite)), + java_class_name="org.apache.ignite.internal.ducktest.tests.smoke_test.SimpleApplication") ignite.start() app.start() app.stop() ignite.stop() + @cluster(num_nodes=2) def test_spark_start_stop(self): """ Test that SparkService correctly start and stop @@ -79,10 +73,11 @@ def test_spark_start_stop(self): spark.start() spark.stop() + @cluster(num_nodes=3) def test_zk_start_stop(self): """ Test that ZookeeperService correctly start and stop """ - zookeeper = ZookeeperService(self.test_context, num_nodes=2) + zookeeper = ZookeeperService(self.test_context, num_nodes=3) zookeeper.start() zookeeper.stop() diff --git a/modules/ducktests/tests/ignitetest/tests/suites/fast_suite.yml b/modules/ducktests/tests/ignitetest/tests/suites/fast_suite.yml new file mode 100644 index 0000000000000..4811574248a8d --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/suites/fast_suite.yml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +smoke: + - ../smoke_test.py + +control_utility: + - ../control_utility_test.py + +pme_free_switch: + - ../pme_free_switch_test.py + +cellular_affinity: + - ../cellular_affinity_test.py + +rebalance: + - ../add_node_rebalance_test.py diff --git a/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml b/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml new file mode 100644 index 0000000000000..6f066d50889a6 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +discovery: + - ../discovery_test.py diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index 76dd8155bc6f0..5b89d58314572 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -31,7 +31,6 @@ packages=find_packages(exclude=["ignitetest.tests", "ignitetest.tests.*"]), include_package_data=True, install_requires=['ducktape==0.8.0'], - dependency_links = [ + dependency_links=[ 'https://github.com/confluentinc/ducktape/tarball/master#egg=ducktape-0.8.0' - ] -) + ]) From 793f0569002c7cb00119bf8df641e8c0167a19cb Mon Sep 17 00:00:00 2001 From: Vladsz83 Date: Fri, 4 Sep 2020 12:08:11 +0300 Subject: [PATCH 48/78] Transaction to the discovery test. (#8194) --- .../tests/ContinuousDataLoadApplication.java | 136 +++++++++++++++--- .../tests/ignitetest/tests/discovery_test.py | 63 +++++--- 2 files changed, 162 insertions(+), 37 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java index 46a4b76f130a6..766ede661edaa 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java @@ -17,11 +17,20 @@ package org.apache.ignite.internal.ducktest.tests; -import java.util.Random; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.transactions.Transaction; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -32,43 +41,130 @@ public class ContinuousDataLoadApplication extends IgniteAwareApplication { /** Logger. */ private static final Logger log = LogManager.getLogger(ContinuousDataLoadApplication.class.getName()); - /** {@inheritDoc} */ - @Override protected void run(JsonNode jsonNode) { - String cacheName = jsonNode.get("cacheName").asText(); - int range = jsonNode.get("range").asInt(); + /** */ + private IgniteCache cache; + + /** Node set to exclusively put data on if required. */ + private List nodesToLoad = Collections.emptyList(); - IgniteCache cache = ignite.getOrCreateCache(cacheName); + /** */ + private Affinity aff; - int warmUpCnt = (int)Math.max(1, 0.1f * range); + /** Data number to put before notifying of the initialized state. */ + private int warmUpCnt; - Random rnd = new Random(); + /** {@inheritDoc} */ + @Override protected void run(JsonNode jsonNode) { + Config cfg = parseConfig(jsonNode); - long streamed = 0; + init(cfg); log.info("Generating data in background..."); long notifyTime = System.nanoTime(); + int loaded = 0; + while (active()) { - cache.put(rnd.nextInt(range), rnd.nextInt(range)); + try (Transaction tx = cfg.transactional ? ignite.transactions().txStart() : null) { + for (int i = 0; i < cfg.range && active(); ++i) { + if (skipDataKey(i)) + continue; - streamed++; + cache.put(i, i); - if (notifyTime + U.millisToNanos(1500) < System.nanoTime()) { - notifyTime = System.nanoTime(); + ++loaded; - if (log.isDebugEnabled()) - log.debug("Streamed " + streamed + " entries."); - } + if (notifyTime + U.millisToNanos(1500) < System.nanoTime()) + notifyTime = System.nanoTime(); - // Delayed notify of the initialization to make sure the data load has completelly began and - // has produced some valuable amount of data. - if (!inited() && warmUpCnt == streamed) - markInitialized(); + // Delayed notify of the initialization to make sure the data load has completelly began and + // has produced some valuable amount of data. + if (!inited() && warmUpCnt == loaded) + markInitialized(); + } + + if (tx != null && active()) + tx.commit(); + } } log.info("Background data generation finished."); markFinished(); } + + /** + * @return {@code True} if data should not be put for {@code dataKey}. {@code False} otherwise. + */ + private boolean skipDataKey(int dataKey) { + if (!nodesToLoad.isEmpty()) { + for (ClusterNode n : nodesToLoad) { + if (aff.isPrimary(n, dataKey)) + return false; + } + + return true; + } + + return false; + } + + /** + * Prepares run settings based on {@code cfg}. + */ + private void init(Config cfg) { + cache = ignite.getOrCreateCache(cfg.cacheName); + + if (cfg.targetNodes != null && !cfg.targetNodes.isEmpty()) { + nodesToLoad = ignite.cluster().nodes().stream().filter(n -> cfg.targetNodes.contains(n.id().toString())) + .collect(Collectors.toList()); + + aff = ignite.affinity(cfg.cacheName); + } + + warmUpCnt = cfg.warmUpRange < 1 ? (int)Math.max(1, 0.1f * cfg.range) : cfg.warmUpRange; + } + + /** + * Converts Json-represented config into {@code Config}. + */ + private static Config parseConfig(JsonNode node) { + ObjectMapper objMapper = new ObjectMapper(); + objMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + + Config cfg; + + try { + cfg = objMapper.treeToValue(node, Config.class); + } + catch (Exception e) { + throw new IllegalStateException("Unable to parse config.", e); + } + + return cfg; + } + + /** + * The configuration holder. + */ + private static class Config { + /** Name of the cache. */ + private String cacheName; + + /** Data/keys number to load. */ + private int range; + + /** Node id set. If not empty, data will be load only on this nodes. */ + private Set targetNodes; + + /** If {@code true}, data will be put within transaction. */ + private boolean transactional; + + /** + * Data number to warn-up and to delay the init-notification. If < 1, ignored and considered default 10% of + * {@code range}. + */ + private int warmUpRange; + } } diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 6daaa2db363e2..9b28e97775fa9 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -19,6 +19,7 @@ import random import re +from enum import IntEnum from datetime import datetime from time import monotonic from typing import NamedTuple @@ -30,6 +31,7 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.utils.ignite_configuration import IgniteConfiguration +from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration from ignitetest.services.utils.ignite_configuration.discovery import from_zookeeper_cluster, from_ignite_cluster, \ TcpDiscoverySpi from ignitetest.services.utils.time_utils import epoch_mills @@ -38,6 +40,15 @@ from ignitetest.utils.version import DEV_BRANCH, LATEST_2_8, IgniteVersion +class ClusterLoad(IntEnum): + """ + Type of cluster loading. + """ + none = 0 + atomic = 1 + transactional = 2 + + class DiscoveryTestConfig(NamedTuple): """ Configuration for DiscoveryTest. @@ -45,7 +56,7 @@ class DiscoveryTestConfig(NamedTuple): version: IgniteVersion nodes_to_kill: int = 1 kill_coordinator: bool = False - with_load: bool = False + load_type: ClusterLoad = ClusterLoad.none with_zk: bool = False @@ -61,19 +72,22 @@ class DiscoveryTest(IgniteTest): FAILURE_DETECTION_TIMEOUT = 2000 - DATA_AMOUNT = 100000 + DATA_AMOUNT = 5_000_000 + + WARMUP_DATA_AMOUNT = 10_000 @cluster(num_nodes=NUM_NODES) @matrix(version=[str(DEV_BRANCH), str(LATEST_2_8)], kill_coordinator=[False, True], nodes_to_kill=[1, 2], - with_load=[False, True]) - def test_node_fail_tcp(self, version, kill_coordinator, nodes_to_kill, with_load): + load_type=[ClusterLoad.none, ClusterLoad.atomic, ClusterLoad.transactional]) + def test_node_fail_tcp(self, version, kill_coordinator, nodes_to_kill, load_type): """ Test nodes failure scenario with TcpDiscoverySpi. + :param load_type: How to load cluster during the test: 0 - no loading; 1 - do some loading; 2 - transactional. """ test_config = DiscoveryTestConfig(version=IgniteVersion(version), kill_coordinator=kill_coordinator, - nodes_to_kill=nodes_to_kill, with_load=with_load, with_zk=False) + nodes_to_kill=nodes_to_kill, load_type=load_type, with_zk=False) return self._perform_node_fail_scenario(test_config) @@ -81,13 +95,14 @@ def test_node_fail_tcp(self, version, kill_coordinator, nodes_to_kill, with_load @matrix(version=[str(DEV_BRANCH), str(LATEST_2_8)], kill_coordinator=[False, True], nodes_to_kill=[1, 2], - with_load=[False, True]) - def test_node_fail_zk(self, version, kill_coordinator, nodes_to_kill, with_load): + load_type=[ClusterLoad.none, ClusterLoad.atomic, ClusterLoad.transactional]) + def test_node_fail_zk(self, version, kill_coordinator, nodes_to_kill, load_type): """ Test node failure scenario with ZooKeeperSpi. + :param load_type: How to load cluster during the test: 0 - no loading; 1 - do some loading; 2 - transactional. """ test_config = DiscoveryTestConfig(version=IgniteVersion(version), kill_coordinator=kill_coordinator, - nodes_to_kill=nodes_to_kill, with_load=with_load, with_zk=True) + nodes_to_kill=nodes_to_kill, load_type=load_type, with_zk=True) return self._perform_node_fail_scenario(test_config) @@ -104,19 +119,35 @@ def _perform_node_fail_scenario(self, test_config): ignite_config = IgniteConfiguration( version=test_config.version, discovery_spi=discovery_spi, - failure_detection_timeout=self.FAILURE_DETECTION_TIMEOUT + failure_detection_timeout=self.FAILURE_DETECTION_TIMEOUT, + caches=[CacheConfiguration(name='test-cache', backups=1, atomicity_mode='TRANSACTIONAL' if + test_config.load_type == ClusterLoad.transactional else 'ATOMIC')] ) servers, start_servers_sec = start_servers(self.test_context, self.NUM_NODES - 1, ignite_config, modules) - if test_config.with_load: + failed_nodes, survived_node = choose_node_to_kill(servers, test_config.kill_coordinator, + test_config.nodes_to_kill) + + if test_config.load_type is not ClusterLoad.none: load_config = ignite_config._replace(client_mode=True) if test_config.with_zk else \ ignite_config._replace(client_mode=True, discovery_spi=from_ignite_cluster(servers)) - start_load_app(self.test_context, ignite_config=load_config, data_amount=self.DATA_AMOUNT, modules=modules) + tran_nodes = [n.discovery_info().node_id for n in failed_nodes] \ + if test_config.load_type == ClusterLoad.transactional else None + + params = {"cacheName": "test-cache", + "range": self.DATA_AMOUNT, + "warmUpRange": self.WARMUP_DATA_AMOUNT, + "targetNodes": tran_nodes, + "transactional": bool(tran_nodes)} + + start_load_app(self.test_context, ignite_config=load_config, params=params, modules=modules) + + data = simulate_nodes_failure(servers, failed_nodes, survived_node) - data = simulate_nodes_failure(servers, test_config.kill_coordinator, test_config.nodes_to_kill) data['Ignite cluster start time (s)'] = start_servers_sec + return data @@ -142,7 +173,7 @@ def start_servers(test_context, num_nodes, ignite_config, modules=None): return servers, round(monotonic() - start, 1) -def start_load_app(test_context, ignite_config, data_amount, modules=None): +def start_load_app(test_context, ignite_config, params, modules=None): """ Start loader application. """ @@ -153,7 +184,7 @@ def start_load_app(test_context, ignite_config, data_amount, modules=None): modules=modules, # mute spam in log. jvm_opts=["-DIGNITE_DUMP_THREADS_ON_FAILURE=false"], - params={"cacheName": "test-cache", "range": data_amount}) + params=params) loader.start() @@ -186,12 +217,10 @@ def choose_node_to_kill(servers, kill_coordinator, nodes_to_kill): return to_kill, survive -def simulate_nodes_failure(servers, kill_coordinator, nodes_to_kill): +def simulate_nodes_failure(servers, failed_nodes, survived_node): """ Perform node failure scenario """ - failed_nodes, survived_node = choose_node_to_kill(servers, kill_coordinator, nodes_to_kill) - ids_to_wait = [node.discovery_info().node_id for node in failed_nodes] _, first_terminated = servers.stop_nodes_async(failed_nodes, clean_shutdown=False, wait_for_stop=False) From 3c537ca7447b05a6f0adb29bf2b2eb700b04bd69 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Fri, 4 Sep 2020 16:15:13 +0300 Subject: [PATCH 49/78] =?UTF-8?q?Implements=20@ignite=5Fversions=20decorat?= =?UTF-8?q?or.=20Add=20support=20to=20override=20parame=E2=80=A6=20(#8213)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/ContinuousDataLoadApplication.java | 4 +- modules/ducktests/tests/docker/run_tests.sh | 5 + .../tests/add_node_rebalance_test.py | 11 +-- .../tests/cellular_affinity_test.py | 16 +-- .../ignitetest/tests/control_utility_test.py | 45 ++++----- .../tests/ignitetest/tests/discovery_test.py | 38 ++++---- .../ignitetest/tests/pme_free_switch_test.py | 13 +-- .../tests/ignitetest/tests/smoke_test.py | 24 +++-- .../tests/ignitetest/utils/__init__.py | 2 +- .../ducktests/tests/ignitetest/utils/_mark.py | 97 +++++++++++++++++-- 10 files changed, 170 insertions(+), 85 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java index 766ede661edaa..634bdde4aa997 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; @@ -29,7 +30,6 @@ import org.apache.ignite.cache.affinity.Affinity; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; -import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.transactions.Transaction; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -75,7 +75,7 @@ public class ContinuousDataLoadApplication extends IgniteAwareApplication { ++loaded; - if (notifyTime + U.millisToNanos(1500) < System.nanoTime()) + if (notifyTime + TimeUnit.MILLISECONDS.toNanos(1500) < System.nanoTime()) notifyTime = System.nanoTime(); // Delayed notify of the initialization to make sure the data load has completelly began and diff --git a/modules/ducktests/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh index b9ee4887b87bd..d8eb270297c20 100755 --- a/modules/ducktests/tests/docker/run_tests.sh +++ b/modules/ducktests/tests/docker/run_tests.sh @@ -76,6 +76,10 @@ The options are as follows: - ignite_client_config_path: abs path within container to Ignite client config template - ignite_server_config_path: abs path within container to Ignite server config template - jvm_opts: array of JVM options to use when Ignite node started + - ignite_version: string representing ignite_versions to test against. + +-gj|--global-json) + Use specified json as globals to pass to test context. Can be extended with -g|--global -t|--tc-paths Path to ducktests. Must be relative path to 'IGNITE/modules/ducktests/tests' directory @@ -119,6 +123,7 @@ while [[ $# -ge 1 ]]; do -p|--param) duck_add_param "$2"; shift 2;; -pj|--params-json) PARAMETERS="$2"; shift 2;; -g|--global) duck_add_global "$2"; shift 2;; + -gj|--global-json) GLOBALS="$2"; shift 2;; -t|--tc-paths) TC_PATHS="$2"; shift 2;; -n|--num-nodes) IGNITE_NUM_CONTAINERS="$2"; shift 2;; -j|--max-parallel) MAX_PARALLEL="$2"; shift 2;; diff --git a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py index b162d2e79203c..068142405f190 100644 --- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py @@ -17,13 +17,13 @@ Module contains node rebalance tests. """ -from ducktape.mark import parametrize from ducktape.mark.resource import cluster from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.utils.ignite_configuration import IgniteConfiguration from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.utils import ignite_versions from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, IgniteVersion, LATEST @@ -39,20 +39,17 @@ class AddNodeRebalanceTest(IgniteTest): REBALANCE_TIMEOUT = 60 @cluster(num_nodes=NUM_NODES + 1) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST)) - def test_add_node(self, version): + @ignite_versions(str(DEV_BRANCH), str(LATEST)) + def test_add_node(self, ignite_version): """ Test performs add node rebalance test which consists of following steps: * Start cluster. * Put data to it via IgniteClientApp. * Start one more node and awaits for rebalance to finish. """ - ignite_version = IgniteVersion(version) - self.stage("Start Ignite nodes") - node_config = IgniteConfiguration(version=ignite_version) + node_config = IgniteConfiguration(version=IgniteVersion(ignite_version)) ignites = IgniteService(self.test_context, config=node_config, num_nodes=self.NUM_NODES - 1) ignites.start() diff --git a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py index 6b1e6647c0f23..c957e43989c55 100644 --- a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py +++ b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py @@ -17,7 +17,6 @@ This module contains Cellular Affinity tests. """ -from ducktape.mark import parametrize from ducktape.mark.resource import cluster from jinja2 import Template @@ -25,6 +24,7 @@ from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteClientConfiguration from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.utils import ignite_versions, version_if from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, IgniteVersion @@ -71,18 +71,20 @@ def properties(): cacheName=CellularAffinity.CACHE_NAME) @cluster(num_nodes=NUM_NODES * 3 + 1) - @parametrize(version=str(DEV_BRANCH)) - def test(self, version): + @version_if(lambda version: version >= DEV_BRANCH) + @ignite_versions(str(DEV_BRANCH)) + def test(self, ignite_version): """ Test Cellular Affinity scenario (partition distribution). """ - cell1 = self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=1']) - self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=2'], joined_cluster=cell1) - self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=XXX', '-DRANDOM=42'], joined_cluster=cell1) + cell1 = self.start_cell(ignite_version, ['-D' + CellularAffinity.ATTRIBUTE + '=1']) + self.start_cell(ignite_version, ['-D' + CellularAffinity.ATTRIBUTE + '=2'], joined_cluster=cell1) + self.start_cell(ignite_version, ['-D' + CellularAffinity.ATTRIBUTE + '=XXX', '-DRANDOM=42'], + joined_cluster=cell1) checker = IgniteApplicationService( self.test_context, - IgniteClientConfiguration(version=IgniteVersion(version), discovery_spi=from_ignite_cluster(cell1)), + IgniteClientConfiguration(version=IgniteVersion(ignite_version), discovery_spi=from_ignite_cluster(cell1)), java_class_name="org.apache.ignite.internal.ducktest.tests.cellular_affinity_test.DistributionChecker", params={"cacheName": CellularAffinity.CACHE_NAME, "attr": CellularAffinity.ATTRIBUTE, diff --git a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py index a4afd092ec82c..a30a381521b92 100644 --- a/modules/ducktests/tests/ignitetest/tests/control_utility_test.py +++ b/modules/ducktests/tests/ignitetest/tests/control_utility_test.py @@ -16,7 +16,7 @@ """ This module contains control.sh utility tests. """ -from ducktape.mark import parametrize + from ducktape.mark.resource import cluster from ducktape.utils.util import wait_until @@ -25,7 +25,7 @@ from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster -from ignitetest.utils import version_if +from ignitetest.utils import version_if, ignite_versions from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, LATEST_2_8, IgniteVersion, LATEST_2_7, V_2_8_0 @@ -38,15 +38,13 @@ class BaselineTests(IgniteTest): NUM_NODES = 3 @cluster(num_nodes=NUM_NODES) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_8)) - @parametrize(version=str(LATEST_2_7)) - def test_baseline_set(self, version): + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8), str(LATEST_2_7)) + def test_baseline_set(self, ignite_version): """ Test baseline set. """ blt_size = self.NUM_NODES - 2 - servers = self.__start_ignite_nodes(version, blt_size) + servers = self.__start_ignite_nodes(ignite_version, blt_size) control_utility = ControlUtility(servers, self.test_context) control_utility.activate() @@ -57,7 +55,7 @@ def test_baseline_set(self, version): self.__check_nodes_in_baseline(servers.nodes, baseline) # Set baseline using list of consisttent ids. - new_node = self.__start_ignite_nodes(version, 1, join_cluster=servers) + new_node = self.__start_ignite_nodes(ignite_version, 1, join_cluster=servers) control_utility.set_baseline(servers.nodes + new_node.nodes) blt_size += 1 @@ -66,7 +64,7 @@ def test_baseline_set(self, version): self.__check_nodes_in_baseline(new_node.nodes, baseline) # Set baseline using topology version. - new_node = self.__start_ignite_nodes(version, 1, join_cluster=servers) + new_node = self.__start_ignite_nodes(ignite_version, 1, join_cluster=servers) _, version, _ = control_utility.cluster_state() control_utility.set_baseline(version) blt_size += 1 @@ -76,22 +74,20 @@ def test_baseline_set(self, version): self.__check_nodes_in_baseline(new_node.nodes, baseline) @cluster(num_nodes=NUM_NODES) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_8)) - @parametrize(version=str(LATEST_2_7)) - def test_baseline_add_remove(self, version): + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8), str(LATEST_2_7)) + def test_baseline_add_remove(self, ignite_version): """ Test add and remove nodes from baseline. """ blt_size = self.NUM_NODES - 1 - servers = self.__start_ignite_nodes(version, blt_size) + servers = self.__start_ignite_nodes(ignite_version, blt_size) control_utility = ControlUtility(servers, self.test_context) control_utility.activate() # Add node to baseline. - new_node = self.__start_ignite_nodes(version, 1, join_cluster=servers) + new_node = self.__start_ignite_nodes(ignite_version, 1, join_cluster=servers) control_utility.add_to_baseline(new_node.nodes) blt_size += 1 @@ -120,14 +116,12 @@ def test_baseline_add_remove(self, version): self.__check_nodes_not_in_baseline(new_node.nodes, baseline) @cluster(num_nodes=NUM_NODES) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_8)) - @parametrize(version=str(LATEST_2_7)) - def test_activate_deactivate(self, version): + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8), str(LATEST_2_7)) + def test_activate_deactivate(self, ignite_version): """ Test activate and deactivate cluster. """ - servers = self.__start_ignite_nodes(version, self.NUM_NODES) + servers = self.__start_ignite_nodes(ignite_version, self.NUM_NODES) control_utility = ControlUtility(servers, self.test_context) @@ -145,21 +139,20 @@ def test_activate_deactivate(self, version): @cluster(num_nodes=NUM_NODES) @version_if(lambda version: version >= V_2_8_0) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_8)) - def test_baseline_autoadjust(self, version): + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8), str(LATEST_2_7)) + def test_baseline_autoadjust(self, ignite_version): """ Test activate and deactivate cluster. """ blt_size = self.NUM_NODES - 2 - servers = self.__start_ignite_nodes(version, blt_size) + servers = self.__start_ignite_nodes(ignite_version, blt_size) control_utility = ControlUtility(servers, self.test_context) control_utility.activate() # Add node. control_utility.enable_baseline_auto_adjust(2000) - new_node = self.__start_ignite_nodes(version, 1, join_cluster=servers) + new_node = self.__start_ignite_nodes(ignite_version, 1, join_cluster=servers) blt_size += 1 wait_until(lambda: len(control_utility.baseline()) == blt_size, timeout_sec=5) @@ -170,7 +163,7 @@ def test_baseline_autoadjust(self, version): # Add node when auto adjust disabled. control_utility.disable_baseline_auto_adjust() old_topology = control_utility.cluster_state().topology_version - new_node = self.__start_ignite_nodes(version, 1, join_cluster=servers) + new_node = self.__start_ignite_nodes(ignite_version, 1, join_cluster=servers) wait_until(lambda: control_utility.cluster_state().topology_version != old_topology, timeout_sec=5) baseline = control_utility.baseline() diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 9b28e97775fa9..8d806a46777b7 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -36,17 +36,18 @@ TcpDiscoverySpi from ignitetest.services.utils.time_utils import epoch_mills from ignitetest.services.zk.zookeeper import ZookeeperService +from ignitetest.utils import ignite_versions, version_if from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH, LATEST_2_8, IgniteVersion +from ignitetest.utils.version import DEV_BRANCH, LATEST_2_8, V_2_8_0, IgniteVersion class ClusterLoad(IntEnum): """ Type of cluster loading. """ - none = 0 - atomic = 1 - transactional = 2 + NONE = 0 + ATOMIC = 1 + TRANSACTIONAL = 2 class DiscoveryTestConfig(NamedTuple): @@ -56,7 +57,7 @@ class DiscoveryTestConfig(NamedTuple): version: IgniteVersion nodes_to_kill: int = 1 kill_coordinator: bool = False - load_type: ClusterLoad = ClusterLoad.none + load_type: ClusterLoad = ClusterLoad.NONE with_zk: bool = False @@ -77,31 +78,32 @@ class DiscoveryTest(IgniteTest): WARMUP_DATA_AMOUNT = 10_000 @cluster(num_nodes=NUM_NODES) - @matrix(version=[str(DEV_BRANCH), str(LATEST_2_8)], - kill_coordinator=[False, True], + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) + @matrix(kill_coordinator=[False, True], nodes_to_kill=[1, 2], - load_type=[ClusterLoad.none, ClusterLoad.atomic, ClusterLoad.transactional]) - def test_node_fail_tcp(self, version, kill_coordinator, nodes_to_kill, load_type): + load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) + def test_node_fail_tcp(self, ignite_version, kill_coordinator, nodes_to_kill, load_type): """ Test nodes failure scenario with TcpDiscoverySpi. :param load_type: How to load cluster during the test: 0 - no loading; 1 - do some loading; 2 - transactional. """ - test_config = DiscoveryTestConfig(version=IgniteVersion(version), kill_coordinator=kill_coordinator, + test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), kill_coordinator=kill_coordinator, nodes_to_kill=nodes_to_kill, load_type=load_type, with_zk=False) return self._perform_node_fail_scenario(test_config) @cluster(num_nodes=NUM_NODES + 3) - @matrix(version=[str(DEV_BRANCH), str(LATEST_2_8)], - kill_coordinator=[False, True], + @version_if(lambda version: version != V_2_8_0) # ignite-zookeeper package is broken in 2.8.0 + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) + @matrix(kill_coordinator=[False, True], nodes_to_kill=[1, 2], - load_type=[ClusterLoad.none, ClusterLoad.atomic, ClusterLoad.transactional]) - def test_node_fail_zk(self, version, kill_coordinator, nodes_to_kill, load_type): + load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) + def test_node_fail_zk(self, ignite_version, kill_coordinator, nodes_to_kill, load_type): """ Test node failure scenario with ZooKeeperSpi. :param load_type: How to load cluster during the test: 0 - no loading; 1 - do some loading; 2 - transactional. """ - test_config = DiscoveryTestConfig(version=IgniteVersion(version), kill_coordinator=kill_coordinator, + test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), kill_coordinator=kill_coordinator, nodes_to_kill=nodes_to_kill, load_type=load_type, with_zk=True) return self._perform_node_fail_scenario(test_config) @@ -121,7 +123,7 @@ def _perform_node_fail_scenario(self, test_config): discovery_spi=discovery_spi, failure_detection_timeout=self.FAILURE_DETECTION_TIMEOUT, caches=[CacheConfiguration(name='test-cache', backups=1, atomicity_mode='TRANSACTIONAL' if - test_config.load_type == ClusterLoad.transactional else 'ATOMIC')] + test_config.load_type == ClusterLoad.TRANSACTIONAL else 'ATOMIC')] ) servers, start_servers_sec = start_servers(self.test_context, self.NUM_NODES - 1, ignite_config, modules) @@ -129,12 +131,12 @@ def _perform_node_fail_scenario(self, test_config): failed_nodes, survived_node = choose_node_to_kill(servers, test_config.kill_coordinator, test_config.nodes_to_kill) - if test_config.load_type is not ClusterLoad.none: + if test_config.load_type is not ClusterLoad.NONE: load_config = ignite_config._replace(client_mode=True) if test_config.with_zk else \ ignite_config._replace(client_mode=True, discovery_spi=from_ignite_cluster(servers)) tran_nodes = [n.discovery_info().node_id for n in failed_nodes] \ - if test_config.load_type == ClusterLoad.transactional else None + if test_config.load_type == ClusterLoad.TRANSACTIONAL else None params = {"cacheName": "test-cache", "range": self.DATA_AMOUNT, diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index 8ec2daa749ad5..395b24bcf810c 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -19,7 +19,6 @@ import time -from ducktape.mark import parametrize from ducktape.mark.resource import cluster from ignitetest.services.ignite import IgniteService @@ -28,6 +27,7 @@ from ignitetest.services.utils.ignite_configuration import IgniteConfiguration from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.utils import ignite_versions from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, LATEST_2_7, V_2_8_0, IgniteVersion @@ -40,9 +40,8 @@ class PmeFreeSwitchTest(IgniteTest): NUM_NODES = 3 @cluster(num_nodes=NUM_NODES + 2) - @parametrize(version=str(DEV_BRANCH)) - @parametrize(version=str(LATEST_2_7)) - def test(self, version): + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_7)) + def test(self, ignite_version): """ Test PME free scenario (node stop). """ @@ -50,10 +49,8 @@ def test(self, version): self.stage("Starting nodes") - ignite_version = IgniteVersion(version) - config = IgniteConfiguration( - version=ignite_version, + version=IgniteVersion(ignite_version), caches=[CacheConfiguration(name='test-cache', backups=2, atomicity_mode='TRANSACTIONAL')] ) @@ -85,7 +82,7 @@ def test(self, version): single_key_tx_streamer.start() - if ignite_version >= V_2_8_0: + if IgniteVersion(ignite_version) >= V_2_8_0: ControlUtility(ignites, self.test_context).disable_baseline_auto_adjust() self.stage("Stopping server node") diff --git a/modules/ducktests/tests/ignitetest/tests/smoke_test.py b/modules/ducktests/tests/ignitetest/tests/smoke_test.py index d7207a23940e9..f58d3c55a7101 100644 --- a/modules/ducktests/tests/ignitetest/tests/smoke_test.py +++ b/modules/ducktests/tests/ignitetest/tests/smoke_test.py @@ -17,15 +17,15 @@ This module contains smoke tests that checks that services work """ -from ducktape.mark import parametrize from ducktape.mark.resource import cluster from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.spark import SparkService from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster -from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteClientConfiguration +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration from ignitetest.services.zk.zookeeper import ZookeeperService +from ignitetest.utils import ignite_versions from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, IgniteVersion @@ -37,26 +37,32 @@ class SmokeServicesTest(IgniteTest): """ @cluster(num_nodes=1) - @parametrize(version=str(DEV_BRANCH)) - def test_ignite_start_stop(self, version): + @ignite_versions(str(DEV_BRANCH)) + def test_ignite_start_stop(self, ignite_version): """ Test that IgniteService correctly start and stop """ - ignite = IgniteService(self.test_context, IgniteConfiguration(version=IgniteVersion(version)), num_nodes=1) + ignite = IgniteService(self.test_context, IgniteConfiguration(version=IgniteVersion(ignite_version)), + num_nodes=1) + print(self.test_context) ignite.start() ignite.stop() @cluster(num_nodes=2) - @parametrize(version=str(DEV_BRANCH)) - def test_ignite_app_start_stop(self, version): + @ignite_versions(str(DEV_BRANCH)) + def test_ignite_app_start_stop(self, ignite_version): """ Test that IgniteService and IgniteApplicationService correctly start and stop """ - ignite = IgniteService(self.test_context, IgniteConfiguration(version=IgniteVersion(version)), num_nodes=1) + server_configuration = IgniteConfiguration(version=IgniteVersion(ignite_version)) + ignite = IgniteService(self.test_context, server_configuration, num_nodes=1) + + client_configuration = server_configuration._replace(client_mode=True, + discovery_spi=from_ignite_cluster(ignite)) app = IgniteApplicationService( self.test_context, - IgniteClientConfiguration(version=IgniteVersion(version), discovery_spi=from_ignite_cluster(ignite)), + client_configuration, java_class_name="org.apache.ignite.internal.ducktest.tests.smoke_test.SimpleApplication") ignite.start() diff --git a/modules/ducktests/tests/ignitetest/utils/__init__.py b/modules/ducktests/tests/ignitetest/utils/__init__.py index 777351b341818..838c33343eff2 100644 --- a/modules/ducktests/tests/ignitetest/utils/__init__.py +++ b/modules/ducktests/tests/ignitetest/utils/__init__.py @@ -14,4 +14,4 @@ # limitations under the License. # pylint: disable=C0114 -from ._mark import version_if +from ._mark import version_if, ignite_versions diff --git a/modules/ducktests/tests/ignitetest/utils/_mark.py b/modules/ducktests/tests/ignitetest/utils/_mark.py index 7387d50978e67..99f328f25af0c 100644 --- a/modules/ducktests/tests/ignitetest/utils/_mark.py +++ b/modules/ducktests/tests/ignitetest/utils/_mark.py @@ -16,7 +16,10 @@ """ Module contains useful test decorators. """ -from ducktape.mark._mark import Ignore, Mark + +from collections.abc import Iterable + +from ducktape.mark._mark import Ignore, Mark, _inject from ignitetest.utils.version import IgniteVersion @@ -25,17 +28,18 @@ class VersionIf(Ignore): """ Ignore test if version doesn't corresponds to condition. """ - def __init__(self, condition): + def __init__(self, condition, variable_name): super().__init__() self.condition = condition + self.variable_name = variable_name def apply(self, seed_context, context_list): assert len(context_list) > 0, "ignore_if decorator is not being applied to any test cases" for ctx in context_list: - assert 'version' in ctx.injected_args, "'version' in injected args not present" - version = ctx.injected_args['version'] - assert isinstance(version, str), "'version' in injected args must be a string" + assert self.variable_name in ctx.injected_args, "'%s' in injected args not present" % (self.variable_name,) + version = ctx.injected_args[self.variable_name] + assert isinstance(version, str), "'%s'n injected args must be a string" % (self.variable_name,) ctx.ignore = ctx.ignore or not self.condition(IgniteVersion(version)) return context_list @@ -44,14 +48,93 @@ def __eq__(self, other): return super().__eq__(other) and self.condition == other.condition -def version_if(condition): +class IgniteVersionParametrize(Mark): + """ + Parametrize function with ignite_version + """ + def __init__(self, *args, version_prefix): + """ + :param args: Can be string, tuple of strings or iterable of them. + :param version_prefix: prefix for variable to inject into test function. + """ + self.versions = list(args) + self.version_prefix = version_prefix + + def apply(self, seed_context, context_list): + if 'ignite_versions' in seed_context.globals: + ver = seed_context.globals['ignite_versions'] + if isinstance(ver, str): + self.versions = [ver] + elif isinstance(ver, Iterable): + self.versions = list(ver) + else: + raise AssertionError("Expected string or iterable as parameter in ignite_versions, " + "%s of type %s passed" % (ver, type(ver))) + + new_context_list = [] + if context_list: + for ctx in context_list: + for version in self.versions: + if ctx.injected_args and 'ignite_version' in ctx.injected_args: + continue + + new_context_list.insert(0, self._prepare_new_ctx(version, seed_context, ctx)) + else: + for version in self.versions: + new_context_list.insert(0, self._prepare_new_ctx(version, seed_context)) + + return new_context_list + + def _prepare_new_ctx(self, version, seed_context, ctx=None): + injected_args = dict(ctx.injected_args) if ctx and ctx.injected_args else {} + + if isinstance(version, (list, tuple)) and len(version) >= 2: + for idx, ver in enumerate(version): + injected_args["%s_%s" % (self.version_prefix, idx + 1)] = ver + elif isinstance(version, str): + injected_args[self.version_prefix] = version + else: + raise AssertionError("Expected string or iterable of size greater than 2 as element in ignite_versions," + "%s of type %s passed" % (version, type(version))) + + injected_fun = _inject(**injected_args)(seed_context.function) + + return seed_context.copy(function=injected_fun, injected_args=injected_args) + + @property + def name(self): + """ + This function should return "PARAMETRIZE" string in order to ducktape's method parametrization works. + Should be fixed after apropriate patch in ducktape will be merged. + """ + return "PARAMETRIZE" + + def __eq__(self, other): + return super().__eq__(other) and self.versions == other.versions + + +def ignite_versions(*args, version_prefix="ignite_version"): + """ + Decorate test function to inject ignite versions. Versions will be overriden by globals "ignite_versions" param. + :param args: Can be string, tuple of strings or iterable of them. + :param version_prefix: prefix for variable to inject into test function. + """ + def parametrizer(func): + Mark.mark(func, IgniteVersionParametrize(*args, version_prefix=version_prefix)) + + return func + return parametrizer + + +def version_if(condition, variable_name='ignite_version'): """ Mark decorated test method as IGNORE if version doesn't corresponds to condition. :param condition: function(IgniteVersion) -> bool + :param variable_name: version variable name """ def ignorer(func): - Mark.mark(func, VersionIf(condition)) + Mark.mark(func, VersionIf(condition, variable_name)) return func return ignorer From 47d131124996cfc3f8a986a3b7406a364424bbf0 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Tue, 15 Sep 2020 10:19:59 +0300 Subject: [PATCH 50/78] IGNITE-13433 Benchmark confirms operation's latency drop decrease on Cellular switch comparing to PME-free switch (#8232) --- .../CellularPreparedTxStreamer.java | 92 +++++++++++ .../CellularTxStreamer.java | 112 +++++++++++++ .../DistributionChecker.java | 4 +- .../SingleKeyTxStreamerApplication.java | 2 +- .../utils/IgniteAwareApplication.java | 153 +++++++++++++----- modules/ducktests/tests/docker/run_tests.sh | 4 +- .../tests/ignitetest/services/ignite_app.py | 54 +++++-- .../ignitetest/services/utils/ignite_spec.py | 9 +- .../tests/add_node_rebalance_test.py | 6 - .../tests/cellular_affinity_test.py | 120 +++++++++++++- .../ignitetest/tests/pme_free_switch_test.py | 14 +- .../tests/ignitetest/utils/ignite_test.py | 7 - 12 files changed, 484 insertions(+), 93 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularPreparedTxStreamer.java create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularPreparedTxStreamer.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularPreparedTxStreamer.java new file mode 100644 index 0000000000000..43424eb05889e --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularPreparedTxStreamer.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.cellular_affinity_test; + +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; +import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.transactions.Transaction; + +/** + * Prepares transactions at specified cell. + */ +public class CellularPreparedTxStreamer extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override protected void run(JsonNode jsonNode) throws Exception { + final String cacheName = jsonNode.get("cacheName").asText(); + final String attr = jsonNode.get("attr").asText(); + final String cell = jsonNode.get("cell").asText(); + final int txCnt = jsonNode.get("txCnt").asInt(); + + markInitialized(); + + waitForActivation(); + + IgniteCache cache = ignite.getOrCreateCache(cacheName); + + log.info("Starting Prepared Txs..."); + + Affinity aff = ignite.affinity(cacheName); + + int cnt = 0; + int i = -1; // Negative keys to have no intersection with load. + + while (cnt != txCnt && !terminated()) { + Collection nodes = aff.mapKeyToPrimaryAndBackups(i); + + Map stat = nodes.stream().collect( + Collectors.groupingBy(n -> n.attributes().get(attr), Collectors.counting())); + + assert 1 == stat.keySet().size() : + "Partition should be located on nodes from only one cell " + + "[key=" + i + ", nodes=" + nodes.size() + ", stat=" + stat + "]"; + + if (stat.containsKey(cell)) { + cnt++; + + Transaction tx = ignite.transactions().txStart(); + + cache.put(i, i); + + ((TransactionProxyImpl)tx).tx().prepare(true); + + if (cnt % 100 == 0) + log.info("Long Tx prepared [key=" + i + ",cnt=" + cnt + ", cell=" + stat.keySet() + "]"); + } + + i--; + } + + log.info("All transactions prepared (" + cnt + ")"); + + while (!terminated()) { + log.info("Waiting for SIGTERM."); + + U.sleep(1000); + } + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java new file mode 100644 index 0000000000000..ebfc6f2464c00 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.cellular_affinity_test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Streams transactions to specified cell. + */ +public class CellularTxStreamer extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jsonNode) throws Exception { + String cacheName = jsonNode.get("cacheName").asText(); + int warmup = jsonNode.get("warmup").asInt(); + String cell = jsonNode.get("cell").asText(); + String attr = jsonNode.get("attr").asText(); + + markInitialized(); + + waitForActivation(); + + IgniteCache cache = ignite.getOrCreateCache(cacheName); + + long[] max = new long[20]; + + Arrays.fill(max, -1); + + int key = 0; + + int cnt = 0; + + long initTime = 0; + + boolean record = false; + + Affinity aff = ignite.affinity(cacheName); + + while (!terminated()) { + key++; + + Collection nodes = aff.mapKeyToPrimaryAndBackups(key); + + Map stat = nodes.stream().collect( + Collectors.groupingBy(n -> n.attributes().get(attr), Collectors.counting())); + + if (!stat.containsKey(cell)) + continue; + + cnt++; + + long start = System.currentTimeMillis(); + + cache.put(key, key); + + long finish = System.currentTimeMillis(); + + long time = finish - start; + + if (!record && cnt > warmup) { + record = true; + + initTime = System.currentTimeMillis(); + + log.info("Warmup finished"); + } + + if (record) { + for (int i = 0; i < max.length; i++) { + if (max[i] <= time) { + System.arraycopy(max, i, max, i + 1, max.length - i - 1); + + max[i] = time; + + break; + } + } + } + + if (cnt % 1000 == 0) + log.info("Application streamed " + cnt + " transactions [worst_latency=" + Arrays.toString(max) + "]"); + } + + recordResult("WORST_LATENCY", Arrays.toString(max)); + recordResult("STREAMED", cnt - warmup); + recordResult("MEASURE_DURATION", System.currentTimeMillis() - initTime); + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java index 2daf63eb1821b..22b2c94ffc625 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/DistributionChecker.java @@ -28,9 +28,7 @@ * */ public class DistributionChecker extends IgniteAwareApplication { - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override protected void run(JsonNode jsonNode) { String cacheName = jsonNode.get("cacheName").asText(); String attr = jsonNode.get("attr").asText(); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java index 8c8be15b53566..14b53a900be0f 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java @@ -55,7 +55,7 @@ boolean record = false; if (!record && cnt > warmup) { record = true; - initTime = System.currentTimeMillis();; + initTime = System.currentTimeMillis(); markInitialized(); } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java index 107787b91de07..a74b53f2b133c 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java @@ -17,9 +17,14 @@ package org.apache.ignite.internal.ducktest.utils; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.Ignite; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -55,8 +60,8 @@ public abstract class IgniteAwareApplication { /** Terminated. */ private static volatile boolean terminated; - /** Shutdown hook. */ - private static volatile Thread hook; + /** State mutex. */ + private static final Object stateMux = new Object(); /** Ignite. */ protected Ignite ignite; @@ -68,7 +73,7 @@ public abstract class IgniteAwareApplication { * Default constructor. */ protected IgniteAwareApplication() { - Runtime.getRuntime().addShutdownHook(hook = new Thread(() -> { + Runtime.getRuntime().addShutdownHook(new Thread(() -> { log.info("SIGTERM recorded."); if (!finished && !broken) @@ -79,8 +84,13 @@ protected IgniteAwareApplication() { if (log.isDebugEnabled()) log.debug("Waiting for graceful termination..."); + int iter = 0; + while (!finished && !broken) { - log.info("Waiting for graceful termination cycle..."); + log.info("Waiting for graceful termination cycle... [iter=" + ++iter + "]"); + + if (iter == 100) + dumpThreads(); try { U.sleep(100); @@ -101,49 +111,72 @@ protected IgniteAwareApplication() { * Used to marks as started to perform actions. Suitable for async runs. */ protected void markInitialized() { - assert !inited; + log.info("Marking as initialized."); - log.info(APP_INITED); + synchronized (stateMux) { + assert !inited; + assert !finished; + assert !broken; - inited = true; + log.info(APP_INITED); + + inited = true; + } } /** * */ protected void markFinished() { - assert !finished; - assert !broken; + log.info("Marking as finished."); - log.info(APP_FINISHED); + synchronized (stateMux) { + assert inited; + assert !finished; + assert !broken; - if (!terminated()) - removeShutdownHook(); + log.info(APP_FINISHED); - finished = true; + finished = true; + } } /** * */ - private void markBroken() { - assert !finished; - assert !broken; + protected void markBroken(Throwable th) { + log.info("Marking as broken."); - log.info(APP_BROKEN); + synchronized (stateMux) { + if (broken) { + log.info("Already marked as broken."); - removeShutdownHook(); + return; + } - broken = true; + recordResult("ERROR", th.toString()); + + assert !finished; + + log.error(APP_BROKEN); + + broken = true; + } } /** * */ - private void removeShutdownHook() { - Runtime.getRuntime().removeShutdownHook(hook); + private void terminate() { + log.info("Marking as initialized."); + + synchronized (stateMux) { + assert !terminated; + + log.info(APP_TERMINATED); - log.info("Shutdown hook removed."); + terminated = true; + } } /** @@ -154,17 +187,6 @@ protected void markSyncExecutionComplete() { markFinished(); } - /** - * - */ - private void terminate() { - assert !terminated; - - log.info(APP_TERMINATED); - - terminated = true; - } - /** * */ @@ -226,12 +248,71 @@ public void start(JsonNode jsonNode) { catch (Throwable th) { log.error("Unexpected Application failure... ", th); - recordResult("ERROR", th.getMessage()); - - markBroken(); + if (!broken) + markBroken(th); } finally { log.info("Application finished."); } } + + /** + * + */ + private static void dumpThreads() { + ThreadInfo[] infos = ManagementFactory.getThreadMXBean().dumpAllThreads(true, true); + + for (ThreadInfo info : infos) { + log.info(info.toString()); + + if ("main".equals(info.getThreadName())) { + StringBuilder sb = new StringBuilder(); + + sb.append("main\n"); + + for (StackTraceElement element : info.getStackTrace()) { + sb.append("\tat ").append(element.toString()); + sb.append('\n'); + } + + log.info(sb.toString()); + } + } + } + + /** + * + */ + protected void waitForActivation() throws IgniteInterruptedCheckedException { + boolean newApi = ignite.cluster().localNode().version().greaterThanEqual(2, 9, 0); + + while (newApi ? ignite.cluster().state() != ClusterState.ACTIVE : !ignite.cluster().active()) { + U.sleep(100); + + log.info("Waiting for cluster activation"); + } + + log.info("Cluster Activated"); + } + + /** + * + */ + protected void waitForRebalanced() throws IgniteInterruptedCheckedException { + boolean possible = ignite.cluster().localNode().version().greaterThanEqual(2, 8, 0); + + if (possible) { + GridCachePartitionExchangeManager mgr = ((IgniteEx)ignite).context().cache().context().exchange(); + + while (!mgr.lastFinishedFuture().rebalanced()) { + U.sleep(1000); + + log.info("Waiting for cluster rebalance finish"); + } + + log.info("Cluster Rebalanced"); + } + else + throw new UnsupportedOperationException("Operation supported since 2.8.0"); + } } diff --git a/modules/ducktests/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh index d8eb270297c20..93a8b7c0290ce 100755 --- a/modules/ducktests/tests/docker/run_tests.sh +++ b/modules/ducktests/tests/docker/run_tests.sh @@ -21,7 +21,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # DuckerUp parameters are specified with env variables # Num of cotainers that ducktape will prepare for tests -IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-11} +IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-13} # Image name to run nodes default_image_name="ducker-ignite-openjdk-8" @@ -55,7 +55,7 @@ The options are as follows: Display this help message. -n|--num-nodes - Specify how many nodes to start. Default number of nodes to start: 11. + Specify how many nodes to start. Default number of nodes to start: 13 (12 + 1 used by ducktape). -j|--max-parallel Specify max number of tests that can be run in parallel. diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index b40d01ada55f2..9e396f2c76342 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -19,6 +19,9 @@ import re +# pylint: disable=W0622 +from ducktape.errors import TimeoutError + from ignitetest.services.utils.ignite_aware import IgniteAwareService @@ -38,7 +41,7 @@ def __init__(self, context, config, java_class_name, params="", timeout_sec=60, self.servicejava_class_name = servicejava_class_name self.java_class_name = java_class_name self.timeout_sec = timeout_sec - self.stop_timeout_sec = 10 + self.params = params def start(self): super().start() @@ -46,25 +49,46 @@ def start(self): self.logger.info("Waiting for Ignite aware Application (%s) to start..." % self.java_class_name) self.await_event("Topology snapshot", self.timeout_sec, from_the_beginning=True) - self.await_event("IGNITE_APPLICATION_INITIALIZED\\|IGNITE_APPLICATION_BROKEN", self.timeout_sec, - from_the_beginning=True) - try: - self.await_event("IGNITE_APPLICATION_INITIALIZED", 1, from_the_beginning=True) - except Exception: - raise Exception("Java application execution failed. %s" % self.extract_result("ERROR")) from None + self.__check_status("IGNITE_APPLICATION_INITIALIZED", timeout=self.timeout_sec) - # pylint: disable=W0221 - def stop_node(self, node, clean_shutdown=True, timeout_sec=20): - self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) - node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=clean_shutdown, allow_fail=True) + def stop_async(self, clean_shutdown=True): + """ + Stops node in async way. + """ + self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(self.nodes[0].account))) + self.nodes[0].account.kill_java_processes(self.servicejava_class_name, clean_shutdown=clean_shutdown, + allow_fail=True) - stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec) + def await_stopped(self, timeout_sec=10): + """ + Awaits node stop finish. + """ + stopped = self.wait_node(self.nodes[0], timeout_sec=timeout_sec) assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ - (str(node.account), str(self.stop_timeout_sec)) + (str(self.nodes[0].account), str(timeout_sec)) - self.await_event("IGNITE_APPLICATION_FINISHED\\|IGNITE_APPLICATION_BROKEN", from_the_beginning=True, - timeout_sec=timeout_sec) + self.__check_status("IGNITE_APPLICATION_FINISHED", timeout=timeout_sec) + + # pylint: disable=W0221 + def stop_node(self, node, clean_shutdown=True, timeout_sec=10): + assert node == self.nodes[0] + self.stop_async(clean_shutdown) + self.await_stopped(timeout_sec) + + def __check_status(self, desired, timeout=1): + self.await_event("%s\\|IGNITE_APPLICATION_BROKEN" % desired, timeout, from_the_beginning=True) + + try: + self.await_event("IGNITE_APPLICATION_BROKEN", 1, from_the_beginning=True) + raise Exception("Java application execution failed. %s" % self.extract_result("ERROR")) + except TimeoutError: + pass + + try: + self.await_event(desired, 1, from_the_beginning=True) + except Exception: + raise Exception("Java application execution failed.") from None def clean_node(self, node): if self.alive(node): diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index 124fefb530f57..afb975a7cfce1 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -22,11 +22,10 @@ import json from abc import ABCMeta, abstractmethod -from ignitetest.services.utils.ignite_path import IgnitePath from ignitetest.services.utils.config_template import IgniteClientConfigTemplate, IgniteServerConfigTemplate -from ignitetest.utils.version import DEV_BRANCH - +from ignitetest.services.utils.ignite_path import IgnitePath from ignitetest.services.utils.ignite_persistence import IgnitePersistenceAware +from ignitetest.utils.version import DEV_BRANCH def resolve_spec(service, context, config, **kwargs): @@ -147,6 +146,8 @@ def __init__(self, modules, **kwargs): libs.append("log4j") libs = list(map(lambda m: self.path.module(m) + "/*", libs)) + libs.append(IgnitePath(DEV_BRANCH).module("ducktests") + "/*") + self.envs = { 'EXCLUDE_TEST_CLASSES': 'true', 'IGNITE_LOG_DIR': self.PERSISTENT_ROOT, @@ -183,7 +184,7 @@ def __init__(self, context, modules, servicejava_class_name, java_class_name, pa } self.jvm_opts.extend([ - "-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file ", + "-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file", "-Dlog4j.configDebug=true", "-DIGNITE_NO_SHUTDOWN_HOOK=true", # allows to perform operations on app termination. "-Xmx1G", diff --git a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py index 068142405f190..82bc730f1ea9b 100644 --- a/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py +++ b/modules/ducktests/tests/ignitetest/tests/add_node_rebalance_test.py @@ -47,15 +47,11 @@ def test_add_node(self, ignite_version): * Put data to it via IgniteClientApp. * Start one more node and awaits for rebalance to finish. """ - self.stage("Start Ignite nodes") - node_config = IgniteConfiguration(version=IgniteVersion(ignite_version)) ignites = IgniteService(self.test_context, config=node_config, num_nodes=self.NUM_NODES - 1) ignites.start() - self.stage("Starting DataGenerationApplication") - # This client just put some data to the cache. app_config = node_config._replace(client_mode=True, discovery_spi=from_ignite_cluster(ignites)) IgniteApplicationService(self.test_context, config=app_config, @@ -66,8 +62,6 @@ def test_add_node(self, ignite_version): ignite = IgniteService(self.test_context, node_config._replace(discovery_spi=from_ignite_cluster(ignites)), num_nodes=1) - self.stage("Starting Ignite node") - ignite.start() start = self.monotonic() diff --git a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py index c957e43989c55..3c38b228181e9 100644 --- a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py +++ b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py @@ -22,11 +22,12 @@ from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteClientConfiguration from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.utils import ignite_versions, version_if from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH, IgniteVersion +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion, LATEST_2_8 # pylint: disable=W0223 @@ -40,6 +41,8 @@ class CellularAffinity(IgniteTest): CACHE_NAME = "test-cache" + PREPARED_TX_CNT = 500 # possible amount at real cluster under load (per cell). + CONFIG_TEMPLATE = """ @@ -55,6 +58,7 @@ class CellularAffinity(IgniteTest): + @@ -73,15 +77,17 @@ def properties(): @cluster(num_nodes=NUM_NODES * 3 + 1) @version_if(lambda version: version >= DEV_BRANCH) @ignite_versions(str(DEV_BRANCH)) - def test(self, ignite_version): + def test_distribution(self, ignite_version): """ - Test Cellular Affinity scenario (partition distribution). + Tests Cellular Affinity scenario (partition distribution). """ cell1 = self.start_cell(ignite_version, ['-D' + CellularAffinity.ATTRIBUTE + '=1']) self.start_cell(ignite_version, ['-D' + CellularAffinity.ATTRIBUTE + '=2'], joined_cluster=cell1) self.start_cell(ignite_version, ['-D' + CellularAffinity.ATTRIBUTE + '=XXX', '-DRANDOM=42'], joined_cluster=cell1) + ControlUtility(cell1, self.test_context).activate() + checker = IgniteApplicationService( self.test_context, IgniteClientConfiguration(version=IgniteVersion(ignite_version), discovery_spi=from_ignite_cluster(cell1)), @@ -92,15 +98,117 @@ def test(self, ignite_version): checker.run() - def start_cell(self, version, jvm_opts, joined_cluster=None): + # pylint: disable=R0914 + @cluster(num_nodes=NUM_NODES * (3 + 1)) + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) + def test_latency(self, ignite_version): + """ + Tests Cellular switch tx latency. + """ + data = {} + + cell1, prepared_tx_loader1 = self.start_cell_with_prepared_txs(ignite_version, "C1") + _, prepared_tx_loader2 = self.start_cell_with_prepared_txs(ignite_version, "C2", joined_cluster=cell1) + _, prepared_tx_loader3 = self.start_cell_with_prepared_txs(ignite_version, "C3", joined_cluster=cell1) + + loaders = [prepared_tx_loader1, prepared_tx_loader2, prepared_tx_loader3] + + failed_loader = prepared_tx_loader3 + + tx_streamer1 = self.start_tx_streamer(ignite_version, "C1", joined_cluster=cell1) + tx_streamer2 = self.start_tx_streamer(ignite_version, "C2", joined_cluster=cell1) + tx_streamer3 = self.start_tx_streamer(ignite_version, "C3", joined_cluster=cell1) + + streamers = [tx_streamer1, tx_streamer2, tx_streamer3] + + for streamer in streamers: # starts tx streaming with latency record (with some warmup). + streamer.start() + + ControlUtility(cell1, self.test_context).disable_baseline_auto_adjust() # baseline set. + ControlUtility(cell1, self.test_context).activate() + + for loader in loaders: + loader.await_event("All transactions prepared", 180, from_the_beginning=True) + + for streamer in streamers: + streamer.await_event("Warmup finished", 180, from_the_beginning=True) + + failed_loader.stop_async() # node left with prepared txs. + + for streamer in streamers: + streamer.await_event("Node left topology\\|Node FAILED", 60, from_the_beginning=True) + + for streamer in streamers: # just an assertion that we have PME-free switch. + streamer.await_event("exchangeFreeSwitch=true", 60, from_the_beginning=True) + + for streamer in streamers: # waiting for streaming continuation. + streamer.await_event("Application streamed", 60) + + for streamer in streamers: # stops streaming and records results. + streamer.stop_async() + + for streamer in streamers: + streamer.await_stopped() + + cell = streamer.params["cell"] + + data["[%s cell %s]" % ("alive" if cell is not failed_loader.params["cell"] else "broken", cell)] = \ + "worst_latency=%s, tx_streamed=%s, measure_duration=%s" % ( + streamer.extract_result("WORST_LATENCY"), streamer.extract_result("STREAMED"), + streamer.extract_result("MEASURE_DURATION")) + + return data + + def start_tx_streamer(self, version, cell, joined_cluster): + """ + Starts transaction streamer. + """ + return IgniteApplicationService( + self.test_context, + IgniteClientConfiguration(version=IgniteVersion(version), properties=self.properties(), + discovery_spi=from_ignite_cluster(joined_cluster)), + java_class_name="org.apache.ignite.internal.ducktest.tests.cellular_affinity_test.CellularTxStreamer", + params={"cacheName": CellularAffinity.CACHE_NAME, + "attr": CellularAffinity.ATTRIBUTE, + "cell": cell, + "warmup": 10000}, + timeout_sec=180) + + def start_cell_with_prepared_txs(self, version, cell_id, joined_cluster=None): + """ + Starts cell with prepared transactions. + """ + nodes = self.start_cell(version, ['-D' + CellularAffinity.ATTRIBUTE + '=' + cell_id], + CellularAffinity.NUM_NODES - 1, joined_cluster) + + prepared_tx_streamer = IgniteApplicationService( # last server node at the cell. + self.test_context, + IgniteConfiguration(version=IgniteVersion(version), properties=self.properties(), + discovery_spi=from_ignite_cluster(nodes)), # Server node. + java_class_name= + "org.apache.ignite.internal.ducktest.tests.cellular_affinity_test.CellularPreparedTxStreamer", + params={"cacheName": CellularAffinity.CACHE_NAME, + "attr": CellularAffinity.ATTRIBUTE, + "cell": cell_id, + "txCnt": CellularAffinity.PREPARED_TX_CNT}, + jvm_opts=['-D' + CellularAffinity.ATTRIBUTE + '=' + cell_id], + timeout_sec=180) + + prepared_tx_streamer.start() # starts last server node and creates prepared txs on it. + + return nodes, prepared_tx_streamer + + def start_cell(self, version, jvm_opts, nodes_cnt=NUM_NODES, joined_cluster=None): """ Starts cell. """ - config = IgniteConfiguration(version=IgniteVersion(version), properties=self.properties()) + config = IgniteConfiguration(version=IgniteVersion(version), properties=self.properties(), + cluster_state="INACTIVE") + if joined_cluster: config = config._replace(discovery_spi=from_ignite_cluster(joined_cluster)) - ignites = IgniteService(self.test_context, config, num_nodes=CellularAffinity.NUM_NODES, jvm_opts=jvm_opts) + ignites = IgniteService(self.test_context, config, num_nodes=nodes_cnt, jvm_opts=jvm_opts) ignites.start() diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index 395b24bcf810c..3b54762d28509 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -43,12 +43,10 @@ class PmeFreeSwitchTest(IgniteTest): @ignite_versions(str(DEV_BRANCH), str(LATEST_2_7)) def test(self, ignite_version): """ - Test PME free scenario (node stop). + Tests PME free scenario (node stop). """ data = {} - self.stage("Starting nodes") - config = IgniteConfiguration( version=IgniteVersion(ignite_version), caches=[CacheConfiguration(name='test-cache', backups=2, atomicity_mode='TRANSACTIONAL')] @@ -58,8 +56,6 @@ def test(self, ignite_version): ignites.start() - self.stage("Starting long_tx_streamer") - client_config = config._replace(client_mode=True, discovery_spi=from_ignite_cluster(ignites, slice(0, self.NUM_NODES - 1))) @@ -71,8 +67,6 @@ def test(self, ignite_version): long_tx_streamer.start() - self.stage("Starting single_key_tx_streamer") - single_key_tx_streamer = IgniteApplicationService( self.test_context, client_config, @@ -85,20 +79,14 @@ def test(self, ignite_version): if IgniteVersion(ignite_version) >= V_2_8_0: ControlUtility(ignites, self.test_context).disable_baseline_auto_adjust() - self.stage("Stopping server node") - ignites.stop_node(ignites.nodes[self.NUM_NODES - 1]) long_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True) time.sleep(30) # keeping txs alive for 30 seconds. - self.stage("Stopping long_tx_streamer") - long_tx_streamer.stop() - self.stage("Stopping single_key_tx_streamer") - single_key_tx_streamer.stop() data["Worst latency (ms)"] = single_key_tx_streamer.extract_result("WORST_LATENCY") diff --git a/modules/ducktests/tests/ignitetest/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/utils/ignite_test.py index a4bfe7da81f87..feb59937e3757 100644 --- a/modules/ducktests/tests/ignitetest/utils/ignite_test.py +++ b/modules/ducktests/tests/ignitetest/utils/ignite_test.py @@ -29,13 +29,6 @@ class IgniteTest(Test): def __init__(self, test_context): super().__init__(test_context=test_context) - def stage(self, msg): - """ - Print stage mark. - :param msg: Stage mark message. - """ - self.logger.info("[TEST_STAGE] " + msg) - @staticmethod def monotonic(): """ From af0f0a7b13c3e9b78900c818bc9ca4040d927953 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Tue, 15 Sep 2020 17:26:18 +0300 Subject: [PATCH 51/78] IGNITE-13429 Integration test of control.sh transactions' management (#8239) --- .../LongRunningTransactionsGenerator.java | 157 +++++++++++++ .../utils/IgniteAwareApplication.java | 6 +- .../tests/ignitetest/services/ignite_app.py | 15 +- .../services/utils/control_utility.py | 221 ++++++++++++++++-- .../tests/control_utility/__init__.py | 18 ++ .../baseline_test.py} | 12 +- .../tests/control_utility/tx_test.py | 192 +++++++++++++++ .../ignitetest/tests/suites/fast_suite.yml | 2 +- modules/ducktests/tests/setup.py | 2 +- modules/ducktests/tests/tox.ini | 1 + 10 files changed, 597 insertions(+), 29 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/control_utility/LongRunningTransactionsGenerator.java create mode 100644 modules/ducktests/tests/ignitetest/tests/control_utility/__init__.py rename modules/ducktests/tests/ignitetest/tests/{control_utility_test.py => control_utility/baseline_test.py} (95%) create mode 100644 modules/ducktests/tests/ignitetest/tests/control_utility/tx_test.py diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/control_utility/LongRunningTransactionsGenerator.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/control_utility/LongRunningTransactionsGenerator.java new file mode 100644 index 0000000000000..3bbf732c7e540 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/control_utility/LongRunningTransactionsGenerator.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.control_utility; + +import java.time.Duration; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.locks.Lock; +import javax.cache.CacheException; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteTransactions; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionRollbackException; + +/** + * Run long running transactions on node with specified param. + */ +public class LongRunningTransactionsGenerator extends IgniteAwareApplication { + /** */ + private static final Duration TOPOLOGY_WAIT_TIMEOUT = Duration.ofSeconds(60); + + /** */ + private static final String KEYS_LOCKED_MESSAGE = "APPLICATION_KEYS_LOCKED"; + + /** */ + private static final String LOCKED_KEY_PREFIX = "KEY_"; + + /** */ + private volatile Executor pool; + + /** {@inheritDoc} */ + @Override protected void run(JsonNode jsonNode) throws Exception { + IgniteCache cache = ignite.cache(jsonNode.get("cache_name").asText()); + + int txCount = jsonNode.get("tx_count") != null ? jsonNode.get("tx_count").asInt() : 1; + + int txSize = jsonNode.get("tx_size") != null ? jsonNode.get("tx_size").asInt() : 1; + + String keyPrefix = jsonNode.get("key_prefix") != null ? jsonNode.get("key_prefix").asText() : LOCKED_KEY_PREFIX; + + String label = jsonNode.get("label") != null ? jsonNode.get("label").asText() : null; + + long expectedTopologyVersion = jsonNode.get("wait_for_topology_version") != null ? + jsonNode.get("wait_for_topology_version").asLong() : -1L; + + CountDownLatch lockLatch = new CountDownLatch(txCount); + + pool = Executors.newFixedThreadPool(2 * txCount); + + markInitialized(); + + if (expectedTopologyVersion > 0) { + log.info("Start waiting for topology version: " + expectedTopologyVersion + ", " + + "current version is: " + ignite.cluster().topologyVersion()); + + long start = System.nanoTime(); + + while (ignite.cluster().topologyVersion() < expectedTopologyVersion + && Duration.ofNanos(start - System.nanoTime()).compareTo(TOPOLOGY_WAIT_TIMEOUT) < 0) + Thread.sleep(100L); + + log.info("Finished waiting for topology version: " + expectedTopologyVersion + ", " + + "current version is: " + ignite.cluster().topologyVersion()); + } + + for (int i = 0; i < txCount; i++) { + String key = keyPrefix + i; + + pool.execute(() -> { + Lock lock = cache.lock(key); + + lock.lock(); + + try { + lockLatch.countDown(); + + while (!terminated()) + Thread.sleep(100L); + } + catch (InterruptedException e) { + markBroken(new RuntimeException("Unexpected thread interruption", e)); + + Thread.currentThread().interrupt(); + } + finally { + lock.unlock(); + } + }); + } + + lockLatch.await(); + + log.info(KEYS_LOCKED_MESSAGE); + + CountDownLatch txLatch = new CountDownLatch(txCount); + + for (int i = 0; i < txCount; i++) { + Map data = new TreeMap<>(); + + for (int j = 0; j < txSize; j++) { + String key = keyPrefix + (j == 0 ? String.valueOf(i) : i + "_" + j); + + data.put(key, key); + } + + IgniteTransactions igniteTransactions = label != null ? ignite.transactions().withLabel(label) : + ignite.transactions(); + + pool.execute(() -> { + IgniteUuid xid = null; + + try (Transaction tx = igniteTransactions.txStart()) { + xid = tx.xid(); + + cache.putAll(data); + + tx.commit(); + } + catch (Exception e) { + if (e instanceof CacheException && e.getCause() != null && + e.getCause() instanceof TransactionRollbackException) + recordResult("TX_ID", xid != null ? xid.toString() : ""); + else + markBroken(new RuntimeException("Transaction is rolled back with unexpected error", e)); + } + finally { + txLatch.countDown(); + } + }); + } + + txLatch.await(); + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java index a74b53f2b133c..35760b367b1f1 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java @@ -144,18 +144,18 @@ protected void markFinished() { /** * */ - protected void markBroken(Throwable th) { + public void markBroken(Throwable th) { log.info("Marking as broken."); synchronized (stateMux) { + recordResult("ERROR", th.toString()); + if (broken) { log.info("Already marked as broken."); return; } - recordResult("ERROR", th.toString()); - assert !finished; log.error(APP_BROKEN); diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 9e396f2c76342..74b7fba048364 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -107,12 +107,23 @@ def extract_result(self, name): :param name: Result parameter's name. :return: Extracted result of application run. """ - res = "" + results = self.extract_results(name) + + assert len(results) <= 1, f"Expected exactly one result occurence, {len(results)} found." + + return results[0] if results else "" + + def extract_results(self, name): + """ + :param name: Results parameter's name. + :return: Extracted results of application run. + """ + res = [] output = self.nodes[0].account.ssh_capture( "grep '%s' %s" % (name + "->", self.STDOUT_STDERR_CAPTURE), allow_fail=False) for line in output: - res = re.search("%s(.*)%s" % (name + "->", "<-"), line).group(1) + res.append(re.search("%s(.*)%s" % (name + "->", "<-"), line).group(1)) return res diff --git a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py index 4b194e68bda68..5e65bca62c639 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/control_utility.py +++ b/modules/ducktests/tests/ignitetest/services/utils/control_utility.py @@ -16,9 +16,11 @@ """ This module contains control utility wrapper. """ + import random import re -from collections import namedtuple +import time +from typing import NamedTuple from ducktape.cluster.remoteaccount import RemoteCommandError @@ -52,10 +54,10 @@ def set_baseline(self, baseline): :param baseline: Baseline nodes or topology version to set as baseline. """ if isinstance(baseline, int): - result = self.__run("--baseline version %d --yes" % baseline) + result = self.__run(f"--baseline version {baseline} --yes") else: - result = self.__run("--baseline set %s --yes" % - ",".join([node.account.externally_routable_ip for node in baseline])) + result = self.__run( + f"--baseline set {','.join([node.account.externally_routable_ip for node in baseline])} --yes") return self.__parse_cluster_state(result) @@ -63,8 +65,8 @@ def add_to_baseline(self, nodes): """ :param nodes: Nodes that should be added to baseline. """ - result = self.__run("--baseline add %s --yes" % - ",".join([node.account.externally_routable_ip for node in nodes])) + result = self.__run( + f"--baseline add {','.join([node.account.externally_routable_ip for node in nodes])} --yes") return self.__parse_cluster_state(result) @@ -72,8 +74,8 @@ def remove_from_baseline(self, nodes): """ :param nodes: Nodes that should be removed to baseline. """ - result = self.__run("--baseline remove %s --yes" % - ",".join([node.account.externally_routable_ip for node in nodes])) + result = self.__run( + f"--baseline remove {','.join([node.account.externally_routable_ip for node in nodes])} --yes") return self.__parse_cluster_state(result) @@ -88,8 +90,8 @@ def enable_baseline_auto_adjust(self, timeout=None): Enable baseline auto adjust. :param timeout: Auto adjust timeout in millis. """ - timeout_str = "timeout %d" % timeout if timeout else "" - return self.__run("--baseline auto_adjust enable %s --yes" % timeout_str) + timeout_str = f"timeout {timeout}" if timeout else "" + return self.__run(f"--baseline auto_adjust enable {timeout_str} --yes") def activate(self): """ @@ -103,6 +105,124 @@ def deactivate(self): """ return self.__run("--deactivate --yes") + def tx(self, **kwargs): + """ + Get list of transactions, various filters can be applied. + """ + output = self.__run(self.__tx_command(**kwargs)) + res = self.__parse_tx_list(output) + return res if res else output + + def tx_info(self, xid): + """ + Get verbose transaction info by xid. + """ + return self.__parse_tx_info(self.__run(f"--tx --info {xid}")) + + def tx_kill(self, **kwargs): + """ + Kill transaction by xid or by various filter. + """ + output = self.__run(self.__tx_command(kill=True, **kwargs)) + res = self.__parse_tx_list(output) + return res if res else output + + @staticmethod + def __tx_command(**kwargs): + tokens = ["--tx"] + + if 'xid' in kwargs: + tokens.append(f"--xid {kwargs['xid']}") + + if kwargs.get('clients'): + tokens.append("--clients") + + if kwargs.get('servers'): + tokens.append("--servers") + + if 'min_duration' in kwargs: + tokens.append(f"--min-duration {kwargs.get('min_duration')}") + + if 'min_size' in kwargs: + tokens.append(f"--min-size {kwargs.get('min_size')}") + + if 'label_pattern' in kwargs: + tokens.append(f"--label {kwargs['label_pattern']}") + + if kwargs.get("nodes"): + tokens.append(f"--nodes {','.join(kwargs.get('nodes'))}") + + if 'limit' in kwargs: + tokens.append(f"--limit {kwargs['limit']}") + + if 'order' in kwargs: + tokens.append(f"--order {kwargs['order']}") + + if kwargs.get('kill'): + tokens.append("--kill --yes") + + return " ".join(tokens) + + @staticmethod + def __parse_tx_info(output): + tx_info_pattern = re.compile( + "Near XID version: (?PGridCacheVersion \\[topVer=\\d+, order=\\d+, nodeOrder=\\d+\\])\\n\\s+" + "Near XID version \\(UUID\\): (?P[^\\s]+)\\n\\s+" + "Isolation: (?P[^\\s]+)\\n\\s+" + "Concurrency: (?P[^\\s]+)\\n\\s+" + "Timeout: (?P\\d+)\\n\\s+" + "Initiator node: (?P[^\\s]+)\\n\\s+" + "Initiator node \\(consistent ID\\): (?P[^\\s+]+)\\n\\s+" + "Label: (?P - + - """ + """ # noqa: E501 @staticmethod def properties(): @@ -185,8 +185,8 @@ def start_cell_with_prepared_txs(self, version, cell_id, joined_cluster=None): self.test_context, IgniteConfiguration(version=IgniteVersion(version), properties=self.properties(), discovery_spi=from_ignite_cluster(nodes)), # Server node. - java_class_name= - "org.apache.ignite.internal.ducktest.tests.cellular_affinity_test.CellularPreparedTxStreamer", + java_class_name="org.apache.ignite.internal.ducktest.tests.cellular_affinity_test." + "CellularPreparedTxStreamer", params={"cacheName": CellularAffinity.CACHE_NAME, "attr": CellularAffinity.ATTRIBUTE, "cell": cell_id, diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 8d806a46777b7..2c8d6a17c32b9 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -122,8 +122,11 @@ def _perform_node_fail_scenario(self, test_config): version=test_config.version, discovery_spi=discovery_spi, failure_detection_timeout=self.FAILURE_DETECTION_TIMEOUT, - caches=[CacheConfiguration(name='test-cache', backups=1, atomicity_mode='TRANSACTIONAL' if - test_config.load_type == ClusterLoad.TRANSACTIONAL else 'ATOMIC')] + caches=[CacheConfiguration( + name='test-cache', + backups=1, + atomicity_mode='TRANSACTIONAL' if test_config.load_type == ClusterLoad.TRANSACTIONAL else 'ATOMIC' + )] ) servers, start_servers_sec = start_servers(self.test_context, self.NUM_NODES - 1, ignite_config, modules) diff --git a/modules/ducktests/tests/ignitetest/utils/__init__.py b/modules/ducktests/tests/ignitetest/utils/__init__.py index 838c33343eff2..2f557eeeb15e6 100644 --- a/modules/ducktests/tests/ignitetest/utils/__init__.py +++ b/modules/ducktests/tests/ignitetest/utils/__init__.py @@ -13,5 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=C0114 +""" +This module contains convenient utils for test. +""" + from ._mark import version_if, ignite_versions + +__all__ = ['version_if', 'ignite_versions'] diff --git a/modules/ducktests/tests/tox.ini b/modules/ducktests/tests/tox.ini index 79551083bca51..809d713d89666 100644 --- a/modules/ducktests/tests/tox.ini +++ b/modules/ducktests/tests/tox.ini @@ -13,24 +13,30 @@ # See the License for the specific language governing permissions and # limitations under the License. [tox] -envlist = linter +envlist = codestyle,linter skipsdist = True [travis] python = - 3.8: linter + 3.8: codestyle, linter [testenv] envdir = {homedir}/.virtualenvs/ignite-ducktests-{envname} deps = -r ./docker/requirements.txt pylint==2.6.0 + flake8==3.8.3 [testenv:linter] basepython = python3.8 commands = pylint --rcfile=tox.ini ./ignitetest +[testenv:codestyle] +basepython = python3.8 +commands = + flake8 ./ignitetest + [BASIC] min-public-methods=0 good-names=i,j,k,x,y,ex,pk,tx @@ -40,3 +46,6 @@ ignore-imports=yes [FORMAT] max-line-length=120 + +[flake8] +max-line-length=120 From f7e1755137222826a2d21dd75bd7b37e4bdfaccf Mon Sep 17 00:00:00 2001 From: Maksim Timonin Date: Wed, 23 Sep 2020 11:57:21 +0300 Subject: [PATCH 53/78] Fix discovery templates discovery path (#8271) --- .../tests/ignitetest/services/utils/config_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/config_template.py b/modules/ducktests/tests/ignitetest/services/utils/config_template.py index 5170c4d37c1a0..875f12b35c73c 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/config_template.py +++ b/modules/ducktests/tests/ignitetest/services/utils/config_template.py @@ -32,7 +32,7 @@ def __init__(self, path): tmpl_dir = os.path.dirname(path) tmpl_file = os.path.basename(path) - tmpl_loader = FileSystemLoader(searchpath=tmpl_dir) + tmpl_loader = FileSystemLoader(searchpath=[DEFAULT_CONFIG_PATH, tmpl_dir]) env = Environment(loader=tmpl_loader) self.template = env.get_template(tmpl_file) From 0db59f7adbebef7fef111ff96de730f2ce3a0b73 Mon Sep 17 00:00:00 2001 From: Maksim Timonin Date: Wed, 23 Sep 2020 13:09:18 +0300 Subject: [PATCH 54/78] Remove tox from ignitetest setup.py due to version conflict with ducktape (#8272) --- modules/ducktests/tests/README.md | 15 ++++++++++++--- modules/ducktests/tests/setup.py | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/modules/ducktests/tests/README.md b/modules/ducktests/tests/README.md index cbf74823cec58..7ca3a83906605 100644 --- a/modules/ducktests/tests/README.md +++ b/modules/ducktests/tests/README.md @@ -7,7 +7,16 @@ the `ducktape` test framework, for information about it check the links: Structure of the `ignitetest` directory is: - `./ignitetest/services` contains basic services functionality; -- `./ignitetest/tests/utils` contains utils for testing. +- `./ignitetest/utils` contains utils for testing; +- `./ignitetest/tests` contains tests. -## Review Checklist -1. All tests must be parameterized with `version`. +Docker is used to emulate distributed environment. Single container represents +a running node. + +## Requirements +To just start tests locally the only requirement is preinstalled `docker`. + +For development process requirements are: +1. `python` >= 3.6; +2. `tox` (check python codestyle); +3. `docker`. diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index de849edb5327d..8ed664ffc058a 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -30,7 +30,7 @@ license="apache2.0", packages=find_packages(exclude=["ignitetest.tests", "ignitetest.tests.*"]), include_package_data=True, - install_requires=["ducktape==0.8.0", "tox==3.15.2"], + install_requires=["ducktape==0.8.0"], dependency_links=[ 'https://github.com/confluentinc/ducktape/tarball/master#egg=ducktape-0.8.0' ]) From cb0b2475249c419bc720ca1d7c4b0c2b06565965 Mon Sep 17 00:00:00 2001 From: oleg-ostanin <48506823+oleg-ostanin@users.noreply.github.com> Date: Thu, 24 Sep 2020 11:20:44 +0300 Subject: [PATCH 55/78] IGNITE-13434 add assertion test (#8259) --- .../smoke_test/AssertionApplication.java | 35 ++++++++++++ .../tests/ignitetest/services/ignite_app.py | 3 +- .../services/ignite_execution_exception.py | 24 ++++++++ .../tests/ignitetest/tests/assertion_test.py | 55 +++++++++++++++++++ 4 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/AssertionApplication.java create mode 100644 modules/ducktests/tests/ignitetest/services/ignite_execution_exception.py create mode 100644 modules/ducktests/tests/ignitetest/tests/assertion_test.py diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/AssertionApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/AssertionApplication.java new file mode 100644 index 0000000000000..0cb0d207e7724 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/smoke_test/AssertionApplication.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.smoke_test; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Application to check java assertions to python exception conversion + */ +public class AssertionApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jsonNode) { + assert false; + + markInitialized(); + + markFinished(); + } +} diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 74b7fba048364..8fbf0351a66a5 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -22,6 +22,7 @@ # pylint: disable=W0622 from ducktape.errors import TimeoutError +from ignitetest.services.ignite_execution_exception import IgniteExecutionException from ignitetest.services.utils.ignite_aware import IgniteAwareService @@ -81,7 +82,7 @@ def __check_status(self, desired, timeout=1): try: self.await_event("IGNITE_APPLICATION_BROKEN", 1, from_the_beginning=True) - raise Exception("Java application execution failed. %s" % self.extract_result("ERROR")) + raise IgniteExecutionException("Java application execution failed. %s" % self.extract_result("ERROR")) except TimeoutError: pass diff --git a/modules/ducktests/tests/ignitetest/services/ignite_execution_exception.py b/modules/ducktests/tests/ignitetest/services/ignite_execution_exception.py new file mode 100644 index 0000000000000..ce6cb562cfdc4 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/ignite_execution_exception.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Ignite execution exception +""" + + +class IgniteExecutionException(Exception): + """ + Ignite execution exception implementation + """ diff --git a/modules/ducktests/tests/ignitetest/tests/assertion_test.py b/modules/ducktests/tests/ignitetest/tests/assertion_test.py new file mode 100644 index 0000000000000..5ab9c90cb1ffd --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/assertion_test.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains smoke tests that checks that ducktape works as expected +""" + +from ducktape.mark.resource import cluster + +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.ignite_execution_exception import IgniteExecutionException +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration +from ignitetest.utils import ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + + +# pylint: disable=W0223 +class SmokeSelfTest(IgniteTest): + """ + Self test + """ + + @cluster(num_nodes=1) + @ignite_versions(str(DEV_BRANCH)) + def test_assertion_convertion(self, ignite_version): + """ + Test to make sure Java assertions are converted to python exceptions + """ + server_configuration = IgniteConfiguration(version=IgniteVersion(ignite_version)) + + app = IgniteApplicationService( + self.test_context, + server_configuration, + java_class_name="org.apache.ignite.internal.ducktest.tests.smoke_test.AssertionApplication") + + try: + app.start() + except IgniteExecutionException as ex: + assert str(ex) == "Java application execution failed. java.lang.AssertionError" + else: + app.stop() + assert False From cedb251b5d2e50a9a65257a65eb291779692dfd6 Mon Sep 17 00:00:00 2001 From: Ivan Daschinskiy Date: Fri, 25 Sep 2020 18:17:56 +0300 Subject: [PATCH 56/78] =?UTF-8?q?IGNITE-13481=20Fix=20decorators=20@ignite?= =?UTF-8?q?=5Fversions=20and=20@version=5Fif.=20Introdu=E2=80=A6=20(#8274)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 24 +- modules/ducktests/tests/README.md | 54 +++- .../tests/checks/utils/check_marks.py | 238 ++++++++++++++++++ .../tests/docker/requirements-dev.txt | 19 ++ .../ducktests/tests/docker/requirements.txt | 2 - .../ducktests/tests/ignitetest/utils/_mark.py | 20 +- modules/ducktests/tests/setup.py | 1 + modules/ducktests/tests/tox.ini | 34 ++- 8 files changed, 363 insertions(+), 29 deletions(-) create mode 100644 modules/ducktests/tests/checks/utils/check_marks.py create mode 100644 modules/ducktests/tests/docker/requirements-dev.txt diff --git a/.travis.yml b/.travis.yml index afb930fadb2f6..c2dbd305a0ff1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +_ducktape-tox: &ducktape-tox + install: pip install tox + before_script: cd modules/ducktests/tests + matrix: include: - language: java @@ -52,9 +56,19 @@ matrix: - dotnet build modules/platforms/dotnet/Apache.Ignite.DotNetCore.sln - language: python - python: - - "3.8" - install: pip install tox-travis - before_script: cd modules/ducktests/tests + python: 3.6.12 + <<: *ducktape-tox + script: + - tox -e py36 + + - language: python + python: 3.7.9 + <<: *ducktape-tox + script: + - tox -e py37 + + - language: python + python: 3.8.5 + <<: *ducktape-tox script: - - tox + - tox -e linter,codestyle,py38 diff --git a/modules/ducktests/tests/README.md b/modules/ducktests/tests/README.md index 7ca3a83906605..18a2a63e30fd5 100644 --- a/modules/ducktests/tests/README.md +++ b/modules/ducktests/tests/README.md @@ -2,21 +2,59 @@ The `ignitetest` framework provides basic functionality and services to write integration tests for Apache Ignite. This framework bases on the `ducktape` test framework, for information about it check the links: -- https://github.com/confluentinc/ducktape - source code of the `ducktape`; +- https://github.com/confluentinc/ducktape - source code of the `ducktape`. - http://ducktape-docs.readthedocs.io - documentation to the `ducktape`. -Structure of the `ignitetest` directory is: -- `./ignitetest/services` contains basic services functionality; -- `./ignitetest/utils` contains utils for testing; +Structure of the `tests` directory is: +- `./ignitetest/services` contains basic services functionality. +- `./ignitetest/utils` contains utils for testing. - `./ignitetest/tests` contains tests. +- `./checks` contains unit tests of utils, tests' decorators etc. Docker is used to emulate distributed environment. Single container represents a running node. ## Requirements To just start tests locally the only requirement is preinstalled `docker`. +For development process requirements are `python` >= 3.6. -For development process requirements are: -1. `python` >= 3.6; -2. `tox` (check python codestyle); -3. `docker`. +## Run tests locally +- Change a current directory to`${IGNITE_HOME}` +- Build Apache IGNITE invoking `${IGNITE_HOME}/scripts/build.sh` +- Change a current directory to `${IGNITE_HOME}/modules/ducktests/tests` +- Run tests in docker containers using a following command: +``` +./docker/run_tests.sh +``` +- For detailed help and instructions, use a following command: +``` +./docker/run_tests.sh --help +``` + +## Preparing development environment +- Create a virtual environment and activate it using following commands: +``` +python3 -m venv ~/.virtualenvs/ignite-ducktests-dev +source ~/.virtualenvs/ignite-ducktests-dev/bin/activate +``` +- Change a current directory to `${IGNITE_HOME}/modules/ducktests/tests`. We refer to it as `${DUCKTESTS_DIR}`. +- Install requirements and `ignitetests` as editable using following commands: +``` +pip install -r docker/requirements-dev.txt +pip install -e . +``` +--- + +- For running unit tests invoke `pytest` in `${DUCKTESTS_DIR}`. +- For checking codestyle invoke `flake8` in `${DUCKTESTS_DIR}`. +- For running linter invoke `pylint --rcfile=tox.ini ignitetests checks` in `${DUCKTESTS_DIR}`. + +#### Run checks over multiple python's versions using tox (optional) +All commits and PR's are checked against multiple python's version, namely 3.6, 3.7 and 3.8. +If you want to check your PR as it will be checked on Travis CI, you should do following steps: + +- Install `pyenv`, see installation instruction [here](https://github.com/pyenv/pyenv#installation). +- Install different versions of python (recommended versions are `3.6.12`, `3.7.9`, `3.8.5`) +- Activate them with a command `pyenv shell 3.6.12 3.7.9 3.8.5` +- Install `tox` by invoking a command `pip install tox` +- Change a current directory to `${DUCKTESTS_DIR}` and invoke `tox` diff --git a/modules/ducktests/tests/checks/utils/check_marks.py b/modules/ducktests/tests/checks/utils/check_marks.py new file mode 100644 index 0000000000000..5eddb3f07cb93 --- /dev/null +++ b/modules/ducktests/tests/checks/utils/check_marks.py @@ -0,0 +1,238 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks custom parametrizers. +""" + +import itertools +from unittest.mock import Mock + +import pytest +from ducktape.mark import parametrized, parametrize, matrix, ignore +from ducktape.mark.mark_expander import MarkedFunctionExpander + +from ignitetest.utils._mark import IgniteVersionParametrize, ignite_versions, version_if +from ignitetest.utils.version import IgniteVersion, V_2_8_0, V_2_8_1, V_2_7_6, DEV_BRANCH + + +def expand_function(*, func, sess_ctx): + """ + Inject parameters into function and generate context list. + """ + assert parametrized(func) + assert next(filter(lambda x: isinstance(x, IgniteVersionParametrize), func.marks), None) + + return MarkedFunctionExpander(session_context=sess_ctx, function=func).expand() + + +def mock_session_ctx(*, global_args=None): + """ + Create mock of session context. + """ + sess_ctx = Mock() + sess_ctx.globals = global_args if global_args else {} + + return sess_ctx + + +class CheckIgniteVersions: + """ + Checks @ignite_version parametrization. + """ + single_params = itertools.product( + [[str(V_2_8_1)], [str(V_2_8_1), str(DEV_BRANCH)]], + [{}, {'ignite_versions': 'dev'}, {'ignite_versions': ['2.8.1', 'dev']}] + ) + + @pytest.mark.parametrize( + ['versions', 'global_args'], + map(lambda x: pytest.param(x[0], x[1]), single_params) + ) + def check_injection(self, versions, global_args): + """ + Checks parametrization with single version. + """ + @ignite_versions(*versions, version_prefix='ver') + def function(ver): + return IgniteVersion(ver) + + context_list = expand_function(func=function, sess_ctx=mock_session_ctx(global_args=global_args)) + + self._check_injection(context_list, versions=versions, global_args=global_args) + + pair_params = itertools.product( + [[(str(DEV_BRANCH), str(V_2_8_1))], [(str(DEV_BRANCH), str(V_2_8_0)), (str(DEV_BRANCH), str(V_2_8_1))]], + [{}, {'ignite_versions': [['2.8.1', '2.7.6'], ['2.8.1', '2.8.0']]}, {'ignite_versions': [['dev', '2.8.1']]}] + ) + + @pytest.mark.parametrize( + ['versions', 'global_args'], + map(lambda x: pytest.param(x[0], x[1]), pair_params) + ) + def check_injection_pairs(self, versions, global_args): + """ + Checks parametrization with pair of versions. + """ + @ignite_versions(*versions, version_prefix='pair') + def function(pair_1, pair_2): + return IgniteVersion(pair_1), IgniteVersion(pair_2) + + context_list = expand_function(func=function, sess_ctx=mock_session_ctx(global_args=global_args)) + + self._check_injection(context_list, versions=versions, global_args=global_args, pairs=True) + + @pytest.mark.parametrize( + ['versions', 'version_prefix', 'global_args'], + [ + pytest.param([(DEV_BRANCH, V_2_8_1)], 'ver', {}), + pytest.param([DEV_BRANCH], 'ver', {'ignite_versions': [['dev', '2.8.1']]}), + pytest.param([DEV_BRANCH], 'invalid_prefix', {}) + ] + ) + def check_injection_fail(self, versions, version_prefix, global_args): + """ + Check incorrect injecting variables with single parameter. + """ + @ignite_versions(*versions, version_prefix=version_prefix) + def function(ver): + return IgniteVersion(ver) + + with pytest.raises(Exception): + context_list = expand_function(func=function, sess_ctx=mock_session_ctx(global_args=global_args)) + + self._check_injection(context_list, versions=versions, global_args=global_args) + + @pytest.mark.parametrize( + ['versions', 'version_prefix', 'global_args'], + [ + pytest.param([DEV_BRANCH, V_2_8_1], 'pair', {}), + pytest.param([(DEV_BRANCH, V_2_8_1)], 'pair', {'ignite_versions': 'dev'}), + pytest.param([(DEV_BRANCH, V_2_8_1)], 'pair', {'ignite_versions': ['dev', '2.8.1']}), + pytest.param([(DEV_BRANCH, V_2_8_1)], 'invalid_prefix', {}) + ] + ) + def check_injection_pairs_fail(self, versions, version_prefix, global_args): + """ + Check incorrect injecting with pairs of versions. + """ + @ignite_versions(*versions, version_prefix=version_prefix) + def function(pair_1, pair_2): + return IgniteVersion(pair_1), IgniteVersion(pair_2) + + with pytest.raises(Exception): + context_list = expand_function(func=function, sess_ctx=mock_session_ctx(global_args=global_args)) + + self._check_injection(context_list, versions=versions, global_args=global_args, pairs=True) + + def check_with_others_marks(self): # pylint: disable=R0201 + """ + Checks that ignite version parametrization works with others correctly. + """ + @ignite_versions(str(DEV_BRANCH), str(V_2_8_1), version_prefix='ver') + @parametrize(x=10, y=20) + @parametrize(x=30, y=40) + def function_parametrize(ver, x, y): + return ver, x, y + + @ignite_versions((str(DEV_BRANCH), str(V_2_8_1)), (str(V_2_8_1), str(V_2_7_6)), version_prefix='pair') + @matrix(i=[10, 20], j=[30, 40]) + def function_matrix(pair_1, pair_2, i, j): + return pair_1, pair_2, i, j + + @ignore(ver=str(DEV_BRANCH)) + @ignite_versions(str(DEV_BRANCH), str(V_2_8_1), version_prefix='ver') + def function_ignore(ver): + return ver + + context_list = expand_function(func=function_parametrize, sess_ctx=mock_session_ctx()) + context_list += expand_function(func=function_matrix, sess_ctx=mock_session_ctx()) + context_list += expand_function(func=function_ignore, sess_ctx=mock_session_ctx()) + + assert len(context_list) == 14 + + parametrized_context = list(filter(lambda x: x.function_name == function_parametrize.__name__, context_list)) + assert len(parametrized_context) == 4 + for ctx in parametrized_context: + args = ctx.injected_args + assert len(args) == 3 + assert ctx.function() == (args['ver'], args['x'], args['y']) + + matrix_context = list(filter(lambda x: x.function_name == function_matrix.__name__, context_list)) + assert len(matrix_context) == 8 + for ctx in matrix_context: + args = ctx.injected_args + assert len(args) == 4 + assert ctx.function() == (args['pair_1'], args['pair_2'], args['i'], args['j']) + + assert len(list(filter(lambda x: x.function_name == function_ignore.__name__, context_list))) == 2 + assert len(list(filter(lambda x: x.ignore, context_list))) == 1 + + @staticmethod + def _check_injection(context_list, *, versions, global_args=None, pairs=False): + if global_args: + global_versions = global_args['ignite_versions'] + + if isinstance(global_versions, str): + check_versions = [IgniteVersion(global_versions)] + elif isinstance(global_args['ignite_versions'], tuple): + check_versions = [tuple(map(IgniteVersion, global_versions))] + elif pairs: + check_versions = list(map(lambda x: (IgniteVersion(x[0]), IgniteVersion(x[1])), global_versions)) + else: + check_versions = list(map(IgniteVersion, global_versions)) + else: + if not pairs: + check_versions = list(map(IgniteVersion, versions)) + else: + check_versions = list(map(lambda x: (IgniteVersion(x[0]), IgniteVersion(x[1])), versions)) + + assert len(context_list) == len(check_versions) + + for i, ctx in enumerate(sorted(context_list, key=lambda x: x.function())): + assert ctx.function() == check_versions[i] + + +class CheckVersionIf: + """ + Checks @version_if parametrization. + """ + def check_common(self): # pylint: disable=R0201 + """ + Check common scenarios with @ignite_versions parametrization. + """ + @version_if(lambda ver: ver != V_2_8_0, variable_name='ver') + @ignite_versions(str(DEV_BRANCH), str(V_2_8_0), version_prefix='ver') + def function_1(ver): + return IgniteVersion(ver) + + @version_if(lambda ver: ver > V_2_7_6, variable_name='ver_1') + @version_if(lambda ver: ver < V_2_8_0, variable_name='ver_2') + @ignite_versions((str(V_2_8_1), str(V_2_8_0)), (str(V_2_8_0), str(V_2_7_6)), version_prefix='ver') + def function_2(ver_1, ver_2): + return IgniteVersion(ver_1), IgniteVersion(ver_2) + + @ignite_versions(str(DEV_BRANCH), str(V_2_8_0)) + def function_3(ignite_version): + return IgniteVersion(ignite_version) + + context_list = expand_function(func=function_1, sess_ctx=mock_session_ctx()) + context_list += expand_function(func=function_2, sess_ctx=mock_session_ctx()) + context_list += expand_function(func=function_3, sess_ctx=mock_session_ctx()) + + assert len(context_list) == 6 + + assert next(filter(lambda x: x.injected_args['ver'] == str(V_2_8_0), context_list)).ignore + assert not next(filter(lambda x: x.injected_args['ver'] == str(DEV_BRANCH), context_list)).ignore diff --git a/modules/ducktests/tests/docker/requirements-dev.txt b/modules/ducktests/tests/docker/requirements-dev.txt new file mode 100644 index 0000000000000..c92e672a53e1c --- /dev/null +++ b/modules/ducktests/tests/docker/requirements-dev.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +-r requirements.txt +pytest==6.0.1 +pylint==2.6.0 +flake8==3.8.3 diff --git a/modules/ducktests/tests/docker/requirements.txt b/modules/ducktests/tests/docker/requirements.txt index 137f8040c9115..e209504b65b84 100644 --- a/modules/ducktests/tests/docker/requirements.txt +++ b/modules/ducktests/tests/docker/requirements.txt @@ -13,6 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -pytest==6.0.1 -mock==4.0.2 git+https://github.com/confluentinc/ducktape diff --git a/modules/ducktests/tests/ignitetest/utils/_mark.py b/modules/ducktests/tests/ignitetest/utils/_mark.py index 99f328f25af0c..56bfabd07f82e 100644 --- a/modules/ducktests/tests/ignitetest/utils/_mark.py +++ b/modules/ducktests/tests/ignitetest/utils/_mark.py @@ -37,10 +37,10 @@ def apply(self, seed_context, context_list): assert len(context_list) > 0, "ignore_if decorator is not being applied to any test cases" for ctx in context_list: - assert self.variable_name in ctx.injected_args, "'%s' in injected args not present" % (self.variable_name,) - version = ctx.injected_args[self.variable_name] - assert isinstance(version, str), "'%s'n injected args must be a string" % (self.variable_name,) - ctx.ignore = ctx.ignore or not self.condition(IgniteVersion(version)) + if self.variable_name in ctx.injected_args: + version = ctx.injected_args[self.variable_name] + assert isinstance(version, str), "'%s'n injected args must be a string" % (self.variable_name,) + ctx.ignore = ctx.ignore or not self.condition(IgniteVersion(version)) return context_list @@ -75,7 +75,7 @@ def apply(self, seed_context, context_list): if context_list: for ctx in context_list: for version in self.versions: - if ctx.injected_args and 'ignite_version' in ctx.injected_args: + if self._check_injected(ctx): continue new_context_list.insert(0, self._prepare_new_ctx(version, seed_context, ctx)) @@ -101,6 +101,14 @@ def _prepare_new_ctx(self, version, seed_context, ctx=None): return seed_context.copy(function=injected_fun, injected_args=injected_args) + def _check_injected(self, ctx): + if ctx.injected_args: + for arg_name in ctx.injected_args.keys(): + if arg_name.startswith(self.version_prefix): + return True + + return False + @property def name(self): """ @@ -126,7 +134,7 @@ def parametrizer(func): return parametrizer -def version_if(condition, variable_name='ignite_version'): +def version_if(condition, *, variable_name='ignite_version'): """ Mark decorated test method as IGNORE if version doesn't corresponds to condition. diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py index 8ed664ffc058a..ee2d8396f2d12 100644 --- a/modules/ducktests/tests/setup.py +++ b/modules/ducktests/tests/setup.py @@ -31,6 +31,7 @@ packages=find_packages(exclude=["ignitetest.tests", "ignitetest.tests.*"]), include_package_data=True, install_requires=["ducktape==0.8.0"], + tests_require=["pytest==6.0.1"], dependency_links=[ 'https://github.com/confluentinc/ducktape/tarball/master#egg=ducktape-0.8.0' ]) diff --git a/modules/ducktests/tests/tox.ini b/modules/ducktests/tests/tox.ini index 809d713d89666..92cc37cd83e13 100644 --- a/modules/ducktests/tests/tox.ini +++ b/modules/ducktests/tests/tox.ini @@ -13,33 +13,46 @@ # See the License for the specific language governing permissions and # limitations under the License. [tox] -envlist = codestyle,linter +envlist = codestyle, linter, py36, py37, py38 skipsdist = True [travis] python = - 3.8: codestyle, linter + 3.8: codestyle, linter, py38 + 3.7: py37 + 3.6: py36 [testenv] envdir = {homedir}/.virtualenvs/ignite-ducktests-{envname} deps = - -r ./docker/requirements.txt - pylint==2.6.0 - flake8==3.8.3 + -r ./docker/requirements-dev.txt +recreate = False +usedevelop = True +commands = + pytest {env:PYTESTARGS:} {posargs} [testenv:linter] basepython = python3.8 commands = - pylint --rcfile=tox.ini ./ignitetest + pylint --rcfile=tox.ini ignitetest checks [testenv:codestyle] basepython = python3.8 commands = - flake8 ./ignitetest + flake8 + +[testenv:py36] +envdir = {homedir}/.virtualenvs/ignite-ducktests-py36 + +[testenv:py37] +envdir = {homedir}/.virtualenvs/ignite-ducktests-py37 + +[testenv:py38] +envdir = {homedir}/.virtualenvs/ignite-ducktests-py38 [BASIC] min-public-methods=0 -good-names=i,j,k,x,y,ex,pk,tx +good-names=i, j, k, x, y, ex, pk, tx [SIMILARITIES] ignore-imports=yes @@ -49,3 +62,8 @@ max-line-length=120 [flake8] max-line-length=120 + +[pytest] +python_files=check_*.py +python_classes=Check +python_functions=check_* From 7fbe36964c4e94253ff779ecbf13668c45a1db24 Mon Sep 17 00:00:00 2001 From: Maksim Timonin Date: Wed, 30 Sep 2020 11:54:50 +0300 Subject: [PATCH 57/78] Update README.md (#8288) --- modules/ducktests/tests/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ducktests/tests/README.md b/modules/ducktests/tests/README.md index 18a2a63e30fd5..fe9f64f88aac0 100644 --- a/modules/ducktests/tests/README.md +++ b/modules/ducktests/tests/README.md @@ -30,6 +30,7 @@ For development process requirements are `python` >= 3.6. ``` ./docker/run_tests.sh --help ``` +- Test reports, including service logs, are located in the `${IGNITE_HOME}/results` directory. ## Preparing development environment - Create a virtual environment and activate it using following commands: From a50949a1546df9d7b0e54288b566b4566a7b6fab Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 5 Oct 2020 17:13:59 +0300 Subject: [PATCH 58/78] typo fix --- .../ignite/internal/ducktest/utils/IgniteAwareApplication.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java index 35760b367b1f1..f75a5b6d600be 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplication.java @@ -168,7 +168,7 @@ public void markBroken(Throwable th) { * */ private void terminate() { - log.info("Marking as initialized."); + log.info("Marking as terminated."); synchronized (stateMux) { assert !terminated; From d9decd748bec3e26c0b614966fd38350d91be50f Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Wed, 7 Oct 2020 10:28:14 +0300 Subject: [PATCH 59/78] Pme test fix (#8321) --- .../CellularPreparedTxStreamer.java | 2 +- .../CellularTxStreamer.java | 4 ++-- .../LongTxStreamerApplication.java | 21 +++++++++++++++---- .../SingleKeyTxStreamerApplication.java | 2 +- .../tests/cellular_affinity_test.py | 6 +++--- .../ignitetest/tests/pme_free_switch_test.py | 2 ++ 6 files changed, 26 insertions(+), 11 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularPreparedTxStreamer.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularPreparedTxStreamer.java index 43424eb05889e..9c7a97eb484f4 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularPreparedTxStreamer.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularPreparedTxStreamer.java @@ -79,7 +79,7 @@ public class CellularPreparedTxStreamer extends IgniteAwareApplication { i--; } - log.info("All transactions prepared (" + cnt + ")"); + log.info("ALL_TRANSACTIONS_PREPARED (" + cnt + ")"); while (!terminated()) { log.info("Waiting for SIGTERM."); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java index ebfc6f2464c00..6ab11285c81e0 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java @@ -84,7 +84,7 @@ record = true; initTime = System.currentTimeMillis(); - log.info("Warmup finished"); + log.info("WARMUP_FINISHED"); } if (record) { @@ -100,7 +100,7 @@ record = true; } if (cnt % 1000 == 0) - log.info("Application streamed " + cnt + " transactions [worst_latency=" + Arrays.toString(max) + "]"); + log.info("APPICATION_STREAMED " + cnt + " transactions [worst_latency=" + Arrays.toString(max) + "]"); } recordResult("WORST_LATENCY", Arrays.toString(max)); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java index 7e3f146573ffd..f83d1252e5070 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java @@ -61,18 +61,23 @@ public class LongTxStreamerApplication extends IgniteAwareApplication { if (tx.state() != TransactionState.ACTIVE) { log.info("Transaction broken. [key=" + finalI + "]"); - break; + markBroken(new IllegalStateException( + "Illegal Tx state [key=" + finalI + " state=" + tx.state() + "]")); } try { - U.sleep(1000); + U.sleep(10); } catch (IgniteInterruptedCheckedException ignored) { // No-op. } } - log.info("Stopping tx thread [state=" + tx.state() + "]"); + log.info("Stopping tx thread [key=" + finalI + " state=" + tx.state() + "]"); + + tx.rollback(); + + log.info("Finishing tx thread [key=" + finalI + " state=" + tx.state() + "]"); }).start(); } @@ -91,10 +96,18 @@ public class LongTxStreamerApplication extends IgniteAwareApplication { U.sleep(100); // Keeping node/txs alive. } catch (IgniteInterruptedCheckedException ignored) { - log.info("Waiting interrupted."); + log.info("Waiting for interrupted."); } } + while (!((IgniteEx)ignite).context().cache().context().tm().activeTransactions().isEmpty()) + try { + U.sleep(100); // Keeping node alive. + } + catch (IgniteInterruptedCheckedException ignored) { + log.info("Waiting for tx rollback."); + } + markFinished(); } } diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java index 14b53a900be0f..5f1fa99df6119 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java @@ -66,7 +66,7 @@ record = true; } if (cnt % 1000 == 0) - log.info("Streamed " + cnt + " transactions [max=" + max + "]"); + log.info("APPICATION_STREAMED " + cnt + " transactions [max=" + max + "]"); } recordResult("WORST_LATENCY", max); diff --git a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py index 5d40f173dc12b..18fb9f7d5b225 100644 --- a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py +++ b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py @@ -128,10 +128,10 @@ def test_latency(self, ignite_version): ControlUtility(cell1, self.test_context).activate() for loader in loaders: - loader.await_event("All transactions prepared", 180, from_the_beginning=True) + loader.await_event("ALL_TRANSACTIONS_PREPARED", 180, from_the_beginning=True) for streamer in streamers: - streamer.await_event("Warmup finished", 180, from_the_beginning=True) + streamer.await_event("WARMUP_FINISHED", 180, from_the_beginning=True) failed_loader.stop_async() # node left with prepared txs. @@ -142,7 +142,7 @@ def test_latency(self, ignite_version): streamer.await_event("exchangeFreeSwitch=true", 60, from_the_beginning=True) for streamer in streamers: # waiting for streaming continuation. - streamer.await_event("Application streamed", 60) + streamer.await_event("APPICATION_STREAMED", 60) for streamer in streamers: # stops streaming and records results. streamer.stop_async() diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index 3b54762d28509..7623ecbf046ba 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -87,6 +87,8 @@ def test(self, ignite_version): long_tx_streamer.stop() + single_key_tx_streamer.await_event("APPICATION_STREAMED", 60) # waiting for streaming continuation. + single_key_tx_streamer.stop() data["Worst latency (ms)"] = single_key_tx_streamer.extract_result("WORST_LATENCY") From c435d93f1b44664fa9714694d222f2c46970f606 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Wed, 7 Oct 2020 10:52:53 +0300 Subject: [PATCH 60/78] typo fix --- .../tests/cellular_affinity_test/CellularTxStreamer.java | 2 +- .../pme_free_switch_test/SingleKeyTxStreamerApplication.java | 2 +- .../ducktests/tests/ignitetest/tests/cellular_affinity_test.py | 2 +- .../ducktests/tests/ignitetest/tests/pme_free_switch_test.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java index 6ab11285c81e0..0fc9b7202e66c 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java @@ -100,7 +100,7 @@ record = true; } if (cnt % 1000 == 0) - log.info("APPICATION_STREAMED " + cnt + " transactions [worst_latency=" + Arrays.toString(max) + "]"); + log.info("APPLICATION_STREAMED " + cnt + " transactions [worst_latency=" + Arrays.toString(max) + "]"); } recordResult("WORST_LATENCY", Arrays.toString(max)); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java index 5f1fa99df6119..568859b6abdeb 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java @@ -66,7 +66,7 @@ record = true; } if (cnt % 1000 == 0) - log.info("APPICATION_STREAMED " + cnt + " transactions [max=" + max + "]"); + log.info("APPLICATION_STREAMED " + cnt + " transactions [max=" + max + "]"); } recordResult("WORST_LATENCY", max); diff --git a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py index 18fb9f7d5b225..3d105092147a7 100644 --- a/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py +++ b/modules/ducktests/tests/ignitetest/tests/cellular_affinity_test.py @@ -142,7 +142,7 @@ def test_latency(self, ignite_version): streamer.await_event("exchangeFreeSwitch=true", 60, from_the_beginning=True) for streamer in streamers: # waiting for streaming continuation. - streamer.await_event("APPICATION_STREAMED", 60) + streamer.await_event("APPLICATION_STREAMED", 60) for streamer in streamers: # stops streaming and records results. streamer.stop_async() diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index 7623ecbf046ba..da96ab20ccc98 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -87,7 +87,7 @@ def test(self, ignite_version): long_tx_streamer.stop() - single_key_tx_streamer.await_event("APPICATION_STREAMED", 60) # waiting for streaming continuation. + single_key_tx_streamer.await_event("APPLICATION_STREAMED", 60) # waiting for streaming continuation. single_key_tx_streamer.stop() From 241e3042ad1f1a416fb939261e720595fabb57b7 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Thu, 8 Oct 2020 10:31:48 +0300 Subject: [PATCH 61/78] IGNITE-13552 PME-free test should check both drops (PME duration vs PME-free, Tx wait vs wait-free) (#8326) --- .../ignitetest/tests/pme_free_switch_test.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index da96ab20ccc98..754b7799e6f0c 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -19,6 +19,7 @@ import time +from ducktape.mark import matrix from ducktape.mark.resource import cluster from ignitetest.services.ignite import IgniteService @@ -38,18 +39,29 @@ class PmeFreeSwitchTest(IgniteTest): Tests PME free switch scenarios. """ NUM_NODES = 3 + CACHES_AMOUNT = 100 @cluster(num_nodes=NUM_NODES + 2) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_7)) - def test(self, ignite_version): + @matrix(long_txs=[False, True]) + def test(self, ignite_version, long_txs): """ - Tests PME free scenario (node stop). + Tests PME-free switch scenario (node stop). """ data = {} + caches = [CacheConfiguration(name='test-cache', backups=2, atomicity_mode='TRANSACTIONAL')] + + # Checking PME (before 2.8) vs PME-free (2.8+) switch duration, but + # focusing on switch duration (which depends on caches amount) when long_txs is false and + # on waiting for previously started txs before the switch (which depends on txs duration) when long_txs of true. + if not long_txs: + for idx in range(1, self.CACHES_AMOUNT): + caches.append(CacheConfiguration(name="cache-%d" % idx, backups=2, atomicity_mode='TRANSACTIONAL')) + config = IgniteConfiguration( version=IgniteVersion(ignite_version), - caches=[CacheConfiguration(name='test-cache', backups=2, atomicity_mode='TRANSACTIONAL')] + caches=caches ) ignites = IgniteService(self.test_context, config, num_nodes=self.NUM_NODES) @@ -65,7 +77,8 @@ def test(self, ignite_version): java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.LongTxStreamerApplication", params={"cacheName": "test-cache"}) - long_tx_streamer.start() + if long_txs: + long_tx_streamer.start() single_key_tx_streamer = IgniteApplicationService( self.test_context, @@ -81,11 +94,12 @@ def test(self, ignite_version): ignites.stop_node(ignites.nodes[self.NUM_NODES - 1]) - long_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True) + if long_txs: + long_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True) - time.sleep(30) # keeping txs alive for 30 seconds. + time.sleep(30) # keeping txs alive for 30 seconds. - long_tx_streamer.stop() + long_tx_streamer.stop() single_key_tx_streamer.await_event("APPLICATION_STREAMED", 60) # waiting for streaming continuation. From 281458e38d3a8bd6b68302029d2b81c94094e41c Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Mon, 12 Oct 2020 17:15:34 +0300 Subject: [PATCH 62/78] IGNITE-13571 (#8349) --- .../ignitetest/tests/pme_free_switch_test.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py index 754b7799e6f0c..d215343d82c7f 100644 --- a/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py +++ b/modules/ducktests/tests/ignitetest/tests/pme_free_switch_test.py @@ -38,7 +38,7 @@ class PmeFreeSwitchTest(IgniteTest): """ Tests PME free switch scenarios. """ - NUM_NODES = 3 + NUM_NODES = 9 CACHES_AMOUNT = 100 @cluster(num_nodes=NUM_NODES + 2) @@ -59,15 +59,17 @@ def test(self, ignite_version, long_txs): for idx in range(1, self.CACHES_AMOUNT): caches.append(CacheConfiguration(name="cache-%d" % idx, backups=2, atomicity_mode='TRANSACTIONAL')) - config = IgniteConfiguration( - version=IgniteVersion(ignite_version), - caches=caches - ) + config = IgniteConfiguration(version=IgniteVersion(ignite_version), caches=caches, cluster_state="INACTIVE") ignites = IgniteService(self.test_context, config, num_nodes=self.NUM_NODES) ignites.start() + if IgniteVersion(ignite_version) >= V_2_8_0: + ControlUtility(ignites, self.test_context).disable_baseline_auto_adjust() + + ControlUtility(ignites, self.test_context).activate() + client_config = config._replace(client_mode=True, discovery_spi=from_ignite_cluster(ignites, slice(0, self.NUM_NODES - 1))) @@ -75,7 +77,8 @@ def test(self, ignite_version, long_txs): self.test_context, client_config, java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.LongTxStreamerApplication", - params={"cacheName": "test-cache"}) + params={"cacheName": "test-cache"}, + timeout_sec=180) if long_txs: long_tx_streamer.start() @@ -85,13 +88,11 @@ def test(self, ignite_version, long_txs): client_config, java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test." "SingleKeyTxStreamerApplication", - params={"cacheName": "test-cache", "warmup": 1000}) + params={"cacheName": "test-cache", "warmup": 1000}, + timeout_sec=180) single_key_tx_streamer.start() - if IgniteVersion(ignite_version) >= V_2_8_0: - ControlUtility(ignites, self.test_context).disable_baseline_auto_adjust() - ignites.stop_node(ignites.nodes[self.NUM_NODES - 1]) if long_txs: From b70d2ddc61156e748a879f2ddd207e00d2dbe868 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Tue, 20 Oct 2020 12:47:03 +0300 Subject: [PATCH 63/78] JVM_OPTS fixed --- bin/control.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/control.sh b/bin/control.sh index c3966d31b5284..b065a3d888010 100755 --- a/bin/control.sh +++ b/bin/control.sh @@ -174,7 +174,7 @@ elif [ $version -ge 11 ] ; then ${CONTROL_JVM_OPTS}" fi -if [ -n "${JVM_OPTS}" ] ; then +if [ -n "${JVM_OPTS:-}" ] ; then echo "JVM_OPTS environment variable is set, but will not be used. To pass JVM options use CONTROL_JVM_OPTS" echo "JVM_OPTS=${JVM_OPTS}" fi From 8e9350e676ac7ab9a631cba12c28ff69ff2ed281 Mon Sep 17 00:00:00 2001 From: Vladsz83 Date: Wed, 21 Oct 2020 10:09:55 +0300 Subject: [PATCH 64/78] Ducktests iptables (#8211) --- .../tests/ignitetest/services/ignite.py | 58 ++-- .../ignitetest/services/utils/ignite_aware.py | 3 +- .../utils/ignite_configuration/__init__.py | 3 + .../ignite_configuration/communication.py | 45 +++ .../utils/ignite_configuration/discovery.py | 11 +- .../utils/templates/communication_macro.j2 | 25 ++ .../utils/templates/discovery_macro.j2 | 32 +- .../services/utils/templates/ignite.xml.j2 | 4 + .../zk/templates/zookeeper.properties.j2 | 1 + .../tests/ignitetest/services/zk/zookeeper.py | 13 +- .../tests/ignitetest/tests/discovery_test.py | 287 +++++++++++++----- .../tests/ignitetest/utils/ignite_test.py | 26 ++ 12 files changed, 389 insertions(+), 119 deletions(-) create mode 100644 modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py create mode 100644 modules/ducktests/tests/ignitetest/services/utils/templates/communication_macro.j2 diff --git a/modules/ducktests/tests/ignitetest/services/ignite.py b/modules/ducktests/tests/ignitetest/services/ignite.py index f50365d7c156e..ad88db239d4e4 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite.py +++ b/modules/ducktests/tests/ignitetest/services/ignite.py @@ -17,9 +17,8 @@ This module contains class to start ignite cluster node. """ -import functools -import operator import os +import re import signal import time from datetime import datetime @@ -70,7 +69,7 @@ def stop_node(self, node, clean_shutdown=True, timeout_sec=60): sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL for pid in pids: - self.__stop_node(node, pid, sig) + node.account.signal(pid, sig, allow_fail=False) try: wait_until(lambda: len(self.pids(node)) == 0, timeout_sec=timeout_sec, @@ -79,21 +78,22 @@ def stop_node(self, node, clean_shutdown=True, timeout_sec=60): self.thread_dump(node) raise - def stop_nodes_async(self, nodes, delay_ms=0, clean_shutdown=True, timeout_sec=20, wait_for_stop=False): + def exec_on_nodes_async(self, nodes, task, simultaneously=True, delay_ms=0, timeout_sec=20): """ - Stops the nodes asynchronously. + Executes given task on the nodes. + :param task: a 'lambda: node'. + :param simultaneously: Enables or disables simultaneous start of the task on each node. + :param delay_ms: delay before task run. Begins with 0, grows by delay_ms for each next node in nodes. + :param timeout_sec: timeout to wait the task. """ - sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL - - sem = CountDownLatch(len(nodes)) + sem = CountDownLatch(len(nodes)) if simultaneously else None time_holder = AtomicValue() delay = 0 threads = [] for node in nodes: - thread = Thread(target=self.__stop_node, - args=(node, next(iter(self.pids(node))), sig, sem, delay, time_holder)) + thread = Thread(target=self.__exec_on_node, args=(node, task, sem, delay, time_holder)) threads.append(thread) @@ -104,19 +104,10 @@ def stop_nodes_async(self, nodes, delay_ms=0, clean_shutdown=True, timeout_sec=2 for thread in threads: thread.join(timeout_sec) - if wait_for_stop: - try: - wait_until(lambda: len(functools.reduce(operator.iconcat, (self.pids(n) for n in nodes), [])) == 0, - timeout_sec=timeout_sec, err_msg="Ignite node failed to stop in %d seconds" % timeout_sec) - except Exception: - for node in nodes: - self.thread_dump(node) - raise - return time_holder.get() @staticmethod - def __stop_node(node, pid, sig, start_waiter=None, delay_ms=0, time_holder=None): + def __exec_on_node(node, task, start_waiter=None, delay_ms=0, time_holder=None): if start_waiter: start_waiter.count_down() start_waiter.wait() @@ -130,7 +121,7 @@ def __stop_node(node, pid, sig, start_waiter=None, delay_ms=0, time_holder=None) time_holder.compare_and_set(None, (mono, timestamp)) - node.account.signal(pid, sig, False) + task(node) def clean_node(self, node): node.account.kill_java_processes(self.APP_SERVICE_CLASS, clean_shutdown=False, allow_fail=True) @@ -154,3 +145,28 @@ def pids(self, node): return pid_arr except (RemoteCommandError, ValueError): return [] + + +def node_failed_event_pattern(failed_node_id=None): + """Failed node pattern in log.""" + return "Node FAILED: .\\{1,\\}Node \\[id=" + (failed_node_id if failed_node_id else "") + \ + ".\\{1,\\}\\(isClient\\|client\\)=false" + + +def get_event_time(service, log_node, log_pattern, from_the_beginning=True, timeout=15): + """ + Extracts event time from ignite log by pattern . + :param service: ducktape service (ignite service) responsible to search log. + :param log_node: ducktape node to search ignite log on. + :param log_pattern: pattern to search ignite log for. + :param from_the_beginning: switches searching log from its beginning. + :param timeout: timeout to wait for the patters in the log. + """ + service.await_event_on_node(log_pattern, log_node, timeout, from_the_beginning=from_the_beginning, + backoff_sec=0.3) + + _, stdout, _ = log_node.account.ssh_client.exec_command( + "grep '%s' %s" % (log_pattern, IgniteAwareService.STDOUT_STDERR_CAPTURE)) + + return datetime.strptime(re.match("^\\[[^\\[]+\\]", stdout.read().decode("utf-8")).group(), + "[%Y-%m-%d %H:%M:%S,%f]") diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index fadbbe6d77d55..c964b6937e6f4 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -122,7 +122,8 @@ def await_event_on_node(self, evt_message, node, timeout_sec, from_the_beginning monitor.offset = 0 monitor.wait_until(evt_message, timeout_sec=timeout_sec, backoff_sec=backoff_sec, - err_msg="Event [%s] was not triggered in %d seconds" % (evt_message, timeout_sec)) + err_msg="Event [%s] was not triggered on '%s' in %d seconds" % (evt_message, node.name, + timeout_sec)) def await_event(self, evt_message, timeout_sec, from_the_beginning=False, backoff_sec=5): """ diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py index b9602e46339a1..5d86a22392b08 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py @@ -19,6 +19,7 @@ from typing import NamedTuple +from ignitetest.services.utils.ignite_configuration.communication import CommunicationSpi, TcpCommunicationSpi from ignitetest.services.utils.ignite_configuration.data_storage import DataStorageConfiguration from ignitetest.services.utils.ignite_configuration.discovery import DiscoverySpi, TcpDiscoverySpi from ignitetest.utils.version import IgniteVersion, DEV_BRANCH @@ -29,11 +30,13 @@ class IgniteConfiguration(NamedTuple): Ignite configuration. """ discovery_spi: DiscoverySpi = TcpDiscoverySpi() + communication_spi: CommunicationSpi = TcpCommunicationSpi() version: IgniteVersion = DEV_BRANCH cluster_state: str = 'ACTIVE' client_mode: bool = False consistent_id: str = None failure_detection_timeout: int = 10000 + sys_worker_blocked_timeout: int = 10000 properties: str = None data_storage: DataStorageConfiguration = None caches: list = [] diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py new file mode 100644 index 0000000000000..edca32cfad326 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +""" +Module contains classes and utility methods to create communication configuration for ignite nodes. +""" + +from abc import ABCMeta, abstractmethod + + +class CommunicationSpi(metaclass=ABCMeta): + """ + Abstract class for CommunicationSpi. + """ + @property + @abstractmethod + def type(self): + """ + Type of CommunicationSpi. + """ + + +class TcpCommunicationSpi(CommunicationSpi): + """ + TcpCommunicationSpi. + """ + def __init__(self, port=47100, port_range=100): + self.port = port + self.port_range = port_range + + @property + def type(self): + return "TCP" diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py index 9f38da5f9de99..2f533619b1aab 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py @@ -45,8 +45,9 @@ class ZookeeperDiscoverySpi(DiscoverySpi): """ ZookeeperDiscoverySpi. """ - def __init__(self, connection_string, root_path): - self.connection_string = connection_string + def __init__(self, zoo_service, root_path): + self.connection_string = zoo_service.connection_string() + self.port = zoo_service.settings.client_port self.root_path = root_path @property @@ -100,8 +101,10 @@ class TcpDiscoverySpi(DiscoverySpi): """ TcpDiscoverySpi. """ - def __init__(self, ip_finder=TcpDiscoveryVmIpFinder()): + def __init__(self, ip_finder=TcpDiscoveryVmIpFinder(), port=47500, port_range=100): self.ip_finder = ip_finder + self.port = port + self.port_range = port_range @property def type(self): @@ -138,4 +141,4 @@ def from_zookeeper_cluster(cluster, root_path="/apacheIgnite"): """ assert isinstance(cluster, ZookeeperService) - return ZookeeperDiscoverySpi(cluster.connection_string(), root_path=root_path) + return ZookeeperDiscoverySpi(cluster, root_path=root_path) diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/communication_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/communication_macro.j2 new file mode 100644 index 0000000000000..1f5c17cb15a85 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/communication_macro.j2 @@ -0,0 +1,25 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +#} + +{% macro communication_spi(spi) %} + + + + + + +{% endmacro %} diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 index fdd632a46b5fd..66bdf43603085 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 @@ -15,20 +15,22 @@ limitations under the License. #} -{% macro ip_finder(finder) %} - {% if finder %} +{% macro ip_finder(spi) %} + {% if spi.ip_finder and spi.ip_finder.type == 'VM' %} - {% if finder.type == 'VM' %} - - - - {% for address in finder.addresses %} - {{ address }} - {% endfor %} - - - - {% endif %} + + + + {% for address in spi.ip_finder.addresses %} + {% if spi.port_range > 0 %} + {{ address }}:{{ spi.port }}..{{ spi.port + spi.port_range }} + {% else %} + {{ address }}:{{ spi.port }} + {% endif %} + {% endfor %} + + + {% endif %} {% endmacro %} @@ -42,7 +44,9 @@ {% macro tcp_discovery_spi(spi) %} - {{ ip_finder(spi.ip_finder) }} + + + {{ ip_finder(spi) }} {% endmacro %} diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2 index 86abd3529587e..798897ad90b48 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/ignite.xml.j2 @@ -17,6 +17,7 @@ limitations under the License. #} +{% import 'communication_macro.j2' as communication %} {% import 'discovery_macro.j2' as disco_utils %} {% import 'cache_macro.j2' as cache_utils %} {% import 'datastorage_macro.j2' as datastorage_utils %} @@ -37,9 +38,12 @@ + {{ misc_utils.cluster_state(config.cluster_state, config.version) }} + {{ communication.communication_spi(config.communication_spi) }} + {{ disco_utils.discovery_spi(config.discovery_spi) }} {{ datastorage_utils.data_storage(config.data_storage) }} diff --git a/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2 b/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2 index 2c80e843e41b5..441b0dbbd0a08 100644 --- a/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2 +++ b/modules/ducktests/tests/ignitetest/services/zk/templates/zookeeper.properties.j2 @@ -16,6 +16,7 @@ #} tickTime={{ settings.tick_time }} +minSessionTimeout={{ settings.min_session_timeout }} initLimit={{ settings.init_limit }} syncLimit={{ settings.sync_limit }} dataDir={{ DATA_DIR }} diff --git a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py index b3da642aedd14..ca8a7fbbc8372 100644 --- a/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py +++ b/modules/ducktests/tests/ignitetest/services/zk/zookeeper.py @@ -27,11 +27,14 @@ class ZookeeperSettings: """ Settings for zookeeper quorum nodes. """ - def __init__(self, tick_time=1000, init_limit=10, sync_limit=5, client_port=2181): - self.tick_time = tick_time - self.init_limit = init_limit - self.sync_limit = sync_limit - self.client_port = client_port + def __init__(self, **kwargs): + self.tick_time = kwargs.get('tick_time', 1000) + self.min_session_timeout = kwargs.get('min_session_timeout', 2000) + self.init_limit = kwargs.get('init_limit', 10) + self.sync_limit = kwargs.get('sync_limit', 5) + self.client_port = kwargs.get('client_port', 2181) + + assert self.tick_time <= self.min_session_timeout // 2, "'tick_time' must be <= 'min_session_timeout' / 2" class ZookeeperService(Service): diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 2c8d6a17c32b9..f2be35bac9326 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -17,25 +17,24 @@ Module contains discovery tests. """ +import os import random -import re +import sys from enum import IntEnum -from datetime import datetime from time import monotonic from typing import NamedTuple from ducktape.mark import matrix from ducktape.mark.resource import cluster -from ignitetest.services.ignite import IgniteAwareService -from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite import IgniteAwareService, IgniteService, get_event_time, node_failed_event_pattern from ignitetest.services.ignite_app import IgniteApplicationService from ignitetest.services.utils.ignite_configuration import IgniteConfiguration from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration from ignitetest.services.utils.ignite_configuration.discovery import from_zookeeper_cluster, from_ignite_cluster, \ TcpDiscoverySpi from ignitetest.services.utils.time_utils import epoch_mills -from ignitetest.services.zk.zookeeper import ZookeeperService +from ignitetest.services.zk.zookeeper import ZookeeperService, ZookeeperSettings from ignitetest.utils import ignite_versions, version_if from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import DEV_BRANCH, LATEST_2_8, V_2_8_0, IgniteVersion @@ -56,8 +55,8 @@ class DiscoveryTestConfig(NamedTuple): """ version: IgniteVersion nodes_to_kill: int = 1 - kill_coordinator: bool = False load_type: ClusterLoad = ClusterLoad.NONE + sequential_failure: bool = False with_zk: bool = False @@ -69,46 +68,76 @@ class DiscoveryTest(IgniteTest): 2. Kill random node. 3. Wait that survived node detects node failure. """ - NUM_NODES = 7 + NUM_NODES = 9 - FAILURE_DETECTION_TIMEOUT = 2000 + FAILURE_DETECTION_TIMEOUT_TCP = 1000 + + FAILURE_DETECTION_TIMEOUT_ZK = 3000 DATA_AMOUNT = 5_000_000 WARMUP_DATA_AMOUNT = 10_000 + def __init__(self, test_context): + super().__init__(test_context=test_context) + + self.netfilter_store_path = None + @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(kill_coordinator=[False, True], - nodes_to_kill=[1, 2], + @matrix(nodes_to_kill=[1, 2], load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) - def test_node_fail_tcp(self, ignite_version, kill_coordinator, nodes_to_kill, load_type): + def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load_type): + """ + Test nodes failure scenario with TcpDiscoverySpi not allowing nodes to fail in a row. + """ + test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=nodes_to_kill, + load_type=load_type, sequential_failure=False) + + return self._perform_node_fail_scenario(test_config) + + @cluster(num_nodes=NUM_NODES) + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) + @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) + def test_2_nodes_fail_sequential_tcp(self, ignite_version, load_type): """ - Test nodes failure scenario with TcpDiscoverySpi. - :param load_type: How to load cluster during the test: 0 - no loading; 1 - do some loading; 2 - transactional. + Test 2 nodes sequential failure scenario with TcpDiscoverySpi. """ - test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), kill_coordinator=kill_coordinator, - nodes_to_kill=nodes_to_kill, load_type=load_type, with_zk=False) + test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=2, load_type=load_type, + sequential_failure=True) return self._perform_node_fail_scenario(test_config) @cluster(num_nodes=NUM_NODES + 3) @version_if(lambda version: version != V_2_8_0) # ignite-zookeeper package is broken in 2.8.0 @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(kill_coordinator=[False, True], - nodes_to_kill=[1, 2], + @matrix(nodes_to_kill=[1, 2], load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) - def test_node_fail_zk(self, ignite_version, kill_coordinator, nodes_to_kill, load_type): + def test_nodes_fail_not_sequential_zk(self, ignite_version, nodes_to_kill, load_type): + """ + Test node failure scenario with ZooKeeperSpi not allowing nodes to fail in a row. + """ + test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=nodes_to_kill, + load_type=load_type, sequential_failure=False, with_zk=True) + + return self._perform_node_fail_scenario(test_config) + + @cluster(num_nodes=NUM_NODES + 3) + @version_if(lambda version: version != V_2_8_0) # ignite-zookeeper package is broken in 2.8.0 + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) + @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) + def test_2_nodes_fail_sequential_zk(self, ignite_version, load_type): """ - Test node failure scenario with ZooKeeperSpi. - :param load_type: How to load cluster during the test: 0 - no loading; 1 - do some loading; 2 - transactional. + Test node failure scenario with ZooKeeperSpi not allowing to fail nodes in a row. """ - test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), kill_coordinator=kill_coordinator, - nodes_to_kill=nodes_to_kill, load_type=load_type, with_zk=True) + test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=2, load_type=load_type, + sequential_failure=True, with_zk=True) return self._perform_node_fail_scenario(test_config) def _perform_node_fail_scenario(self, test_config): + results = {} + modules = ['zookeeper'] if test_config.with_zk else None if test_config.with_zk: @@ -121,7 +150,8 @@ def _perform_node_fail_scenario(self, test_config): ignite_config = IgniteConfiguration( version=test_config.version, discovery_spi=discovery_spi, - failure_detection_timeout=self.FAILURE_DETECTION_TIMEOUT, + failure_detection_timeout=self.FAILURE_DETECTION_TIMEOUT_ZK if test_config.with_zk + else self.FAILURE_DETECTION_TIMEOUT_TCP, caches=[CacheConfiguration( name='test-cache', backups=1, @@ -131,14 +161,16 @@ def _perform_node_fail_scenario(self, test_config): servers, start_servers_sec = start_servers(self.test_context, self.NUM_NODES - 1, ignite_config, modules) - failed_nodes, survived_node = choose_node_to_kill(servers, test_config.kill_coordinator, - test_config.nodes_to_kill) + results['Ignite cluster start time (s)'] = start_servers_sec + + failed_nodes, survived_node = choose_node_to_kill(servers, test_config.nodes_to_kill, + test_config.sequential_failure) if test_config.load_type is not ClusterLoad.NONE: load_config = ignite_config._replace(client_mode=True) if test_config.with_zk else \ ignite_config._replace(client_mode=True, discovery_spi=from_ignite_cluster(servers)) - tran_nodes = [n.discovery_info().node_id for n in failed_nodes] \ + tran_nodes = [node_id(n) for n in failed_nodes] \ if test_config.load_type == ClusterLoad.TRANSACTIONAL else None params = {"cacheName": "test-cache", @@ -149,18 +181,128 @@ def _perform_node_fail_scenario(self, test_config): start_load_app(self.test_context, ignite_config=load_config, params=params, modules=modules) - data = simulate_nodes_failure(servers, failed_nodes, survived_node) + results.update(self._simulate_nodes_failure(servers, node_fail_task(ignite_config, test_config), failed_nodes, + survived_node)) + + return results + + def _simulate_nodes_failure(self, servers, kill_node_task, failed_nodes, survived_node): + """ + Perform node failure scenario + """ + for node in failed_nodes: + self.logger.info( + "Simulating failure of node '%s' (order %d) on '%s'" % (node_id(node), order(node), node.name)) + + ids_to_wait = [node_id(n) for n in failed_nodes] + + _, first_terminated = servers.exec_on_nodes_async(failed_nodes, kill_node_task) + + for node in failed_nodes: + self.logger.debug( + "Netfilter activated on '%s': %s" % (node.name, dump_netfilter_settings(node))) - data['Ignite cluster start time (s)'] = start_servers_sec + # Keeps dates of logged node failures. + logged_timestamps = [] + data = {} + + for failed_id in ids_to_wait: + logged_timestamps.append( + get_event_time(servers, survived_node, node_failed_event_pattern(failed_id))) + + self._check_failed_number(failed_nodes, survived_node) + self._check_not_segmented(failed_nodes) + + logged_timestamps.sort(reverse=True) + + first_kill_time = epoch_mills(first_terminated) + detection_delay = epoch_mills(logged_timestamps[0]) - first_kill_time + + data['Detection of node(s) failure (ms)'] = detection_delay + data['All detection delays (ms):'] = str([epoch_mills(ts) - first_kill_time for ts in logged_timestamps]) + data['Nodes failed'] = len(failed_nodes) return data + def _check_failed_number(self, failed_nodes, survived_node): + """Ensures number of failed nodes is correct.""" + cmd = "grep '%s' %s | wc -l" % (node_failed_event_pattern(), IgniteAwareService.STDOUT_STDERR_CAPTURE) + + failed_cnt = int(str(survived_node.account.ssh_client.exec_command(cmd)[1].read(), sys.getdefaultencoding())) + + if failed_cnt != len(failed_nodes): + failed = str(survived_node.account.ssh_client.exec_command( + "grep '%s' %s" % (node_failed_event_pattern(), IgniteAwareService.STDOUT_STDERR_CAPTURE))[1].read(), + sys.getdefaultencoding()) + + self.logger.warn("Node '%s' (%s) has detected the following failures:%s%s" % ( + survived_node.name, node_id(survived_node), os.linesep, failed)) + + raise AssertionError( + "Wrong number of failed nodes: %d. Expected: %d. Check the logs." % (failed_cnt, len(failed_nodes))) + + def _check_not_segmented(self, failed_nodes): + """Ensures only target nodes failed""" + for service in [srv for srv in self.test_context.services if isinstance(srv, IgniteAwareService)]: + for node in [srv_node for srv_node in service.nodes if srv_node not in failed_nodes]: + cmd = "grep -i '%s' %s | wc -l" % ("local node segmented", IgniteAwareService.STDOUT_STDERR_CAPTURE) + + failed = str(node.account.ssh_client.exec_command(cmd)[1].read(), sys.getdefaultencoding()) + + if int(failed) > 0: + raise AssertionError( + "Wrong node failed (segmented) on '%s'. Check the logs." % node.name) + + def setup(self): + super().setup() + + self.netfilter_store_path = os.path.join(self.tmp_path_root, "iptables.bak") + + # Store current network filter settings. + for node in self.test_context.cluster.nodes: + cmd = "sudo iptables-save | tee " + self.netfilter_store_path + + exec_error = str(node.account.ssh_client.exec_command(cmd)[2].read(), sys.getdefaultencoding()) + + if "Warning: iptables-legacy tables present" in exec_error: + cmd = "sudo iptables-legacy-save | tee " + self.netfilter_store_path + + exec_error = str(node.account.ssh_client.exec_command(cmd)[2].read(), sys.getdefaultencoding()) + + assert len(exec_error) == 0, "Failed to store iptables rules on '%s': %s" % (node.name, exec_error) + + self.logger.debug("Netfilter before launch on '%s': %s" % (node.name, dump_netfilter_settings(node))) + + def teardown(self): + # Restore previous network filter settings. + cmd = "sudo iptables-restore < " + self.netfilter_store_path + + errors = [] + + for node in self.test_context.cluster.nodes: + exec_error = str(node.account.ssh_client.exec_command(cmd)[2].read(), sys.getdefaultencoding()) + + if len(exec_error) > 0: + errors.append("Failed to restore iptables rules on '%s': %s" % (node.name, exec_error)) + else: + self.logger.debug("Netfilter after launch on '%s': %s" % (node.name, dump_netfilter_settings(node))) + + if len(errors) > 0: + self.logger.error("Failed restoring actions:" + os.linesep + os.linesep.join(errors)) + + raise RuntimeError("Unable to restore node states. See the log above.") + + super().teardown() + def start_zookeeper(test_context, num_nodes): """ Start zookeeper cluster. """ - zk_quorum = ZookeeperService(test_context, num_nodes) + zk_settings = ZookeeperSettings(min_session_timeout=DiscoveryTest.FAILURE_DETECTION_TIMEOUT_ZK, + tick_time=DiscoveryTest.FAILURE_DETECTION_TIMEOUT_ZK // 3) + + zk_quorum = ZookeeperService(test_context, num_nodes, settings=zk_settings) zk_quorum.start() return zk_quorum @@ -182,76 +324,73 @@ def start_load_app(test_context, ignite_config, params, modules=None): """ Start loader application. """ - loader = IgniteApplicationService( + IgniteApplicationService( test_context, config=ignite_config, java_class_name="org.apache.ignite.internal.ducktest.tests.ContinuousDataLoadApplication", modules=modules, # mute spam in log. jvm_opts=["-DIGNITE_DUMP_THREADS_ON_FAILURE=false"], - params=params) + params=params).start() - loader.start() +def choose_node_to_kill(servers, nodes_to_kill, sequential): + """Choose node to kill during test""" + assert nodes_to_kill > 0, "No nodes to kill passed. Check the parameters." -def failed_pattern(failed_node_id): - """ - Failed node pattern in log - """ - return "Node FAILED: .\\{1,\\}Node \\[id=" + failed_node_id + idx = random.randint(0, len(servers.nodes)-1) + to_kill = servers.nodes[idx:] + servers.nodes[:idx-1] -def choose_node_to_kill(servers, kill_coordinator, nodes_to_kill): - """Choose node to kill during test""" - assert nodes_to_kill > 0, " No nodes to kill passed. Check the parameters." + if not sequential: + to_kill = to_kill[0::2] - nodes = servers.nodes - coordinator = nodes[0].discovery_info().coordinator - to_kill = [] + idx = random.randint(0, len(to_kill) - nodes_to_kill) + to_kill = to_kill[idx:idx + nodes_to_kill] - if kill_coordinator: - to_kill.append(next(node for node in nodes if node.discovery_info().node_id == coordinator)) - nodes_to_kill -= 1 + survive = random.choice([node for node in servers.nodes if node not in to_kill]) - if nodes_to_kill > 0: - choice = random.sample([n for n in nodes if n.discovery_info().node_id != coordinator], nodes_to_kill) - to_kill.extend([choice] if not isinstance(choice, list) else choice) + assert len(to_kill) == nodes_to_kill, "Unable to pick up required number of nodes to kill." - survive = random.choice([node for node in servers.nodes if node not in to_kill]) + assert survive, "Unable to select survived node to monitor the cluster on it." return to_kill, survive -def simulate_nodes_failure(servers, failed_nodes, survived_node): - """ - Perform node failure scenario - """ - ids_to_wait = [node.discovery_info().node_id for node in failed_nodes] +def order(node): + """Return discovery order of the node.""" + return node.discovery_info().order + - _, first_terminated = servers.stop_nodes_async(failed_nodes, clean_shutdown=False, wait_for_stop=False) +def node_id(node): + """Return node id.""" + return node.discovery_info().node_id - # Keeps dates of logged node failures. - logged_timestamps = [] - data = {} - for failed_id in ids_to_wait: - servers.await_event_on_node(failed_pattern(failed_id), survived_node, 20, - from_the_beginning=True, backoff_sec=0.1) +def node_fail_task(ignite_config, test_config): + """ + Creates proper task to simulate network failure depending on the configurations. + """ + cm_spi = ignite_config.communication_spi + dsc_spi = ignite_config.discovery_spi - _, stdout, _ = survived_node.account.ssh_client.exec_command( - "grep '%s' %s" % (failed_pattern(failed_id), IgniteAwareService.STDOUT_STDERR_CAPTURE)) + cm_ports = str(cm_spi.port) if cm_spi.port_range < 1 else str(cm_spi.port) + ':' + str( + cm_spi.port + cm_spi.port_range) - logged_timestamps.append( - datetime.strptime(re.match("^\\[[^\\[]+\\]", stdout.read().decode("utf-8")).group(), - "[%Y-%m-%d %H:%M:%S,%f]")) + if test_config.with_zk: + dsc_ports = str(ignite_config.discovery_spi.port) + else: + dsc_ports = str(dsc_spi.port) if dsc_spi.port_range < 1 else str(dsc_spi.port) + ':' + str( + dsc_spi.port + dsc_spi.port_range) - logged_timestamps.sort(reverse=True) + cmd = f"sudo iptables -I %s 1 -p tcp -m multiport --dport {dsc_ports},{cm_ports} -j DROP" - first_kill_time = epoch_mills(first_terminated) - detection_delay = epoch_mills(logged_timestamps[0]) - first_kill_time + return lambda node: (node.account.ssh_client.exec_command(cmd % "INPUT"), + node.account.ssh_client.exec_command(cmd % "OUTPUT")) - data['Detection of node(s) failure (ms)'] = detection_delay - data['All detection delays (ms):'] = str([epoch_mills(ts) - first_kill_time for ts in logged_timestamps]) - data['Nodes failed'] = len(failed_nodes) - return data +def dump_netfilter_settings(node): + """ + Reads current netfilter settings on the node for debugging purposes. + """ + return str(node.account.ssh_client.exec_command("sudo iptables -L -n")[1].read(), sys.getdefaultencoding()) diff --git a/modules/ducktests/tests/ignitetest/utils/ignite_test.py b/modules/ducktests/tests/ignitetest/utils/ignite_test.py index feb59937e3757..3b0323d4db241 100644 --- a/modules/ducktests/tests/ignitetest/utils/ignite_test.py +++ b/modules/ducktests/tests/ignitetest/utils/ignite_test.py @@ -16,6 +16,9 @@ """ This module contains basic ignite test. """ +import os +import random +import string from time import monotonic from ducktape.tests.test import Test @@ -29,6 +32,29 @@ class IgniteTest(Test): def __init__(self, test_context): super().__init__(test_context=test_context) + self.tmp_path_root = None + + def setup(self): + super().setup() + + self.tmp_path_root = os.path.join("/tmp", ''.join(random.choices(string.ascii_letters + string.digits, k=10)), + self.test_context.cls_name) + + self.clear_tmp_dir(True) + + def teardown(self): + self.clear_tmp_dir() + + super().teardown() + + def clear_tmp_dir(self, recreate=False): + """Creates temporary directory for current test.""" + for node in self.test_context.cluster.nodes: + node.account.ssh_client.exec_command("rm -drf " + self.tmp_path_root) + + if recreate: + node.account.ssh_client.exec_command("mkdir -p " + self.tmp_path_root) + @staticmethod def monotonic(): """ From 797dbfd9839215c402b98e1ac866f81ba6d8bfab Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Thu, 22 Oct 2020 11:32:47 +0300 Subject: [PATCH 65/78] IGNITE-13599 Switch tests should have better precision (#8374) --- .../CellularTxStreamer.java | 63 +++++++++++++------ .../LongTxStreamerApplication.java | 2 +- .../SingleKeyTxStreamerApplication.java | 24 +++---- 3 files changed, 54 insertions(+), 35 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java index 0fc9b7202e66c..e0c5eb019fd7e 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/cellular_affinity_test/CellularTxStreamer.java @@ -17,9 +17,14 @@ package org.apache.ignite.internal.ducktest.tests.cellular_affinity_test; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Map; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.IgniteCache; @@ -44,11 +49,12 @@ public class CellularTxStreamer extends IgniteAwareApplication { IgniteCache cache = ignite.getOrCreateCache(cacheName); - long[] max = new long[20]; + int precision = 5; - Arrays.fill(max, -1); + long[] latencies = new long[precision]; + LocalDateTime[] opStartTimes = new LocalDateTime[precision]; - int key = 0; + Arrays.fill(latencies, -1); int cnt = 0; @@ -58,26 +64,35 @@ boolean record = false; Affinity aff = ignite.affinity(cacheName); - while (!terminated()) { - key++; + List cellKeys = new ArrayList<>(); + + int candidate = 0; - Collection nodes = aff.mapKeyToPrimaryAndBackups(key); + while (cellKeys.size() < 100) { + Collection nodes = aff.mapKeyToPrimaryAndBackups(++candidate); - Map stat = nodes.stream().collect( - Collectors.groupingBy(n -> n.attributes().get(attr), Collectors.counting())); + Set stat = nodes.stream() + .filter(n -> n.attributes().get(attr).equals(cell)) + .collect(Collectors.toSet()); - if (!stat.containsKey(cell)) + if (stat.isEmpty()) continue; + assert nodes.size() == stat.size(); + + cellKeys.add(candidate); + } + + while (!terminated()) { cnt++; - long start = System.currentTimeMillis(); + LocalDateTime start = LocalDateTime.now(); - cache.put(key, key); + long from = System.nanoTime(); - long finish = System.currentTimeMillis(); + cache.put(cellKeys.get(cnt % cellKeys.size()), cnt); // Cycled update. - long time = finish - start; + long latency = System.nanoTime() - from; if (!record && cnt > warmup) { record = true; @@ -88,11 +103,13 @@ record = true; } if (record) { - for (int i = 0; i < max.length; i++) { - if (max[i] <= time) { - System.arraycopy(max, i, max, i + 1, max.length - i - 1); + for (int i = 0; i < latencies.length; i++) { + if (latencies[i] <= latency) { + System.arraycopy(latencies, i, latencies, i + 1, latencies.length - i - 1); + System.arraycopy(opStartTimes, i, opStartTimes, i + 1, opStartTimes.length - i - 1); - max[i] = time; + latencies[i] = latency; + opStartTimes[i] = start; break; } @@ -100,10 +117,16 @@ record = true; } if (cnt % 1000 == 0) - log.info("APPLICATION_STREAMED " + cnt + " transactions [worst_latency=" + Arrays.toString(max) + "]"); + log.info("APPLICATION_STREAMED " + cnt + " transactions [worst_latency=" + Arrays.toString(latencies) + "]"); } - recordResult("WORST_LATENCY", Arrays.toString(max)); + List result = new ArrayList<>(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS"); + + for (int i = 0; i < precision; i++) + result.add(Duration.ofNanos(latencies[i]).toMillis() + " ms at " + formatter.format(opStartTimes[i])); + + recordResult("WORST_LATENCY", result.toString()); recordResult("STREAMED", cnt - warmup); recordResult("MEASURE_DURATION", System.currentTimeMillis() - initTime); diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java index f83d1252e5070..a9f10d7961264 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/LongTxStreamerApplication.java @@ -45,7 +45,7 @@ public class LongTxStreamerApplication extends IgniteAwareApplication { log.info("Starting Long Tx..."); - for (int i = 0; i < TX_CNT; i++) { + for (int i = -1; i >= -TX_CNT; i--) { // Negative keys to have no intersection with load. int finalI = i; new Thread(() -> { diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java index 568859b6abdeb..bb91df8d2f670 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/pme_free_switch_test/SingleKeyTxStreamerApplication.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.ducktest.tests.pme_free_switch_test; +import java.time.Duration; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; @@ -31,26 +32,21 @@ public class SingleKeyTxStreamerApplication extends IgniteAwareApplication { int warmup = jsonNode.get("warmup").asInt(); - long max = -1; - - int key = 10_000_000; - + int key = 0; int cnt = 0; - long initTime = 0; + long maxLatency = -1; boolean record = false; while (!terminated()) { cnt++; - long start = System.currentTimeMillis(); - - cache.put(key++, key); + long from = System.nanoTime(); - long finish = System.currentTimeMillis(); + cache.put(key++ % 100, key); // Cycled update. - long time = finish - start; + long latency = System.nanoTime() - from; if (!record && cnt > warmup) { record = true; @@ -61,15 +57,15 @@ record = true; } if (record) { - if (max < time) - max = time; + if (maxLatency < latency) + maxLatency = latency; } if (cnt % 1000 == 0) - log.info("APPLICATION_STREAMED " + cnt + " transactions [max=" + max + "]"); + log.info("APPLICATION_STREAMED " + cnt + " transactions [max=" + maxLatency + "]"); } - recordResult("WORST_LATENCY", max); + recordResult("WORST_LATENCY", Duration.ofNanos(maxLatency).toMillis()); recordResult("STREAMED", cnt - warmup); recordResult("MEASURE_DURATION", System.currentTimeMillis() - initTime); From bd2624291a5b1c06e48b37671e0e7f45db605f2a Mon Sep 17 00:00:00 2001 From: SwirMix Date: Fri, 23 Oct 2020 14:19:34 +0300 Subject: [PATCH 66/78] IGNITE-13489 Clients log in and out of the topology (#8362) --- .../IgniteCachePutClient.java | 63 +++++++ .../tests/ignitetest/services/ignite_app.py | 51 ++++-- .../ignitetest/tests/client_in_out_test.py | 168 ++++++++++++++++++ .../ignitetest/tests/suites/fast_suite.yml | 4 + 4 files changed, 268 insertions(+), 18 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/start_stop_client/IgniteCachePutClient.java create mode 100644 modules/ducktests/tests/ignitetest/tests/client_in_out_test.py diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/start_stop_client/IgniteCachePutClient.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/start_stop_client/IgniteCachePutClient.java new file mode 100644 index 0000000000000..b0ecd63d8c881 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/start_stop_client/IgniteCachePutClient.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests.start_stop_client; + +import java.util.Optional; +import java.util.UUID; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; + +/** + * Java client. Tx put operation + */ +public class IgniteCachePutClient extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override protected void run(JsonNode jsonNode) throws Exception { + String cacheName = jsonNode.get("cacheName").asText(); + + long pacing = Optional.ofNullable(jsonNode.get("pacing")) + .map(JsonNode::asLong) + .orElse(0l); + + log.info("Test props:" + + " cacheName=" + cacheName + + " pacing=" + pacing); + + IgniteCache cache = ignite.getOrCreateCache(cacheName); + log.info("Node name: " + ignite.name() + " starting cache operations."); + + markInitialized(); + + while (!terminated()) { + UUID uuid = UUID.randomUUID(); + + long startTime = System.nanoTime(); + + cache.put(uuid, uuid); + + long resultTime = System.nanoTime() - startTime; + + log.info("Success put, latency: " + resultTime + "ns."); + + Thread.sleep(pacing); + } + + markFinished(); + } +} diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py index 8fbf0351a66a5..fa5cc5d6820bc 100644 --- a/modules/ducktests/tests/ignitetest/services/ignite_app.py +++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py @@ -34,9 +34,9 @@ class IgniteApplicationService(IgniteAwareService): SERVICE_JAVA_CLASS_NAME = "org.apache.ignite.internal.ducktest.utils.IgniteAwareApplicationService" # pylint: disable=R0913 - def __init__(self, context, config, java_class_name, params="", timeout_sec=60, modules=None, + def __init__(self, context, config, java_class_name, num_nodes=1, params="", timeout_sec=60, modules=None, servicejava_class_name=SERVICE_JAVA_CLASS_NAME, jvm_opts=None, start_ignite=True): - super().__init__(context, config, 1, modules=modules, servicejava_class_name=servicejava_class_name, + super().__init__(context, config, num_nodes, modules=modules, servicejava_class_name=servicejava_class_name, java_class_name=java_class_name, params=params, jvm_opts=jvm_opts, start_ignite=start_ignite) self.servicejava_class_name = servicejava_class_name @@ -54,28 +54,42 @@ def start(self): self.__check_status("IGNITE_APPLICATION_INITIALIZED", timeout=self.timeout_sec) def stop_async(self, clean_shutdown=True): + """ + Stop in async way. + """ + for node in self.nodes: + self.stop_node(node=node, clean_shutdown=clean_shutdown) + + # pylint: disable=W0221 + def stop_node(self, node, clean_shutdown=True): """ Stops node in async way. """ - self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(self.nodes[0].account))) - self.nodes[0].account.kill_java_processes(self.servicejava_class_name, clean_shutdown=clean_shutdown, - allow_fail=True) + self.logger.info("%s Stopping node %s" % (self.__class__.__name__, str(node.account))) + node.account.kill_java_processes(self.servicejava_class_name, clean_shutdown=clean_shutdown, + allow_fail=True) def await_stopped(self, timeout_sec=10): """ Awaits node stop finish. """ - stopped = self.wait_node(self.nodes[0], timeout_sec=timeout_sec) - assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ - (str(self.nodes[0].account), str(timeout_sec)) + for node in self.nodes: + stopped = self.wait_node(node, timeout_sec=timeout_sec) + assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \ + (str(node.account), str(timeout_sec)) self.__check_status("IGNITE_APPLICATION_FINISHED", timeout=timeout_sec) # pylint: disable=W0221 - def stop_node(self, node, clean_shutdown=True, timeout_sec=10): - assert node == self.nodes[0] - self.stop_async(clean_shutdown) - self.await_stopped(timeout_sec) + def stop(self, clean_shutdown=True, timeout_sec=10): + """ + Stop services. + """ + if clean_shutdown: + self.stop_async(clean_shutdown) + self.await_stopped(timeout_sec) + else: + self.stop_async(clean_shutdown) def __check_status(self, desired, timeout=1): self.await_event("%s\\|IGNITE_APPLICATION_BROKEN" % desired, timeout, from_the_beginning=True) @@ -110,7 +124,8 @@ def extract_result(self, name): """ results = self.extract_results(name) - assert len(results) <= 1, f"Expected exactly one result occurence, {len(results)} found." + assert len(results) == len(self.nodes), f"Expected exactly {len(self.nodes)} occurence," \ + f" but found {len(results)}." return results[0] if results else "" @@ -121,10 +136,10 @@ def extract_results(self, name): """ res = [] - output = self.nodes[0].account.ssh_capture( - "grep '%s' %s" % (name + "->", self.STDOUT_STDERR_CAPTURE), allow_fail=False) - - for line in output: - res.append(re.search("%s(.*)%s" % (name + "->", "<-"), line).group(1)) + for node in self.nodes: + output = node.account.ssh_capture( + "grep '%s' %s" % (name + "->", self.STDOUT_STDERR_CAPTURE), allow_fail=False) + for line in output: + res.append(re.search("%s(.*)%s" % (name + "->", "<-"), line).group(1)) return res diff --git a/modules/ducktests/tests/ignitetest/tests/client_in_out_test.py b/modules/ducktests/tests/ignitetest/tests/client_in_out_test.py new file mode 100644 index 0000000000000..aab699a9ef579 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/client_in_out_test.py @@ -0,0 +1,168 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains client tests +""" +import time + +from ducktape.mark.resource import cluster + +from ducktape.mark import parametrize +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration +from ignitetest.utils import ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, V_2_8_1, IgniteVersion + + +# pylint: disable=W0223 +class ClientTest(IgniteTest): + """ + cluster - cluster size + CACHE_NAME - name of the cache to create for the test. + PACING - the frequency of the operation on clients (ms). + JAVA_CLIENT_CLASS_NAME - running classname. + client_work_time - clients working time (s). + iteration_count - the number of iterations of starting and stopping client nodes (s). + static_clients - the number of permanently employed clients. + temp_client - number of clients who come log in and out. + """ + + CACHE_NAME = "simple-tx-cache" + PACING = 10 + JAVA_CLIENT_CLASS_NAME = "org.apache.ignite.internal.ducktest.tests.start_stop_client.IgniteCachePutClient" + + @ignite_versions(str(V_2_8_1), str(DEV_BRANCH)) + @cluster(num_nodes=7) + @parametrize(num_nodes=7, + static_clients=2, + temp_client=3, + iteration_count=3, + client_work_time=30) + # pylint: disable=R0913 + def test_ignite_start_stop_nodes(self, ignite_version, + num_nodes, static_clients, temp_client, iteration_count, client_work_time): + """ + Start and stop clients node test without kill java process. + Check topology. + """ + self.ignite_start_stop(ignite_version, True, num_nodes, static_clients, + temp_client, iteration_count, client_work_time) + + @ignite_versions(str(V_2_8_1), str(DEV_BRANCH)) + @cluster(num_nodes=7) + @parametrize(num_nodes=7, + static_clients=2, + temp_client=3, + iteration_count=3, + client_work_time=30) + # pylint: disable=R0913 + def test_ignite_kill_start_nodes(self, ignite_version, + num_nodes, static_clients, temp_client, iteration_count, client_work_time): + """ + Start and kill client nodes, Check topology + """ + self.ignite_start_stop(ignite_version, False, num_nodes, static_clients, + temp_client, iteration_count, client_work_time) + + # pylint: disable=R0914 + # pylint: disable=R0913 + def ignite_start_stop(self, ignite_version, correct_stop_temp_node, + nodes_num, static_clients_num, temp_client, iteration_count, client_work_time): + """ + Test for starting and stopping fat clients. + """ + + servers_count = nodes_num - static_clients_num - temp_client + + current_top_v = servers_count + # Topology version after test. + fin_top_ver = servers_count + (2 * static_clients_num) + (2 * iteration_count * temp_client) + + server_cfg = IgniteConfiguration( + version=IgniteVersion(ignite_version), + caches=[CacheConfiguration(name=self.CACHE_NAME, backups=1, atomicity_mode='TRANSACTIONAL')] + ) + + ignite = IgniteService(self.test_context, server_cfg, num_nodes=servers_count) + control_utility = ControlUtility(ignite, self.test_context) + + client_cfg = server_cfg._replace(client_mode=True) + + static_clients = IgniteApplicationService( + self.test_context, + client_cfg, + java_class_name=self.JAVA_CLIENT_CLASS_NAME, + num_nodes=static_clients_num, + params={"cacheName": self.CACHE_NAME, + "pacing": self.PACING}) + + temp_clients = IgniteApplicationService( + self.test_context, + client_cfg, + java_class_name=self.JAVA_CLIENT_CLASS_NAME, + num_nodes=temp_client, + params={"cacheName": self.CACHE_NAME, + "pacing": self.PACING}) + + ignite.start() + + static_clients.start() + + current_top_v += static_clients_num + check_topology(control_utility, current_top_v) + + # Start / stop temp_clients node. Check cluster. + for i in range(iteration_count): + self.logger.debug(f'Starting iteration: {i}.') + + temp_clients.start() + current_top_v += temp_client + + await_event(static_clients, f'ver={current_top_v}, locNode=') + check_topology(control_utility, current_top_v) + + await_event(temp_clients, f'clients={static_clients_num + temp_client}') + + time.sleep(client_work_time) + temp_clients.stop(correct_stop_temp_node) + + current_top_v += temp_client + + await_event(static_clients, f'ver={current_top_v}, locNode=') + static_clients.stop() + + check_topology(control_utility, fin_top_ver) + + +def await_event(service: IgniteApplicationService, message): + """ + :param service: target service for wait + :param message: message + """ + service.await_event(message, timeout_sec=80, from_the_beginning=True) + + +def check_topology(control_utility: ControlUtility, fin_top_ver: int): + """ + Check current topology version. + """ + top_ver = control_utility.cluster_state().topology_version + assert top_ver == fin_top_ver, f'Cluster current topology version={top_ver}, ' \ + f'expected topology version={fin_top_ver}.' diff --git a/modules/ducktests/tests/ignitetest/tests/suites/fast_suite.yml b/modules/ducktests/tests/ignitetest/tests/suites/fast_suite.yml index 698c1d84c74e0..714a9af43e840 100644 --- a/modules/ducktests/tests/ignitetest/tests/suites/fast_suite.yml +++ b/modules/ducktests/tests/ignitetest/tests/suites/fast_suite.yml @@ -27,3 +27,7 @@ cellular_affinity: rebalance: - ../add_node_rebalance_test.py + +clients: + - ../client_in_out_test.py + From d81902369522f59041670967118a0718d31fda9f Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Wed, 28 Oct 2020 16:24:17 +0300 Subject: [PATCH 67/78] Logging improvement --- .../internal/ducktest/utils/IgniteAwareApplicationService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java index c9127655b72fa..7e8518035470a 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java @@ -72,6 +72,9 @@ public static void main(String[] args) throws Exception { app.start(jsonNode); } + finally { + log.info("Ignite instance closed. [interrupted=" + Thread.currentThread().isInterrupted() + "]"); + } } else app.start(jsonNode); From d09d6513d263072c695484a1676b7b3fb8d8d666 Mon Sep 17 00:00:00 2001 From: Vladsz83 Date: Thu, 29 Oct 2020 11:42:17 +0300 Subject: [PATCH 68/78] IGNITE-13638 : Bring log config to ducktape tests (#8405) --- .../ducktests/tests/ignitetest/services/utils/ignite_spec.py | 1 + .../tests/ignitetest/services/utils/templates/log4j.xml.j2 | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index afb975a7cfce1..5977d233b7e1b 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -156,6 +156,7 @@ def __init__(self, modules, **kwargs): self.jvm_opts.extend([ "-DIGNITE_SUCCESS_FILE=" + self.PERSISTENT_ROOT + "/success_file", + "-Dlog4j.configuration=file:" + self.LOG4J_CONFIG_FILE, "-Dlog4j.configDebug=true" ]) diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 index e8b3be8bbfef1..b2a9a7b48d9a1 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 @@ -23,8 +23,6 @@ - - From 7bb28862cb273bd83e332d2f131d9e8123dd46f7 Mon Sep 17 00:00:00 2001 From: Vladsz83 Date: Thu, 29 Oct 2020 13:32:58 +0300 Subject: [PATCH 69/78] IGNITE-13620 : Bind ignite node to 1 address in the ducktests (#8399) --- .../ducktests/tests/ignitetest/services/utils/ignite_aware.py | 4 ++++ .../services/utils/ignite_configuration/discovery.py | 3 ++- .../ignitetest/services/utils/templates/discovery_macro.j2 | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index c964b6937e6f4..bebafc7da0fa0 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -17,6 +17,8 @@ This module contains the base class to build services aware of Ignite. """ +import socket + from abc import abstractmethod, ABCMeta from ducktape.services.background_thread import BackgroundThreadService @@ -75,6 +77,8 @@ def _prepare_config(self, node): config.discovery_spi.prepare_on_start(cluster=self) + config.discovery_spi.local_address = socket.gethostbyname(node.account.hostname) + node_config = self.spec.config_template.render(config_dir=self.PERSISTENT_ROOT, work_dir=self.WORK_DIR, config=config) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py index 2f533619b1aab..1cd310f16921a 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py @@ -101,10 +101,11 @@ class TcpDiscoverySpi(DiscoverySpi): """ TcpDiscoverySpi. """ - def __init__(self, ip_finder=TcpDiscoveryVmIpFinder(), port=47500, port_range=100): + def __init__(self, ip_finder=TcpDiscoveryVmIpFinder(), port=47500, port_range=100, local_address=None): self.ip_finder = ip_finder self.port = port self.port_range = port_range + self.local_address = local_address @property def type(self): diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 index 66bdf43603085..dc666dfda99da 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 @@ -44,6 +44,9 @@ {% macro tcp_discovery_spi(spi) %} + {% if spi.local_address %} + + {% endif %} {{ ip_finder(spi) }} From 13fa4bf13754ef9c65386441567867576c6cd22e Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Thu, 29 Oct 2020 16:29:29 +0300 Subject: [PATCH 70/78] test --- .../ignite/internal/util/IgniteUtils.java | 3 + .../ignite/spi/discovery/tcp/ServerImpl.java | 114 ++++++++++++++---- .../spi/discovery/tcp/TcpDiscoverySpi.java | 10 ++ .../services/utils/templates/log4j.xml.j2 | 4 + .../tests/ignitetest/tests/discovery_test.py | 6 +- 5 files changed, 109 insertions(+), 28 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index 6bdc6fab34138..aa7c409c42745 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -7155,6 +7155,9 @@ public static void wrapThreadLoader(ClassLoader ldr, Runnable c) { * @return Short string representing the node. */ public static String toShortString(ClusterNode n) { + if (n == null) + return "ClusterNode [null]"; + return "ClusterNode [id=" + n.id() + ", order=" + n.order() + ", addr=" + n.addresses() + ", daemon=" + n.isDaemon() + ']'; } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index d0c8e8e49d989..0f009af0bc855 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -3508,12 +3508,20 @@ else if (log.isTraceEnabled()) if (changeTop) hndMsg.changeTopology(ring.previousNodeOf(next).id()); - if (log.isDebugEnabled()) - log.debug("Sending handshake [hndMsg=" + hndMsg + ", sndState=" + sndState + ']'); + if (log.isDebugEnabled()) { + log.debug("Sending handshake [" + hndMsg + "] with timeout " + + timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()) + ", sndState=[" + + sndState + ']'); + } spi.writeToSocket(sock, out, hndMsg, timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout())); + if (log.isTraceEnabled()) { + log.trace("Reading handshake response for [" + hndMsg + "] with timeout " + + timeoutHelper.nextTimeoutChunk(ackTimeout0) + ']'); + } + TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(ackTimeout0)); @@ -3604,7 +3612,7 @@ else if (log.isTraceEnabled()) } } - updateLastSentMessageTime(); + updateLastSentMessageTime(msg); if (log.isDebugEnabled()) log.debug("Initialized connection with next node: " + next.id()); @@ -3706,6 +3714,11 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof if (timeoutHelper == null) timeoutHelper = serverOperationTimeoutHelper(sndState, lastRingMsgSentTime); + if (log.isTraceEnabled()) { + log.trace("Sending pending message [" + pendingMsg + "] with timeout " + + timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()) + '.'); + } + try { spi.writeToSocket(sock, out, pendingMsg, timeoutHelper.nextTimeoutChunk( spi.getSocketTimeout())); @@ -3716,9 +3729,14 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof long tsNanos0 = System.nanoTime(); + if (log.isTraceEnabled()) { + log.trace("Reading receipt for pending message [" + pendingMsg + + "] with timeout " + timeoutHelper.nextTimeoutChunk(ackTimeout0) + '.'); + } + int res = spi.readReceipt(sock, timeoutHelper.nextTimeoutChunk(ackTimeout0)); - updateLastSentMessageTime(); + updateLastSentMessageTime(pendingMsg); spi.stats.onMessageSent(pendingMsg, U.nanosToMillis(tsNanos0 - tsNanos)); @@ -3757,6 +3775,11 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof if (latencyCheck && log.isInfoEnabled()) log.info("Latency check message has been written to socket: " + msg.id()); + if (log.isTraceEnabled()) { + log.trace("Sending [" + msg + "] to " + U.toShortString(next) + + " with timeout " + timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()) + ']'); + } + spi.writeToSocket(newNextNode ? newNext : next, sock, out, @@ -3765,9 +3788,14 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof long tsNanos0 = System.nanoTime(); + if (log.isTraceEnabled()) { + log.trace("Reading receipt for [" + msg + "] from " + U.toShortString(next) + + " with timeout " + timeoutHelper.nextTimeoutChunk(ackTimeout0) + ']'); + } + int res = spi.readReceipt(sock, timeoutHelper.nextTimeoutChunk(ackTimeout0)); - updateLastSentMessageTime(); + updateLastSentMessageTime(msg); if (latencyCheck && log.isInfoEnabled()) log.info("Latency check message has been acked: " + msg.id()); @@ -3808,25 +3836,44 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof errs.add(e); - if (log.isDebugEnabled()) - U.error(log, "Failed to send message to next node [next=" + next.id() + ", msg=" + msg + - ", err=" + e + ']', e); + if (log.isDebugEnabled()) { + U.error(log, "Failed to send message to next node [next=" + next.id() + + ", msg=" + msg + ", err=" + e + ']', e); + } onException("Failed to send message to next node [next=" + next.id() + ", msg=" + msg + ']', e); - if (spi.failureDetectionTimeoutEnabled() && timeoutHelper.checkFailureTimeoutReached(e)) + if (spi.failureDetectionTimeoutEnabled() && timeoutHelper.checkFailureTimeoutReached(e)) { + if (log.isDebugEnabled()) { + log.debug("Failure detection timeout has been reached: " + + spi.failureDetectionTimeout()); + } + break; + } if (!spi.failureDetectionTimeoutEnabled()) { - if (++reconCnt == spi.getReconnectCount()) + if (++reconCnt == spi.getReconnectCount()) { + if (log.isDebugEnabled()) { + log.debug("Maximum reconnection number has been reached: " + + spi.getReconnectCount()); + } + break; + } else if (e instanceof SocketTimeoutException || X.hasCause(e, SocketTimeoutException.class)) { ackTimeout0 *= 2; - if (!checkAckTimeout(ackTimeout0)) + if (!checkAckTimeout(ackTimeout0)) { + if (log.isDebugEnabled()) { + log.debug("Maximum ack tmeout has been reached: " + + spi.getMaxAckTimeout()); + } + break; + } } } } @@ -3850,8 +3897,22 @@ else if (e instanceof SocketTimeoutException || } // Iterating node's addresses. if (!sent) { - if (sndState == null && spi.getEffectiveConnectionRecoveryTimeout() > 0) + Exception err = errs != null ? + U.exceptionWithSuppressed("Failed to send message to next node [msg=" + msg + + ", next=" + U.toShortString(next) + ']', errs) : + null; + + // If node existed on connection initialization we should check + // whether it has not gone yet. + U.warn(log, "Failed to send message to next node [msg=" + msg + ", next=" + next + + ", errMsg=" + (err != null ? err.getMessage() : "N/A") + ']'); + + if (sndState == null && spi.getEffectiveConnectionRecoveryTimeout() > 0) { sndState = new CrossRingMessageSendState(); + + if (log.isDebugEnabled()) + log.debug("Initialized send state for first fail to send the message: " + state); + } else if (sndState != null && sndState.checkTimeout()) { segmentLocalNodeOnSendFail(failedNodes); @@ -3861,19 +3922,10 @@ else if (sndState != null && sndState.checkTimeout()) { boolean failedNextNode = sndState == null || sndState.markNextNodeFailed(); if (failedNextNode && !failedNodes.contains(next)) { - failedNodes.add(next); - - if (state == CONNECTED) { - Exception err = errs != null ? - U.exceptionWithSuppressed("Failed to send message to next node [msg=" + msg + - ", next=" + U.toShortString(next) + ']', errs) : - null; + if (log.isDebugEnabled()) + log.debug("Marking next node [" + U.toShortString(next) + "] as failed."); - // If node existed on connection initialization we should check - // whether it has not gone yet. - U.warn(log, "Failed to send message to next node [msg=" + msg + ", next=" + next + - ", errMsg=" + (err != null ? err.getMessage() : "N/A") + ']'); - } + failedNodes.add(next); } else if (!failedNextNode && sndState != null && sndState.isBackward()) { boolean prev = sndState.markLastFailedNodeAlive(); @@ -3887,6 +3939,9 @@ else if (!failedNextNode && sndState != null && sndState.isBackward()) { newNextNode = false; next = ring.nextNode(failedNodes); + + if (log.isDebugEnabled()) + log.debug("Trying new next node: " + U.toShortString(next)); } } @@ -4041,6 +4096,9 @@ private void registerPendingMessage(TcpDiscoveryAbstractMessage msg) { assert msg != null; if (spi.ensured(msg)) { + if (log.isTraceEnabled()) + log.trace("Registering pending message: " + msg); + pendingMsgs.add(msg); spi.stats.onPendingMessageRegistered(); @@ -6545,8 +6603,11 @@ private IgniteSpiOperationTimeoutHelper serverOperationTimeoutHelper(@Nullable C } /** Fixates time of last sent message. */ - private void updateLastSentMessageTime() { + private void updateLastSentMessageTime(TcpDiscoveryAbstractMessage msg) { lastRingMsgSentTime = System.nanoTime(); + + if (log.isTraceEnabled()) + log.trace("Updated message sent time for message [" + msg + "]."); } /** Thread that executes {@link TcpServer}'s code. */ @@ -8249,6 +8310,9 @@ boolean checkTimeout() { if (System.nanoTime() >= failTimeNanos) { state = RingMessageSendState.FAILED; + if (log.isDebugEnabled()) + log.debug("Reconnect/resend timeout has been reached. Send state: " + this); + return true; } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java index f1e8406a9afe9..4919089f9f116 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java @@ -1589,8 +1589,18 @@ protected Socket openSocket(Socket sock, InetSocketAddress remAddr, IgniteSpiOpe assert addr != null; + if (log.isTraceEnabled()) { + log.trace("Connecting to " + resolved + " with timeout " + + timeoutHelper.nextTimeoutChunk(sockTimeout) + '.'); + } + sock.connect(resolved, (int)timeoutHelper.nextTimeoutChunk(sockTimeout)); + if (log.isTraceEnabled()) { + log.trace("Sending the header to " + resolved + " with timeout " + + timeoutHelper.nextTimeoutChunk(sockTimeout) + '.'); + } + writeToSocket(sock, null, U.IGNITE_HEADER, timeoutHelper.nextTimeoutChunk(sockTimeout)); return sock; diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 index b2a9a7b48d9a1..f2c2d9f742caa 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 @@ -48,6 +48,10 @@ + + + + diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index f2be35bac9326..deedaa0d7b44d 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -68,7 +68,7 @@ class DiscoveryTest(IgniteTest): 2. Kill random node. 3. Wait that survived node detects node failure. """ - NUM_NODES = 9 + NUM_NODES = 5 FAILURE_DETECTION_TIMEOUT_TCP = 1000 @@ -85,8 +85,8 @@ def __init__(self, test_context): @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(nodes_to_kill=[1, 2], - load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) + @matrix(nodes_to_kill=[1], + load_type=[ClusterLoad.NONE]) def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load_type): """ Test nodes failure scenario with TcpDiscoverySpi not allowing nodes to fail in a row. From f917c71c52bba19784901c693370e8aceb39796b Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 30 Oct 2020 18:58:46 +0300 Subject: [PATCH 71/78] first impl. --- .../utils/ignite_configuration/discovery.py | 1 + .../utils/templates/discovery_macro.j2 | 1 + .../tests/ignitetest/tests/discovery_test.py | 54 +++++++++++-------- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py index 1cd310f16921a..b22baa1dd2d62 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py @@ -106,6 +106,7 @@ def __init__(self, ip_finder=TcpDiscoveryVmIpFinder(), port=47500, port_range=10 self.port = port self.port_range = port_range self.local_address = local_address + self.socket_linger: int = 0 @property def type(self): diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 index dc666dfda99da..3be087bcef802 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 @@ -49,6 +49,7 @@ {% endif %} + {{ ip_finder(spi) }} {% endmacro %} diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index deedaa0d7b44d..86fd5696bd5a9 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -49,6 +49,7 @@ class ClusterLoad(IntEnum): TRANSACTIONAL = 2 +# pylint: disable=R0913 class DiscoveryTestConfig(NamedTuple): """ Configuration for DiscoveryTest. @@ -58,6 +59,8 @@ class DiscoveryTestConfig(NamedTuple): load_type: ClusterLoad = ClusterLoad.NONE sequential_failure: bool = False with_zk: bool = False + socket_linger: int = 0 + failure_detection_timeout: int = 1000 # pylint: disable=W0223 @@ -70,10 +73,6 @@ class DiscoveryTest(IgniteTest): """ NUM_NODES = 5 - FAILURE_DETECTION_TIMEOUT_TCP = 1000 - - FAILURE_DETECTION_TIMEOUT_ZK = 3000 - DATA_AMOUNT = 5_000_000 WARMUP_DATA_AMOUNT = 10_000 @@ -85,53 +84,61 @@ def __init__(self, test_context): @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(nodes_to_kill=[1], - load_type=[ClusterLoad.NONE]) - def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load_type): + @matrix(nodes_to_kill=[1, 2], + load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 1, 3], + failure_detection_timeout=[500, 1000, 5000]) + def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load_type, socket_linger, + failure_detection_timeout): """ Test nodes failure scenario with TcpDiscoverySpi not allowing nodes to fail in a row. """ test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=nodes_to_kill, - load_type=load_type, sequential_failure=False) + load_type=load_type, sequential_failure=False, socket_linger=socket_linger, + failure_detection_timeout=failure_detection_timeout) return self._perform_node_fail_scenario(test_config) @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) - def test_2_nodes_fail_sequential_tcp(self, ignite_version, load_type): + @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 1, 3], + failure_detection_timeout=[500, 1000, 5000]) + def test_2_nodes_fail_sequential_tcp(self, ignite_version, load_type, socket_linger, failure_detection_timeout): """ Test 2 nodes sequential failure scenario with TcpDiscoverySpi. """ test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=2, load_type=load_type, - sequential_failure=True) + sequential_failure=True, socket_linger=socket_linger, + failure_detection_timeout=failure_detection_timeout) return self._perform_node_fail_scenario(test_config) @cluster(num_nodes=NUM_NODES + 3) @version_if(lambda version: version != V_2_8_0) # ignite-zookeeper package is broken in 2.8.0 @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(nodes_to_kill=[1, 2], + @matrix(nodes_to_kill=[1, 2], failure_detection_timeout=[1000, 5000], load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) - def test_nodes_fail_not_sequential_zk(self, ignite_version, nodes_to_kill, load_type): + def test_nodes_fail_not_sequential_zk(self, ignite_version, nodes_to_kill, load_type, failure_detection_timeout): """ Test node failure scenario with ZooKeeperSpi not allowing nodes to fail in a row. """ test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=nodes_to_kill, - load_type=load_type, sequential_failure=False, with_zk=True) + load_type=load_type, sequential_failure=False, with_zk=True, + failure_detection_timeout=failure_detection_timeout) return self._perform_node_fail_scenario(test_config) @cluster(num_nodes=NUM_NODES + 3) @version_if(lambda version: version != V_2_8_0) # ignite-zookeeper package is broken in 2.8.0 @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) - def test_2_nodes_fail_sequential_zk(self, ignite_version, load_type): + @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], + failure_detection_timeout=[1000, 5000]) + def test_2_nodes_fail_sequential_zk(self, ignite_version, load_type, failure_detection_timeout): """ Test node failure scenario with ZooKeeperSpi not allowing to fail nodes in a row. """ test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=2, load_type=load_type, - sequential_failure=True, with_zk=True) + sequential_failure=True, with_zk=True, + failure_detection_timeout=failure_detection_timeout) return self._perform_node_fail_scenario(test_config) @@ -141,17 +148,18 @@ def _perform_node_fail_scenario(self, test_config): modules = ['zookeeper'] if test_config.with_zk else None if test_config.with_zk: - zk_quorum = start_zookeeper(self.test_context, 3) + zk_quorum = start_zookeeper(self.test_context, 3, test_config) discovery_spi = from_zookeeper_cluster(zk_quorum) else: discovery_spi = TcpDiscoverySpi() + discovery_spi.socket_linger = test_config.socket_linger + ignite_config = IgniteConfiguration( version=test_config.version, discovery_spi=discovery_spi, - failure_detection_timeout=self.FAILURE_DETECTION_TIMEOUT_ZK if test_config.with_zk - else self.FAILURE_DETECTION_TIMEOUT_TCP, + failure_detection_timeout=test_config.failure_detection_timeout, caches=[CacheConfiguration( name='test-cache', backups=1, @@ -295,12 +303,12 @@ def teardown(self): super().teardown() -def start_zookeeper(test_context, num_nodes): +def start_zookeeper(test_context, num_nodes, test_config): """ Start zookeeper cluster. """ - zk_settings = ZookeeperSettings(min_session_timeout=DiscoveryTest.FAILURE_DETECTION_TIMEOUT_ZK, - tick_time=DiscoveryTest.FAILURE_DETECTION_TIMEOUT_ZK // 3) + zk_settings = ZookeeperSettings(min_session_timeout=test_config.failure_detection_timeout, + tick_time=test_config.failure_detection_timeout // 3) zk_quorum = ZookeeperService(test_context, num_nodes, settings=zk_settings) zk_quorum.start() From 0d881efadf2509f692cab087bc87b203744b33cd Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 30 Oct 2020 19:59:01 +0300 Subject: [PATCH 72/78] first impl. --- .../ducktests/tests/ignitetest/tests/discovery_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 86fd5696bd5a9..a8896209bea8c 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -85,8 +85,8 @@ def __init__(self, test_context): @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) @matrix(nodes_to_kill=[1, 2], - load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 1, 3], - failure_detection_timeout=[500, 1000, 5000]) + load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 3], + failure_detection_timeout=[500, 3000]) def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load_type, socket_linger, failure_detection_timeout): """ @@ -100,8 +100,8 @@ def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 1, 3], - failure_detection_timeout=[500, 1000, 5000]) + @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 3], + failure_detection_timeout=[500, 3000]) def test_2_nodes_fail_sequential_tcp(self, ignite_version, load_type, socket_linger, failure_detection_timeout): """ Test 2 nodes sequential failure scenario with TcpDiscoverySpi. From 1d924cc1056a1ecd0b5b8e791d3e3bc92917f5c9 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 30 Oct 2020 20:14:04 +0300 Subject: [PATCH 73/78] fix --- .../ignite/internal/util/IgniteUtils.java | 3 - .../ignite/spi/discovery/tcp/ServerImpl.java | 114 ++++-------------- .../spi/discovery/tcp/TcpDiscoverySpi.java | 10 -- .../services/utils/templates/log4j.xml.j2 | 4 - .../tests/ignitetest/tests/discovery_test.py | 2 +- 5 files changed, 26 insertions(+), 107 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index aa7c409c42745..6bdc6fab34138 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -7155,9 +7155,6 @@ public static void wrapThreadLoader(ClassLoader ldr, Runnable c) { * @return Short string representing the node. */ public static String toShortString(ClusterNode n) { - if (n == null) - return "ClusterNode [null]"; - return "ClusterNode [id=" + n.id() + ", order=" + n.order() + ", addr=" + n.addresses() + ", daemon=" + n.isDaemon() + ']'; } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index 0f009af0bc855..d0c8e8e49d989 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -3508,20 +3508,12 @@ else if (log.isTraceEnabled()) if (changeTop) hndMsg.changeTopology(ring.previousNodeOf(next).id()); - if (log.isDebugEnabled()) { - log.debug("Sending handshake [" + hndMsg + "] with timeout " + - timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()) + ", sndState=[" + - sndState + ']'); - } + if (log.isDebugEnabled()) + log.debug("Sending handshake [hndMsg=" + hndMsg + ", sndState=" + sndState + ']'); spi.writeToSocket(sock, out, hndMsg, timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout())); - if (log.isTraceEnabled()) { - log.trace("Reading handshake response for [" + hndMsg + "] with timeout " + - timeoutHelper.nextTimeoutChunk(ackTimeout0) + ']'); - } - TcpDiscoveryHandshakeResponse res = spi.readMessage(sock, null, timeoutHelper.nextTimeoutChunk(ackTimeout0)); @@ -3612,7 +3604,7 @@ else if (log.isTraceEnabled()) } } - updateLastSentMessageTime(msg); + updateLastSentMessageTime(); if (log.isDebugEnabled()) log.debug("Initialized connection with next node: " + next.id()); @@ -3714,11 +3706,6 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof if (timeoutHelper == null) timeoutHelper = serverOperationTimeoutHelper(sndState, lastRingMsgSentTime); - if (log.isTraceEnabled()) { - log.trace("Sending pending message [" + pendingMsg + "] with timeout " + - timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()) + '.'); - } - try { spi.writeToSocket(sock, out, pendingMsg, timeoutHelper.nextTimeoutChunk( spi.getSocketTimeout())); @@ -3729,14 +3716,9 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof long tsNanos0 = System.nanoTime(); - if (log.isTraceEnabled()) { - log.trace("Reading receipt for pending message [" + pendingMsg + - "] with timeout " + timeoutHelper.nextTimeoutChunk(ackTimeout0) + '.'); - } - int res = spi.readReceipt(sock, timeoutHelper.nextTimeoutChunk(ackTimeout0)); - updateLastSentMessageTime(pendingMsg); + updateLastSentMessageTime(); spi.stats.onMessageSent(pendingMsg, U.nanosToMillis(tsNanos0 - tsNanos)); @@ -3775,11 +3757,6 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof if (latencyCheck && log.isInfoEnabled()) log.info("Latency check message has been written to socket: " + msg.id()); - if (log.isTraceEnabled()) { - log.trace("Sending [" + msg + "] to " + U.toShortString(next) + - " with timeout " + timeoutHelper.nextTimeoutChunk(spi.getSocketTimeout()) + ']'); - } - spi.writeToSocket(newNextNode ? newNext : next, sock, out, @@ -3788,14 +3765,9 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof long tsNanos0 = System.nanoTime(); - if (log.isTraceEnabled()) { - log.trace("Reading receipt for [" + msg + "] from " + U.toShortString(next) + - " with timeout " + timeoutHelper.nextTimeoutChunk(ackTimeout0) + ']'); - } - int res = spi.readReceipt(sock, timeoutHelper.nextTimeoutChunk(ackTimeout0)); - updateLastSentMessageTime(msg); + updateLastSentMessageTime(); if (latencyCheck && log.isInfoEnabled()) log.info("Latency check message has been acked: " + msg.id()); @@ -3836,44 +3808,25 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof errs.add(e); - if (log.isDebugEnabled()) { - U.error(log, "Failed to send message to next node [next=" + next.id() + - ", msg=" + msg + ", err=" + e + ']', e); - } + if (log.isDebugEnabled()) + U.error(log, "Failed to send message to next node [next=" + next.id() + ", msg=" + msg + + ", err=" + e + ']', e); onException("Failed to send message to next node [next=" + next.id() + ", msg=" + msg + ']', e); - if (spi.failureDetectionTimeoutEnabled() && timeoutHelper.checkFailureTimeoutReached(e)) { - if (log.isDebugEnabled()) { - log.debug("Failure detection timeout has been reached: " + - spi.failureDetectionTimeout()); - } - + if (spi.failureDetectionTimeoutEnabled() && timeoutHelper.checkFailureTimeoutReached(e)) break; - } if (!spi.failureDetectionTimeoutEnabled()) { - if (++reconCnt == spi.getReconnectCount()) { - if (log.isDebugEnabled()) { - log.debug("Maximum reconnection number has been reached: " + - spi.getReconnectCount()); - } - + if (++reconCnt == spi.getReconnectCount()) break; - } else if (e instanceof SocketTimeoutException || X.hasCause(e, SocketTimeoutException.class)) { ackTimeout0 *= 2; - if (!checkAckTimeout(ackTimeout0)) { - if (log.isDebugEnabled()) { - log.debug("Maximum ack tmeout has been reached: " + - spi.getMaxAckTimeout()); - } - + if (!checkAckTimeout(ackTimeout0)) break; - } } } } @@ -3897,22 +3850,8 @@ else if (e instanceof SocketTimeoutException || } // Iterating node's addresses. if (!sent) { - Exception err = errs != null ? - U.exceptionWithSuppressed("Failed to send message to next node [msg=" + msg + - ", next=" + U.toShortString(next) + ']', errs) : - null; - - // If node existed on connection initialization we should check - // whether it has not gone yet. - U.warn(log, "Failed to send message to next node [msg=" + msg + ", next=" + next + - ", errMsg=" + (err != null ? err.getMessage() : "N/A") + ']'); - - if (sndState == null && spi.getEffectiveConnectionRecoveryTimeout() > 0) { + if (sndState == null && spi.getEffectiveConnectionRecoveryTimeout() > 0) sndState = new CrossRingMessageSendState(); - - if (log.isDebugEnabled()) - log.debug("Initialized send state for first fail to send the message: " + state); - } else if (sndState != null && sndState.checkTimeout()) { segmentLocalNodeOnSendFail(failedNodes); @@ -3922,10 +3861,19 @@ else if (sndState != null && sndState.checkTimeout()) { boolean failedNextNode = sndState == null || sndState.markNextNodeFailed(); if (failedNextNode && !failedNodes.contains(next)) { - if (log.isDebugEnabled()) - log.debug("Marking next node [" + U.toShortString(next) + "] as failed."); - failedNodes.add(next); + + if (state == CONNECTED) { + Exception err = errs != null ? + U.exceptionWithSuppressed("Failed to send message to next node [msg=" + msg + + ", next=" + U.toShortString(next) + ']', errs) : + null; + + // If node existed on connection initialization we should check + // whether it has not gone yet. + U.warn(log, "Failed to send message to next node [msg=" + msg + ", next=" + next + + ", errMsg=" + (err != null ? err.getMessage() : "N/A") + ']'); + } } else if (!failedNextNode && sndState != null && sndState.isBackward()) { boolean prev = sndState.markLastFailedNodeAlive(); @@ -3939,9 +3887,6 @@ else if (!failedNextNode && sndState != null && sndState.isBackward()) { newNextNode = false; next = ring.nextNode(failedNodes); - - if (log.isDebugEnabled()) - log.debug("Trying new next node: " + U.toShortString(next)); } } @@ -4096,9 +4041,6 @@ private void registerPendingMessage(TcpDiscoveryAbstractMessage msg) { assert msg != null; if (spi.ensured(msg)) { - if (log.isTraceEnabled()) - log.trace("Registering pending message: " + msg); - pendingMsgs.add(msg); spi.stats.onPendingMessageRegistered(); @@ -6603,11 +6545,8 @@ private IgniteSpiOperationTimeoutHelper serverOperationTimeoutHelper(@Nullable C } /** Fixates time of last sent message. */ - private void updateLastSentMessageTime(TcpDiscoveryAbstractMessage msg) { + private void updateLastSentMessageTime() { lastRingMsgSentTime = System.nanoTime(); - - if (log.isTraceEnabled()) - log.trace("Updated message sent time for message [" + msg + "]."); } /** Thread that executes {@link TcpServer}'s code. */ @@ -8310,9 +8249,6 @@ boolean checkTimeout() { if (System.nanoTime() >= failTimeNanos) { state = RingMessageSendState.FAILED; - if (log.isDebugEnabled()) - log.debug("Reconnect/resend timeout has been reached. Send state: " + this); - return true; } diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java index 4919089f9f116..f1e8406a9afe9 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java @@ -1589,18 +1589,8 @@ protected Socket openSocket(Socket sock, InetSocketAddress remAddr, IgniteSpiOpe assert addr != null; - if (log.isTraceEnabled()) { - log.trace("Connecting to " + resolved + " with timeout " + - timeoutHelper.nextTimeoutChunk(sockTimeout) + '.'); - } - sock.connect(resolved, (int)timeoutHelper.nextTimeoutChunk(sockTimeout)); - if (log.isTraceEnabled()) { - log.trace("Sending the header to " + resolved + " with timeout " + - timeoutHelper.nextTimeoutChunk(sockTimeout) + '.'); - } - writeToSocket(sock, null, U.IGNITE_HEADER, timeoutHelper.nextTimeoutChunk(sockTimeout)); return sock; diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 index f2c2d9f742caa..b2a9a7b48d9a1 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/log4j.xml.j2 @@ -48,10 +48,6 @@ - - - - diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index a8896209bea8c..b55a72f6a01c9 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -71,7 +71,7 @@ class DiscoveryTest(IgniteTest): 2. Kill random node. 3. Wait that survived node detects node failure. """ - NUM_NODES = 5 + NUM_NODES = 9 DATA_AMOUNT = 5_000_000 From 71704c4b9b3cbb22341f5ffb6e372fac10c45a08 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 3 Nov 2020 16:19:14 +0300 Subject: [PATCH 74/78] timeouts to fasten tests. --- .../tests/ignitetest/tests/discovery_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index b55a72f6a01c9..c507c9286b8f7 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -85,8 +85,8 @@ def __init__(self, test_context): @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) @matrix(nodes_to_kill=[1, 2], - load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 3], - failure_detection_timeout=[500, 3000]) + load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 2], + failure_detection_timeout=[1000]) def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load_type, socket_linger, failure_detection_timeout): """ @@ -100,8 +100,8 @@ def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 3], - failure_detection_timeout=[500, 3000]) + @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 2], + failure_detection_timeout=[1000]) def test_2_nodes_fail_sequential_tcp(self, ignite_version, load_type, socket_linger, failure_detection_timeout): """ Test 2 nodes sequential failure scenario with TcpDiscoverySpi. @@ -115,7 +115,7 @@ def test_2_nodes_fail_sequential_tcp(self, ignite_version, load_type, socket_lin @cluster(num_nodes=NUM_NODES + 3) @version_if(lambda version: version != V_2_8_0) # ignite-zookeeper package is broken in 2.8.0 @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(nodes_to_kill=[1, 2], failure_detection_timeout=[1000, 5000], + @matrix(nodes_to_kill=[1, 2], failure_detection_timeout=[1000], load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) def test_nodes_fail_not_sequential_zk(self, ignite_version, nodes_to_kill, load_type, failure_detection_timeout): """ @@ -131,7 +131,7 @@ def test_nodes_fail_not_sequential_zk(self, ignite_version, nodes_to_kill, load_ @version_if(lambda version: version != V_2_8_0) # ignite-zookeeper package is broken in 2.8.0 @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], - failure_detection_timeout=[1000, 5000]) + failure_detection_timeout=[1000]) def test_2_nodes_fail_sequential_zk(self, ignite_version, load_type, failure_detection_timeout): """ Test node failure scenario with ZooKeeperSpi not allowing to fail nodes in a row. From 255b9abd2add8533f3bc104c35fd050ebcaec4d0 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 3 Nov 2020 16:24:29 +0300 Subject: [PATCH 75/78] timeouts to fasten tests 2. --- modules/ducktests/tests/ignitetest/tests/discovery_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index c507c9286b8f7..2e943add3f189 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -85,7 +85,7 @@ def __init__(self, test_context): @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) @matrix(nodes_to_kill=[1, 2], - load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 2], + load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 1], failure_detection_timeout=[1000]) def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load_type, socket_linger, failure_detection_timeout): @@ -100,7 +100,7 @@ def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 2], + @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 1], failure_detection_timeout=[1000]) def test_2_nodes_fail_sequential_tcp(self, ignite_version, load_type, socket_linger, failure_detection_timeout): """ From 09d10815bd45939817e994a8861f42db2c60b6e5 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 3 Nov 2020 20:03:17 +0300 Subject: [PATCH 76/78] First impl. --- .../utils/ignite_configuration/discovery.py | 1 - .../utils/templates/discovery_macro.j2 | 2 +- .../tests/ignitetest/tests/discovery_test.py | 49 +++++++------------ 3 files changed, 20 insertions(+), 32 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py index b22baa1dd2d62..1cd310f16921a 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/discovery.py @@ -106,7 +106,6 @@ def __init__(self, ip_finder=TcpDiscoveryVmIpFinder(), port=47500, port_range=10 self.port = port self.port_range = port_range self.local_address = local_address - self.socket_linger: int = 0 @property def type(self): diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 index 3be087bcef802..41eb15dac0d6d 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 @@ -49,7 +49,7 @@ {% endif %} - + {{ ip_finder(spi) }} {% endmacro %} diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index 2e943add3f189..b970562d97eb8 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -49,7 +49,6 @@ class ClusterLoad(IntEnum): TRANSACTIONAL = 2 -# pylint: disable=R0913 class DiscoveryTestConfig(NamedTuple): """ Configuration for DiscoveryTest. @@ -59,8 +58,6 @@ class DiscoveryTestConfig(NamedTuple): load_type: ClusterLoad = ClusterLoad.NONE sequential_failure: bool = False with_zk: bool = False - socket_linger: int = 0 - failure_detection_timeout: int = 1000 # pylint: disable=W0223 @@ -73,6 +70,8 @@ class DiscoveryTest(IgniteTest): """ NUM_NODES = 9 + FAILURE_DETECTION_TIMEOUT = 1_000 + DATA_AMOUNT = 5_000_000 WARMUP_DATA_AMOUNT = 10_000 @@ -85,60 +84,52 @@ def __init__(self, test_context): @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) @matrix(nodes_to_kill=[1, 2], - load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 1], - failure_detection_timeout=[1000]) - def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load_type, socket_linger, - failure_detection_timeout): + load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) + def test_nodes_fail_not_sequential_tcp(self, ignite_version, nodes_to_kill, load_type): """ Test nodes failure scenario with TcpDiscoverySpi not allowing nodes to fail in a row. """ test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=nodes_to_kill, - load_type=load_type, sequential_failure=False, socket_linger=socket_linger, - failure_detection_timeout=failure_detection_timeout) + load_type=load_type, sequential_failure=False) return self._perform_node_fail_scenario(test_config) @cluster(num_nodes=NUM_NODES) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], socket_linger=[0, 1], - failure_detection_timeout=[1000]) - def test_2_nodes_fail_sequential_tcp(self, ignite_version, load_type, socket_linger, failure_detection_timeout): + @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) + def test_2_nodes_fail_sequential_tcp(self, ignite_version, load_type): """ Test 2 nodes sequential failure scenario with TcpDiscoverySpi. """ test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=2, load_type=load_type, - sequential_failure=True, socket_linger=socket_linger, - failure_detection_timeout=failure_detection_timeout) + sequential_failure=True) return self._perform_node_fail_scenario(test_config) @cluster(num_nodes=NUM_NODES + 3) @version_if(lambda version: version != V_2_8_0) # ignite-zookeeper package is broken in 2.8.0 @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(nodes_to_kill=[1, 2], failure_detection_timeout=[1000], + @matrix(nodes_to_kill=[1, 2], load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) - def test_nodes_fail_not_sequential_zk(self, ignite_version, nodes_to_kill, load_type, failure_detection_timeout): + def test_nodes_fail_not_sequential_zk(self, ignite_version, nodes_to_kill, load_type): """ Test node failure scenario with ZooKeeperSpi not allowing nodes to fail in a row. """ test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=nodes_to_kill, - load_type=load_type, sequential_failure=False, with_zk=True, - failure_detection_timeout=failure_detection_timeout) + load_type=load_type, sequential_failure=False, with_zk=True) return self._perform_node_fail_scenario(test_config) @cluster(num_nodes=NUM_NODES + 3) @version_if(lambda version: version != V_2_8_0) # ignite-zookeeper package is broken in 2.8.0 @ignite_versions(str(DEV_BRANCH), str(LATEST_2_8)) - @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL], - failure_detection_timeout=[1000]) - def test_2_nodes_fail_sequential_zk(self, ignite_version, load_type, failure_detection_timeout): + @matrix(load_type=[ClusterLoad.NONE, ClusterLoad.ATOMIC, ClusterLoad.TRANSACTIONAL]) + def test_2_nodes_fail_sequential_zk(self, ignite_version, load_type): """ Test node failure scenario with ZooKeeperSpi not allowing to fail nodes in a row. """ test_config = DiscoveryTestConfig(version=IgniteVersion(ignite_version), nodes_to_kill=2, load_type=load_type, - sequential_failure=True, with_zk=True, - failure_detection_timeout=failure_detection_timeout) + sequential_failure=True, with_zk=True) return self._perform_node_fail_scenario(test_config) @@ -148,18 +139,16 @@ def _perform_node_fail_scenario(self, test_config): modules = ['zookeeper'] if test_config.with_zk else None if test_config.with_zk: - zk_quorum = start_zookeeper(self.test_context, 3, test_config) + zk_quorum = start_zookeeper(self.test_context, 3) discovery_spi = from_zookeeper_cluster(zk_quorum) else: discovery_spi = TcpDiscoverySpi() - discovery_spi.socket_linger = test_config.socket_linger - ignite_config = IgniteConfiguration( version=test_config.version, discovery_spi=discovery_spi, - failure_detection_timeout=test_config.failure_detection_timeout, + failure_detection_timeout=self.FAILURE_DETECTION_TOMEOT, caches=[CacheConfiguration( name='test-cache', backups=1, @@ -303,12 +292,12 @@ def teardown(self): super().teardown() -def start_zookeeper(test_context, num_nodes, test_config): +def start_zookeeper(test_context, num_nodes): """ Start zookeeper cluster. """ - zk_settings = ZookeeperSettings(min_session_timeout=test_config.failure_detection_timeout, - tick_time=test_config.failure_detection_timeout // 3) + zk_settings = ZookeeperSettings(min_session_timeout=DiscoveryTest.FAILURE_DETECTION_TOMEOT, + tick_time=DiscoveryTest.FAILURE_DETECTION_TOMEOT // 3) zk_quorum = ZookeeperService(test_context, num_nodes, settings=zk_settings) zk_quorum.start() From e6e19349aead07672d7d8891eee75b8f2e6289e0 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 3 Nov 2020 20:09:02 +0300 Subject: [PATCH 77/78] fix. --- modules/ducktests/tests/ignitetest/tests/discovery_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/discovery_test.py b/modules/ducktests/tests/ignitetest/tests/discovery_test.py index b970562d97eb8..d02cf5df01716 100644 --- a/modules/ducktests/tests/ignitetest/tests/discovery_test.py +++ b/modules/ducktests/tests/ignitetest/tests/discovery_test.py @@ -148,7 +148,7 @@ def _perform_node_fail_scenario(self, test_config): ignite_config = IgniteConfiguration( version=test_config.version, discovery_spi=discovery_spi, - failure_detection_timeout=self.FAILURE_DETECTION_TOMEOT, + failure_detection_timeout=self.FAILURE_DETECTION_TIMEOUT, caches=[CacheConfiguration( name='test-cache', backups=1, @@ -296,8 +296,8 @@ def start_zookeeper(test_context, num_nodes): """ Start zookeeper cluster. """ - zk_settings = ZookeeperSettings(min_session_timeout=DiscoveryTest.FAILURE_DETECTION_TOMEOT, - tick_time=DiscoveryTest.FAILURE_DETECTION_TOMEOT // 3) + zk_settings = ZookeeperSettings(min_session_timeout=DiscoveryTest.FAILURE_DETECTION_TIMEOUT, + tick_time=DiscoveryTest.FAILURE_DETECTION_TIMEOUT // 3) zk_quorum = ZookeeperService(test_context, num_nodes, settings=zk_settings) zk_quorum.start() From 23b89961179284470b8c2c0ee39d0075352d5637 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Tue, 3 Nov 2020 20:25:10 +0300 Subject: [PATCH 78/78] fix. --- .../ignitetest/services/utils/templates/discovery_macro.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 index 41eb15dac0d6d..de2f9472b4c81 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/discovery_macro.j2 @@ -49,7 +49,7 @@ {% endif %} - + {{ ip_finder(spi) }} {% endmacro %}