From 23b85ccf6c3195be56dbca918dd130de8c68504f Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Tue, 20 Jun 2017 14:31:46 -0700 Subject: [PATCH 1/2] KAFKA-5484: Refactor kafkatest docker support --- tests/cluster_file_generator.sh | 63 ---- tests/docker/Dockerfile | 32 +- tests/docker/ducker-ak | 511 ++++++++++++++++++++++++++++++++ tests/docker/run_tests.sh | 52 +--- tests/docker/ssh-config | 21 ++ 5 files changed, 565 insertions(+), 114 deletions(-) delete mode 100644 tests/cluster_file_generator.sh create mode 100755 tests/docker/ducker-ak create mode 100644 tests/docker/ssh-config diff --git a/tests/cluster_file_generator.sh b/tests/cluster_file_generator.sh deleted file mode 100644 index 7894e741a19ec..0000000000000 --- a/tests/cluster_file_generator.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cat<> /etc/sudoers +USER ducker + +CMD sudo service ssh start && tail -f /dev/null diff --git a/tests/docker/ducker-ak b/tests/docker/ducker-ak new file mode 100755 index 0000000000000..1f0618914818b --- /dev/null +++ b/tests/docker/ducker-ak @@ -0,0 +1,511 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Ducker-AK: a tool for running Apache Kafka system tests inside Docker images. +# +# Note: this should be compatible with the version of bash that ships on most +# Macs, bash 3.2.57. +# + +script_path="${0}" + +# The absolute path to the directory which this script is in. This will also be the directory +# which we run docker build from. +ducker_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# The absolute path to the root Kafka directory +kafka_dir="$( cd "${ducker_dir}/../.." && pwd )" + +# The memory consumption to allow during the docker build. +# This does not include swap. +docker_build_memory_limit="3200m" + +# The maximum mmemory consumption to allow in containers. +docker_run_memory_limit="2000m" + +# The default number of cluster nodes to bring up if a number is not specified. +default_num_nodes=14 + +# The default ducker-ak image name. +default_image_name="ducker-ak" + +# Display a usage message on the terminal and exit. +# +# $1: The exit status to use +usage() { + local exit_status="${1}" + cat < /dev/null || die "You must install ${cmd} to run this script." + done +} + +# Set a global variable to a value. +# +# $1: The variable name to set. This function will die if the variable already has a value. The +# variable will be made readonly to prevent any future modifications. +# $2: The value to set the variable to. This function will die if the value is empty or starts +# with a dash. +# $3: A human-readable description of the variable. +set_once() { + local key="${1}" + local value="${2}" + local what="${3}" + [[ -n "${!key}" ]] && die "Error: more than one value specified for ${what}." + verify_command_line_argument "${value}" "${what}" + # It would be better to use declare -g, but older bash versions don't support it. + export ${key}="${value}" +} + +# Verify that a command-line argument is present and does not start with a slash. +# +# $1: The command-line argument to verify. +# $2: A human-readable description of the variable. +verify_command_line_argument() { + local value="${1}" + local what="${2}" + [[ -n "${value}" ]] || die "Error: no value specified for ${what}" + [[ ${value} == -* ]] && die "Error: invalid value ${value} specified for ${what}" +} + +# Echo a message if a flag is set. +# +# $1: If this is 1, the message will be echoed. +# $@: The message +maybe_echo() { + local verbose="${1}" + shift + [[ "${verbose}" -eq 1 ]] && echo "${@}" +} + +# Counts the number of elements passed to this subroutine. +count() { + echo $# +} + +# Push a new directory on to the bash directory stack, or exit with a failure message. +# +# $1: The directory push on to the directory stack. +must_pushd() { + local target_dir="${1}" + pushd -- "${target_dir}" &> /dev/null || die "failed to change directory to ${target_dir}" +} + +# Pop a directory from the bash directory stack, or exit with a failure message. +must_popd() { + popd &> /dev/null || die "failed to popd" +} + +# Run a command and die if it fails. +# +# $1: If this equals 1, we will print the command before executing it. +# $2: If this equals 1, we will show the output of the command. +# $@: The command to run. +must_do() { + local verbose="${1}" + local output="/dev/null" + [[ "${2}" -eq 1 ]] && output="/dev/stdout" + shift 2 + local cmd="${@}" + [[ "${verbose}" -eq 1 ]] && echo "${cmd}" + ${cmd} >${output} || die "${1} failed" +} + +# Ask the user a yes/no question. +# +# $1: The prompt to use +# $_return: 0 if the user answered no; 1 if the user answered yes. +ask_yes_no() { + local prompt="${1}" + while true; do + read -r -p "${prompt} " response + case "${response}" in + [yY]|[yY][eE][sS]) _return=1; return;; + [nN]|[nN][oO]) _return=0; return;; + *);; + esac + echo "Please respond 'yes' or 'no'." + echo + done +} + +# Build a docker image. +# +# $1: The name of the image to build. +ducker_build() { + local image_name="${1}" + + # Use SECONDS, a builtin bash variable that gets incremented each second, to measure the docker + # build duration. + SECONDS=0 + + must_pushd "${ducker_dir}" + must_do 1 1 docker build --memory="${docker_build_memory_limit}" \ + --build-arg "ducker_creator=${user_name}" -t "${image_name}" \ + -f "${ducker_dir}/Dockerfile" ${docker_args} -- . + docker_status=$? + must_popd + duration="${SECONDS}" + if [[ ${docker_status} -ne 0 ]]; then + die "** ERROR: Failed to build ${what} image after $((${duration} / 60))m \ +$((${duration} % 60))s. See ${build_log} for details." + fi + echo "** Successfully built ${what} image in $((${duration} / 60))m \ +$((${duration} % 60))s. See ${build_log} for details." +} + +ducker_up() { + require_commands docker + while [[ $# -ge 1 ]]; do + case "${1}" in + -f|--force) force=1; shift;; + -n|--num-nodes) set_once num_nodes "${2}" "number of nodes"; shift 2;; + *) set_once image_name "${1}" "docker image name"; shift;; + esac + done + [[ -n "${num_nodes}" ]] || num_nodes="${default_num_nodes}" + [[ -n "${image_name}" ]] || image_name=ducker-ak + [[ "${num_nodes}" =~ ^-?[0-9]+$ ]] || \ + die "ducker_up: the number of nodes must be an integer." + [[ "${num_nodes}" -gt 0 ]] || die "ducker_up: the number of nodes must be greater than 0." + if [[ "${num_nodes}" -lt 2 ]]; then + if [[ "${force}" -ne 1 ]]; then + echo "ducker_up: It is recommended to run at least 2 nodes, since ducker01 is only \ +used to run ducktape itself. If you want to do it anyway, you can use --force to attempt to \ +use only ${num_nodes}." + exit 1 + fi + fi + ducker_build "${image_name}" + + docker ps >/dev/null || die "ducker_up: failed to run docker. Please check that the daemon is started." + docker inspect --format='{{.Config.Labels}}' --type=image "${image_name}" | grep -q 'ducker.type' + local docker_status=${PIPESTATUS[0]} + local grep_status=${PIPESTATUS[1]} + [[ "${docker_status}" -eq 0 ]] || die "ducker_up: failed to inspect image ${image_name}. \ +Please check that it exists." + if [[ "${grep_status}" -ne 0 ]]; then + if [[ "${force}" -ne 1 ]]; then + echo "ducker_up: ${image_name} does not appear to be a ducker image. It lacks the \ +ducker.type label. If you think this is a mistake, you can use --force to attempt to bring \ +it up anyway." + exit 1 + fi + fi + local running_containers="$(docker ps -f=network=ducknet -q)" + local num_running_containers=$(count ${running_containers}) + if [[ ${num_running_containers} -gt 0 ]]; then + die "ducker_up: there are ${num_running_containers} ducker containers \ +running already. Use ducker down to bring down these containers before \ +attempting to start new ones." + fi + + echo "ducker_up: Bringing up ${image_name} with ${num_nodes} nodes..." + if docker network inspect ducknet &>/dev/null; then + must_do 1 0 docker network rm ducknet + fi + must_do 1 0 docker network create ducknet + for n in $(seq -f %02g 1 ${num_nodes}); do + local node="ducker${n}" + # Invoke docker-run. Enable the NET_ADMIN and NET_RAW capabilities so that we can use + # iptables inside the container. + must_do 1 0 docker run --cap-add=NET_ADMIN --cap-add=NET_RAW \ + -d -t --net=host -h "${node}" --network ducknet \ + --memory=${docker_run_memory_limit} --memory-swappiness=1 \ + -v "${kafka_dir}:/opt/kafka-dev" --name "${node}" -- "${image_name}" + done + mkdir -p "${ducker_dir}/build" + exec 3<> "${ducker_dir}/build/node_hosts" + for n in $(seq -f %02g 1 ${num_nodes}); do + local node="ducker${n}" + docker exec --user=root "${node}" grep "${node}" /etc/hosts >&3 + [[ $? -ne 0 ]] && die "failed to find the /etc/hosts entry for ${node}" + done + exec 3>&- + for n in $(seq -f %02g 1 ${num_nodes}); do + local node="ducker${n}" + docker exec --user=root "${node}" \ + bash -c "grep -v ${node} /opt/kafka-dev/tests/docker/build/node_hosts >> /etc/hosts" + [[ $? -ne 0 ]] && die "failed to append to the /etc/hosts file on ${node}" + done + + echo "ducker_up: added the latest entries to /etc/hosts on each node." + generate_cluster_json_file "${num_nodes}" "${ducker_dir}/build/cluster.json" + echo "ducker_up: successfully wrote ${ducker_dir}/build/cluster.json" + echo "** ducker_up: successfully brought up ${num_nodes} nodes." +} + +# Generate the cluster.json file used by ducktape to identify cluster nodes. +# +# $1: The number of cluster nodes. +# $2: The path to write the cluster.json file to. +generate_cluster_json_file() { + local num_nodes="${1}" + local path="${2}" + exec 3<> "${path}" +cat<&3 +{ + "_comment": [ + "Licensed to the Apache Software Foundation (ASF) under one or more", + "contributor license agreements. See the NOTICE file distributed with", + "this work for additional information regarding copyright ownership.", + "The ASF licenses this file to You under the Apache License, Version 2.0", + "(the \"License\"); you may not use this file except in compliance with", + "the License. You may obtain a copy of the License at", + "", + "http://www.apache.org/licenses/LICENSE-2.0", + "", + "Unless required by applicable law or agreed to in writing, software", + "distributed under the License is distributed on an \"AS IS\" BASIS,", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", + "See the License for the specific language governing permissions and", + "limitations under the License." + ], + "nodes": [ +EOF + for n in $(seq 2 ${num_nodes}); do + if [[ ${n} -eq ${num_nodes} ]]; then + suffix="" + else + suffix="," + fi + local node=$(printf ducker%02d ${n}) +cat<&3 + { + "externally_routable_ip": "${node}", + "ssh_config": { + "host": "${node}", + "hostname": "${node}", + "identityfile": "/home/ducker/.ssh/id_rsa", + "password": "", + "port": 22, + "user": "ducker" + } + }${suffix} +EOF + done +cat<&3 + ] +} +EOF + exec 3>&- +} + +ducker_test() { + require_commands docker + docker inspect ducker01 &>/dev/null || \ + die "ducker_test: the ducker01 instance appears to be down. Did you run 'ducker up'?" + [[ $# -lt 1 ]] && \ + die "ducker_test: you must supply at least one system test to run. Type --help for help." + local args="" + local kafka_test=0 + for arg in "${@}"; do + local regex=".*\/kafkatest\/(.*)" + if [[ $arg =~ $regex ]]; then + local kpath=${BASH_REMATCH[1]} + args="${args} ./tests/kafkatest/${kpath}" + else + args="${args} ${arg}" + fi + done + must_pushd "${kafka_dir}" + ls ./core/build/distributions/kafka_*.tgz &> /dev/null + [[ $? -eq 0 ]] || die "Failed to find core/build/distributions/kafka_*.tgz. Did you run ./gradlew releaseTarGz?" + must_popd + cmd="cd /opt/kafka-dev && ducktape --cluster-file /opt/kafka-dev/tests/docker/build/cluster.json $args" + echo "docker exec -it ducker01 bash -c \"${cmd}\"" + exec docker exec --user=ducker -it ducker01 bash -c "${cmd}" +} + +ducker_ssh() { + require_commands docker + [[ $# -eq 0 ]] && die "ducker_ssh: Please specify a container name to log into. \ +Currently active containers: $(echo_running_container_names)" + local node_info="${1}" + shift + local guest_command="$*" + local user_name="ducker" + if [[ "${node_info}" =~ @ ]]; then + user_name="${node_info%%@*}" + local node_name="${node_info##*@}" + else + local node_name="${node_info}" + fi + local docker_flags="" + if [[ -z "${guest_command}" ]]; then + local docker_flags="${docker_flags} -t" + local guest_command_prefix="" + guest_command=bash + else + local guest_command_prefix="bash -c" + fi + if [[ "${node_name}" == "all" ]]; then + local nodes=$(echo_running_container_names) + [[ "${nodes}" == "(none)" ]] && die "ducker_ssh: can't locate any running ducker nodes." + for node in ${nodes}; do + docker exec --user=${user_name} -i ${docker_flags} "${node}" \ + ${guest_command_prefix} "${guest_command}" || die "docker exec ${node} failed" + done + else + docker inspect --type=container -- "${node_name}" &>/dev/null || \ + die "ducker_ssh: can't locate node ${node_name}. Currently running nodes: \ +$(echo_running_container_names)" + exec docker exec --user=${user_name} -i ${docker_flags} "${node_name}" \ + ${guest_command_prefix} "${guest_command}" + fi +} + +# Echo all the running Ducker container names, or (none) if there are no running Ducker containers. +echo_running_container_names() { + node_names="$(docker ps -f=network=ducknet -q --format '{{.Names}}' | sort)" + if [[ -z "${node_names}" ]]; then + echo "(none)" + else + echo ${node_names//$'\n'/ } + fi +} + +ducker_down() { + require_commands docker + local verbose=1 + while [[ $# -ge 1 ]]; do + case "${1}" in + -q|--quiet) verbose=0; shift;; + *) die "ducker_down: unexpected command-line argument ${1}";; + esac + done + local running_containers + running_containers="$(docker ps -f=network=ducknet -q)" + [[ $? -eq 0 ]] || die "ducker_down: docker command failed. Is the docker daemon running?" + running_containers=${running_containers//$'\n'/ } + local all_containers="$(docker ps -a -f=network=ducknet -q)" + all_containers=${all_containers//$'\n'/ } + if [[ -z "${all_containers}" ]]; then + maybe_echo "${verbose}" "No ducker containers found." + return + fi + if [[ -n "${running_containers}" ]]; then + must_do ${verbose} 0 docker kill "${running_containers}" + fi + must_do ${verbose} 0 docker rm "${all_containers}" + must_do ${verbose} 1 rm -f -- "${ducker_dir}/build/node_hosts" "${ducker_dir}/build/cluster.json" + if docker network inspect ducknet &>/dev/null; then + must_do 1 0 docker network rm ducknet + fi + maybe_echo "${verbose}" "ducker_down: removed $(count ${all_containers}) containers." +} + +ducker_purge() { + require_commands docker + local force_str="" + while [[ $# -ge 1 ]]; do + case "${1}" in + -f|--force) force_str="-f"; shift;; + *) die "ducker_purge: unknown argument ${1}";; + esac + done + echo "** ducker_purge: attempting to locate ducker images to purge" + local images + images=$(docker images -q -a -f label=ducker.creator) + [[ $? -ne 0 ]] && die "docker images command failed" + images=${images//$'\n'/ } + declare -a purge_images=() + if [[ -z "${images}" ]]; then + echo "** ducker_purge: no images found to purge." + exit 0 + fi + echo "** ducker_purge: images to delete:" + for image in ${images}; do + echo -n "${image} " + docker inspect --format='{{.Config.Labels}} {{.Created}}' --type=image "${image}" + [[ $? -ne 0 ]] && die "docker inspect ${image} failed" + done + ask_yes_no "Delete these docker images? [y/n]" + [[ "${_return}" -eq 0 ]] && exit 0 + must_do 1 1 docker rmi ${force_str} ${images} +} + +# Parse command-line arguments +[[ $# -lt 1 ]] && usage 0 +# Display the help text if -h or --help appears in the command line +for arg in ${@}; do + case "${arg}" in + -h|--help) usage 0;; + --) break;; + *);; + esac +done +action="${1}" +shift +case "${action}" in + help) usage 0;; + + up|test|ssh|down|purge) + ducker_${action} "${@}"; exit 0;; + + *) echo "Unknown command '${action}'. Type '${script_path} --help' for usage information." + exit 1;; +esac diff --git a/tests/docker/run_tests.sh b/tests/docker/run_tests.sh index 94d39a7b2b731..329b556e67440 100755 --- a/tests/docker/run_tests.sh +++ b/tests/docker/run_tests.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,55 +15,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -# To run tests use a command like: -# TC_PATHS="tests/kafkatest/tests/streams tests/kafkatest/tests/tools" bash tests/docker/run_tests.sh -set -x +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +KAFKA_NUM_CONTAINERS=${KAFKA_NUM_CONTAINERS:-14} +TC_PATHS=${TC_PATHS:-./kafkatest/} die() { echo $@ exit 1 } -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -TESTS_DIR=`dirname ${SCRIPT_DIR}` -KAFKA_SRC=`dirname ${TESTS_DIR}` -KAFKA_VERSION=$(grep "version=.*" ${KAFKA_SRC}/gradle.properties | cut -f 2 -d =) -JDK_INFO="openjdk8" -KAFKA_IMAGE=${KAFKA_IMAGE:-kafkadev/kafka-image:${KAFKA_VERSION}}_${JDK_INFO} -export KAFKA_NUM_CONTAINERS=13 - -chmod 600 ${SCRIPT_DIR}/ssh/id_rsa -cd ${KAFKA_SRC} -(( $(ls -1 core/build/distributions/kafka_*SNAPSHOT.tgz | wc -l) != 1 )) && echo 'Expecting a single file like core/build/distributions/kafka_*SNAPSHOT.tgz, found:' $(ls -1 core/build/distributions/kafka_*SNAPSHOT.tgz) && echo "Did you run ./gradlew clean releaseTarGz ?" && exit 1 - -docker network rm knw -docker network create knw - -docker kill $(docker ps -f=network=knw -q) -docker rm $(docker ps -a -f=network=knw -q) - -docker run --rm -it ${KAFKA_IMAGE} "true" -if [[ $? != 0 || ${KAFKA_IMAGE_REBUILD} != "" ]]; then - echo "kafka image ${KAFKA_IMAGE} does not exist. Building it from scratch." - COMMIT_INFO=$(git describe HEAD) - docker build -t ${KAFKA_IMAGE} --label=commit_info=${COMMIT_INFO} ${SCRIPT_DIR} \ - || die "docker build failed" +if ${SCRIPT_DIR}/ducker-ak ssh | grep -q '(none)'; then + ${SCRIPT_DIR}/ducker-ak up -n "${KAFKA_NUM_CONTAINERS}" || die "ducker-ak up failed" fi - -echo "Using kafka image: ${KAFKA_IMAGE}" -docker inspect ${KAFKA_IMAGE} -for i in $(seq -w 1 ${KAFKA_NUM_CONTAINERS}); do - docker run -d -t --name knode${i} --network knw -v ${KAFKA_SRC}:/opt/kafka-dev ${KAFKA_IMAGE} -done - -docker info -docker ps -docker network inspect knw - -for i in $(seq -w 1 ${KAFKA_NUM_CONTAINERS}); do - echo knode${i} - docker exec knode01 bash -c "ssh knode$i hostname" -done - -bash tests/cluster_file_generator.sh > tests/cluster_file.json -docker exec knode01 bash -c "cd /opt/kafka-dev; ducktape ${_DUCKTAPE_OPTIONS} --cluster-file tests/cluster_file.json ${TC_PATHS:-tests/kafkatest/tests}" +${SCRIPT_DIR}/ducker-ak test ${TC_PATHS} || die "ducker-ak test failed" diff --git a/tests/docker/ssh-config b/tests/docker/ssh-config new file mode 100644 index 0000000000000..1f874178074ed --- /dev/null +++ b/tests/docker/ssh-config @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Host * + ControlMaster auto + ControlPath ~/.ssh/master-%r@%h:%p + StrictHostKeyChecking no + ConnectTimeout=10 + IdentityFile ~/.ssh/id_rsa From c8902558e0878396ce11ec26b0c550c65c6703f5 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Thu, 29 Jun 2017 10:25:47 -0700 Subject: [PATCH 2/2] Add a comment to setup.py Add a comment to setup.py explaining that when the ducktape version changes the Dockerfile needs to change too. --- tests/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/setup.py b/tests/setup.py index e43a4abf075fe..631beace6a39b 100644 --- a/tests/setup.py +++ b/tests/setup.py @@ -42,6 +42,7 @@ def run_tests(self): errno = pytest.main(self.pytest_args) sys.exit(errno) +# Note: when changing the version of ducktape, also revise tests/docker/Dockerfile setup(name="kafkatest", version=version, description="Apache Kafka System Tests",