HDDS-8962. Ensure docker env is stopped (#5011)

hadoop-ozone/dist/src/main/compose/common/hadoop-test.sh

@@ -55,7 +55,3 @@ for HADOOP_VERSION in 2.7.3 3.1.2 3.2.2 3.3.6; do
   save_container_logs nm rm
   stop_containers nm rm
 done
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/compatibility/test.sh

@@ -33,7 +33,3 @@ execute_robot_test om compatibility/om.robot
 execute_robot_test recon compatibility/recon.robot
 execute_robot_test scm compatibility/scm.robot
 execute_robot_test datanode compatibility/dn-one-rocksdb.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozone-csi/test.sh

@@ -27,7 +27,3 @@ source "$COMPOSE_DIR/../testlib.sh"
 start_docker_env
 
 execute_robot_test csi csi.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh

@@ -44,7 +44,3 @@ done
 
 execute_robot_test ${SCM} freon
 execute_robot_test ${SCM} -v USERNAME:httpfs httpfs
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozone-om-ha/disabled-test.sh

@@ -35,7 +35,3 @@ start_docker_env
 startOMs
 
 execute_robot_test scm omha/testOMHA.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozone-om-prepare/test.sh

@@ -68,7 +68,3 @@ start_docker_env
 # Writes should now succeed.
 execute_robot_test scm topology/loaddata.robot
 execute_robot_test scm topology/readdata.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozone-topology/test.sh

@@ -46,7 +46,3 @@ wait_for_port datanode_3 9858 60
 stop_containers datanode_4 datanode_5 datanode_6
 
 execute_robot_test scm -N readdata-second-half topology/readdata.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozone/test-ec.sh

@@ -38,7 +38,3 @@ docker-compose up -d --no-recreate --scale datanode=3
 execute_robot_test scm -v PREFIX:${prefix} -N read-3-datanodes ec/read.robot
 docker-compose up -d --no-recreate --scale datanode=5
 execute_robot_test scm -v container:1 -v count:5 -N EC-recovery replication/wait.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/failing2/test.sh → hadoop-ozone/dist/src/main/compose/ozone/test-failures1.sh

@@ -28,9 +28,6 @@ source "$COMPOSE_DIR/../testlib.sh"
 
 start_docker_env
 
+wait_for_port scm 1234 5
 execute_robot_test scm failing/test1.robot
 execute_robot_test scm failing/test2.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/failing1/test.sh → hadoop-ozone/dist/src/main/compose/ozone/test-failures2.sh

@@ -30,7 +30,3 @@ start_docker_env
 
 execute_robot_test scm failing/test1.robot
 execute_robot_test scm failing/test2.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozone/test-legacy-bucket.sh

@@ -34,7 +34,3 @@ source "$COMPOSE_DIR/../testlib.sh"
 start_docker_env 3
 
 execute_robot_test scm -v BUCKET_LAYOUT:LEGACY recon/recon-nssummary.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozone/test-s3-haproxy.sh

@@ -27,7 +27,3 @@ source "$COMPOSE_DIR/../testlib.sh"
 start_docker_env
 
 execute_robot_test scm s3
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozone/test.sh

@@ -56,7 +56,3 @@ execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-o3fs-buck
 execute_robot_test s3g grpc/grpc-om-s3-metrics.robot
 
 execute_robot_test scm --exclude pre-finalized-snapshot-tests snapshot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozonescripts/test.sh

@@ -41,7 +41,3 @@ execute_robot_test scm basic/single_node.robot
 execute_robot_test scm admincli/pipeline.robot
 
 ${COMPOSE_DIR}/stop.sh
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh

@@ -80,7 +80,3 @@ docker-compose stop scm4.org
 execute_robot_test scm3.org kinit.robot
 wait_for_execute_command scm3.org 60 "ozone admin scm decommission --nodeid=${SCMID} | grep Decommissioned"
 execute_robot_test scm3.org scmha/scm-decommission.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozonesecure-mr/test.sh

@@ -43,7 +43,3 @@ for scheme in o3fs ofs; do
   execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
   execute_robot_test rm -v "SCHEME:${scheme}" -N "mapreduce-${scheme}" mapreduce.robot
 done
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozonesecure/test-certificate-rotation.sh

@@ -31,7 +31,3 @@ export COMPOSE_FILE=docker-compose.yaml:certificate-rotation.yaml
 start_docker_env
 
 execute_robot_test scm certrotation
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozonesecure/test-fcq.sh

@@ -29,7 +29,3 @@ export COMPOSE_FILE=docker-compose.yaml:fcq.yaml
 start_docker_env
 
 execute_robot_test s3g fcq/s3_om_fcq.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh

@@ -29,7 +29,3 @@ export COMPOSE_FILE=docker-compose.yaml:vault.yaml
 start_docker_env
 
 execute_robot_test scm s3
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh

@@ -54,7 +54,3 @@ docker-compose up -d --scale datanode=2
 execute_robot_test scm -v container:1 -v count:2 replication/wait.robot
 docker-compose up -d --scale datanode=3
 execute_robot_test scm -v container:1 -v count:3 replication/wait.robot
-
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/restart/test.sh

@@ -47,6 +47,3 @@ execute_robot_test scm -v PREFIX:post freon/generate.robot
 execute_robot_test scm -v PREFIX:post freon/validate.robot
 execute_robot_test scm -v PREFIX:post freon/generate-chunk.robot
 execute_robot_test scm -v PREFIX:post freon/validate-chunk.robot
-stop_docker_env
-
-generate_report

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -14,7 +14,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-set -e
+set -e -o pipefail
 
 _testlib_this="${BASH_SOURCE[0]}"
 _testlib_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
@@ -120,9 +120,9 @@ wait_for_om_leader() {
   while [[ $SECONDS -lt 120 ]]; do
     local command="ozone admin om getserviceroles --service-id '${OM_SERVICE_ID}'"
     if [[ "${SECURITY_ENABLED}" == 'true' ]]; then
-      status=$(docker-compose exec -T ${SCM} bash -c "kinit -k scm/scm@EXAMPLE.COM -t /etc/security/keytabs/scm.keytab && $command" | grep LEADER)
+      status=$(docker-compose exec -T ${SCM} bash -c "kinit -k scm/scm@EXAMPLE.COM -t /etc/security/keytabs/scm.keytab && $command" | grep LEADER || true)
     else
-      status=$(docker-compose exec -T ${SCM} bash -c "$command" | grep LEADER)
+      status=$(docker-compose exec -T ${SCM} bash -c "$command" | grep LEADER || true)
     fi
     if [[ -n "${status}" ]]; then
       echo "Found OM leader for service ${OM_SERVICE_ID}: $status"
@@ -158,13 +158,12 @@ start_docker_env(){
   export OZONE_SAFEMODE_MIN_DATANODES="${datanode_count}"
 
   docker-compose --ansi never down
-  if ! { docker-compose --ansi never up -d --scale datanode="${datanode_count}" \
-      && wait_for_safemode_exit \
-      && wait_for_om_leader ; }; then
-    [[ -n "$OUTPUT_NAME" ]] || OUTPUT_NAME="$COMPOSE_ENV_NAME"
-    stop_docker_env
-    return 1
-  fi
+
+  trap stop_docker_env EXIT HUP INT TERM
+
+  docker-compose --ansi never up -d --scale datanode="${datanode_count}"
+  wait_for_safemode_exit
+  wait_for_om_leader
 }
 
 ## @description  Execute robot tests in a specific container.
@@ -179,20 +178,22 @@ execute_robot_test(){
   unset 'ARGUMENTS[${#ARGUMENTS[@]}-1]' #Remove the last element, remainings are the custom parameters
   TEST_NAME=$(basename "$TEST")
   TEST_NAME="$(basename "$COMPOSE_DIR")-${TEST_NAME%.*}"
-  set +e
   [[ -n "$OUTPUT_NAME" ]] || OUTPUT_NAME="$COMPOSE_ENV_NAME-$TEST_NAME-$CONTAINER"
 
   # find unique filename
   declare -i i=0
   OUTPUT_FILE="robot-${OUTPUT_NAME}.xml"
   while [[ -f $RESULT_DIR/$OUTPUT_FILE ]]; do
-    let i++
+    let ++i
     OUTPUT_FILE="robot-${OUTPUT_NAME}-${i}.xml"
   done
 
   SMOKETEST_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/smoketest"
 
   OUTPUT_PATH="$RESULT_DIR_INSIDE/${OUTPUT_FILE}"
+
+  set +e
+
   # shellcheck disable=SC2068
   docker-compose exec -T "$CONTAINER" mkdir -p "$RESULT_DIR_INSIDE" \
     && docker-compose exec -T "$CONTAINER" robot \
@@ -209,18 +210,12 @@ execute_robot_test(){
   FULL_CONTAINER_NAME=$(docker-compose ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
   docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/"
 
-  copy_daemon_logs
-
   if [[ ${rc} -gt 0 ]] && [[ ${rc} -le 250 ]]; then
     create_stack_dumps
   fi
 
   set -e
 
-  if [[ ${rc} -gt 0 ]]; then
-    stop_docker_env
-  fi
-
   return ${rc}
 }
 
@@ -263,37 +258,37 @@ copy_daemon_logs() {
 ## @param        container name
 ## @param        specific command to execute
 execute_command_in_container(){
-  set -e
   # shellcheck disable=SC2068
   docker-compose exec -T "$@"
-  set +e
 }
 
 ## @description Stop a list of named containers
 ## @param       List of container names, eg datanode_1 datanode_2
 stop_containers() {
-  set -e
   docker-compose --ansi never stop $@
-  set +e
 }
 
 
 ## @description Start a list of named containers
 ## @param       List of container names, eg datanode_1 datanode_2
 start_containers() {
-  set -e
   docker-compose --ansi never start $@
-  set +e
 }
 
 create_containers() {
-  set -e
   docker-compose --ansi never up -d $@
-  set +e
 }
 
 save_container_logs() {
-  docker-compose --ansi never logs $@ >> "$RESULT_DIR/docker-$OUTPUT_NAME.log"
+  local output_name="${OUTPUT_NAME:-}"
+  if [[ -z "${output_name}" ]]; then
+    output_name="$COMPOSE_ENV_NAME"
+  fi
+  if [[ -z "${output_name}" ]]; then
+    output_name="$(basename $(pwd))"
+  fi
+
+  docker-compose --ansi never logs $@ >> "$RESULT_DIR/docker-${output_name}.log"
 }
 
 
@@ -310,13 +305,9 @@ wait_for_port(){
   SECONDS=0
 
   while [[ $SECONDS -lt $timeout ]]; do
-     set +e
-     docker-compose exec -T ${SCM} /bin/bash -c "nc -z $host $port"
-     status=$?
-     set -e
-     if [ $status -eq 0 ] ; then
-         echo "Port $port is available on $host"
-         return;
+     if docker-compose exec -T ${SCM} /bin/bash -c "nc -z $host $port"; then
+       echo "Port $port is available on $host"
+       return
      fi
      echo "Port $port is not available on $host yet"
      sleep 1
@@ -338,13 +329,9 @@ wait_for_execute_command(){
   SECONDS=0
 
   while [[ $SECONDS -lt $timeout ]]; do
-     set +e
-     docker-compose exec -T $container bash -c '$command'
-     status=$?
-     set -e
-     if [ $status -eq 0 ] ; then
-         echo "$command succeed"
-         return;
+     if docker-compose exec -T $container bash -c '$command'; then
+       echo "$command succeed"
+       return
      fi
      echo "$command hasn't succeed yet"
      sleep 1
@@ -355,6 +342,7 @@ wait_for_execute_command(){
 
 ## @description  Stops a docker-compose based test environment (with saving the logs)
 stop_docker_env(){
+  copy_daemon_logs
   save_container_logs
   if [ "${KEEP_RUNNING:-false}" = false ]; then
      docker-compose --ansi never down

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/manual-upgrade/driver.sh

@@ -52,5 +52,3 @@ callback setup_this_version
 echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO ---"
 OZONE_KEEP_RESULTS=true start_docker_env
 callback with_this_version
-
-stop_docker_env

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh

@@ -84,6 +84,3 @@ OZONE_KEEP_RESULTS=true start_docker_env
 # Sends commands to finalize OM and SCM.
 execute_robot_test "$SCM" upgrade/finalize.robot
 callback with_this_version_finalized
-
-stop_docker_env
-generate_report

hadoop-ozone/dist/src/main/compose/xcompat/test.sh

@@ -126,5 +126,3 @@ for cluster_version in ${old_versions}; do
 done
 
 test_ec_cross_compatibility
-
-generate_report