HDDS-1424. Support multi-container robot test execution

Closes #726

hadoop-ozone/dist/dev-support/bin/dist-layout-stitching

@@ -122,3 +122,6 @@ run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" .
 run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/blockade" .
 run cp -p -r "${ROOT}/hadoop-ozone/dist/target/k8s" kubernetes
 run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/Dockerfile" .
+
+#workaround for https://issues.apache.org/jira/browse/MRESOURCES-236
+find ./compose -name "*.sh" -exec chmod 755 {} \;

hadoop-ozone/dist/pom.xml

@@ -140,7 +140,7 @@
         <version>3.1.0</version>
         <executions>
           <execution>
-            <id>copy-compose</id>
+            <id>copy-compose-files</id>
             <phase>compile</phase>
             <goals>
               <goal>copy-resources</goal>

hadoop-ozone/dist/src/main/compose/ozone/test.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+#Due to the limitation of the current auditparser test, it should be the
+#first test in a clean cluster.
+
+execute_robot_test scm auditparser
+
+execute_robot_test scm basic/basic.robot
+
+stop_docker_env
+
+generate_report

hadoop-ozone/dist/src/main/compose/ozonefs/test.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+execute_robot_test scm ozonefs/ozonefs.robot
+
+
+## TODO: As of now the o3fs tests are unstable.
+
+export OZONE_HOME=/opt/ozone
+
+#execute_robot_test hadoop32 ozonefs/hadoopo3fs.robot
+
+#execute_robot_test hadoop31 ozonefs/hadoopo3fs.robot
+
+stop_docker_env
+
+generate_report

hadoop-ozone/dist/src/main/compose/ozoneperf/test.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+execute_robot_test scm basic/basic.robot
+
+stop_docker_env
+
+generate_report

hadoop-ozone/dist/src/main/compose/ozones3/test.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+execute_robot_test scm basic/basic.robot
+
+execute_robot_test scm s3
+
+stop_docker_env
+
+generate_report

hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+export SECURITY_ENABLED=true
+
+start_docker_env
+
+execute_robot_test scm kinit.robot
+
+execute_robot_test scm basic
+
+execute_robot_test scm security
+
+execute_robot_test scm ozonefs/ozonefs.robot
+
+execute_robot_test scm s3
+
+stop_docker_env
+
+generate_report

hadoop-ozone/dist/src/main/compose/test-all.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#
+# Test executor to test all the compose/*/test.sh test scripts.
+#
+
+SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )
+ALL_RESULT_DIR="$SCRIPT_DIR/result"
+
+mkdir -p "$ALL_RESULT_DIR"
+rm "$ALL_RESULT_DIR/*"
+
+RESULT=0
+IFS=$'\n'
+# shellcheck disable=SC2044
+for test in $(find $SCRIPT_DIR -name test.sh); do
+  echo "Executing test in $(dirname "$test")"
+
+  #required to read the .env file from the right location
+  cd "$(dirname "$test")" || continue
+  $test
+  ret=$?
+  if [[ $ret -ne 0 ]]; then
+      RESULT=-1
+  fi
+  RESULT_DIR="$(dirname "$test")/result"
+  cp "$RESULT_DIR"/robot-*.xml "$ALL_RESULT_DIR"
+done
+
+docker run --rm -v "$SCRIPT_DIR/result:/opt/result" apache/hadoop-runner rebot -N "smoketests" -d "/opt/result" "/opt/result/robot-*.xml"
+exit $RESULT

hadoop-ozone/dist/src/main/compose/test-single.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Single test executor, can start a single robot test in any running container.
+#
+
+
+COMPOSE_DIR="$PWD"
+export COMPOSE_DIR
+
+if [[ ! -f "$COMPOSE_DIR/docker-compose.yaml" ]]; then
+    echo "docker-compose.yaml is missing from the current dir. Please run this command from a docker-compose environment."
+    exit 1
+fi
+if (( $# != 2 )); then
+cat << EOF
+   Single test executor
+
+   Usage:
+
+     ../test-single.sh <container> <robot_test>
+
+        container: Name of the running docker-compose container (docker-compose.yaml is required in the current directory)
+
+        robot_test: name of the robot test or directory relative to the smoketest dir.
+
+
+
+EOF
+
+fi
+
+# shellcheck source=testlib.sh
+source "$COMPOSE_DIR/../testlib.sh"
+
+execute_robot_test "$1" "$2"
+
+generate_report

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+COMPOSE_ENV_NAME=$(basename "$COMPOSE_DIR")
+COMPOSE_FILE=$COMPOSE_DIR/docker-compose.yaml
+RESULT_DIR="$COMPOSE_DIR/result"
+RESULT_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/compose/$(basename "$COMPOSE_ENV_NAME")/result"
+SMOKETEST_DIR_INSIDE="${OZONE_DIR:-/opt/hadoop}/smoketest"
+
+#delete previous results
+rm -rf "$RESULT_DIR"
+mkdir -p "$RESULT_DIR"
+#Should be writeable from the docker containers where user is different.
+chmod ogu+w "$RESULT_DIR"
+
+## @description wait until 3 datanodes are up (or 30 seconds)
+## @param the docker-compose file
+wait_for_datanodes(){
+
+  #Reset the timer
+  SECONDS=0
+
+  #Don't give it up until 30 seconds
+  while [[ $SECONDS -lt 90 ]]; do
+
+     #This line checks the number of HEALTHY datanodes registered in scm over the
+     # jmx HTTP servlet
+     datanodes=$(docker-compose -f "$1" exec -T scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value')
+      if [[ "$datanodes" == "3" ]]; then
+
+        #It's up and running. Let's return from the function.
+         echo "$datanodes datanodes are up and registered to the scm"
+         return
+      else
+
+         #Print it only if a number. Could be not a number if scm is not yet started
+         if [[ "$datanodes" ]]; then
+            echo "$datanodes datanode is up and healthy (until now)"
+         fi
+      fi
+
+      sleep 2
+   done
+   echo "WARNING! Datanodes are not started successfully. Please check the docker-compose files"
+}
+
+## @description  Starts a docker-compose based test environment
+start_docker_env(){
+  docker-compose -f "$COMPOSE_FILE" down
+  docker-compose -f "$COMPOSE_FILE" up -d --scale datanode=3
+  wait_for_datanodes "$COMPOSE_FILE"
+  sleep 10
+}
+
+## @description  Execute robot tests in a specific container.
+## @param        Name of the container in the docker-compose file
+## @param        robot test file or directory relative to the smoketest dir
+execute_robot_test(){
+  CONTAINER="$1"
+  TEST="$2"
+  TEST_NAME=$(basename "$TEST")
+  TEST_NAME=${TEST_NAME%.*}
+  set +e
+  OUTPUT_NAME="$COMPOSE_ENV_NAME-$TEST_NAME-$CONTAINER"
+  docker-compose -f "$COMPOSE_FILE" exec -e  SECURITY_ENABLED="${SECURITY_ENABLED}" -T "$CONTAINER" python -m robot --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$RESULT_DIR_INSIDE/robot-$OUTPUT_NAME.xml" "$SMOKETEST_DIR_INSIDE/$TEST"
+  set -e
+
+}
+
+## @description  Stops a docker-compose based test environment (with saving the logs)
+stop_docker_env(){
+  docker-compose -f "$COMPOSE_FILE" logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log"
+  if [ "${KEEP_RUNNING:-false}" = false ]; then
+     docker-compose -f "$COMPOSE_FILE" down
+  fi
+}
+
+## @description  Generate robot framework reports based on the saved results.
+generate_report(){
+  #Generate the combined output and return with the right exit code (note: robot = execute test, rebot = generate output)
+  docker run --rm -v "$DIR/..:${OZONE_DIR:-/opt/hadoop}" apache/hadoop-runner rebot -d "$RESULT_DIR_INSIDE" "$RESULT_DIR_INSIDE/robot-*.xml"
+}