branch-3.0 [Opt](docker) kerberos docker healthy check (#46662)

docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master-2.sh

@@ -25,12 +25,26 @@ cp /etc/trino/conf/presto-server.keytab /keytabs/other-presto-server.keytab
 cp /keytabs/update-location.sh /etc/hadoop-init.d/update-location.sh
 /usr/local/hadoop-run.sh &
 
-sleep 30
+# check healthy hear
+echo "Waiting for hadoop to be healthy"
+
+for i in {1..10}; do
+    if /usr/local/health.sh; then
+        echo "Hadoop is healthy"
+        break
+    fi
+    echo "Hadoop is not healthy yet. Retrying in 20 seconds..."
+    sleep 20
+done
+
+if [ $i -eq 10 ]; then
+    echo "Hadoop did not become healthy after 120 attempts. Exiting."
+    exit 1
+fi
 
 echo "Init kerberos test data"
 kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM
 hive  -f /usr/local/sql/create_kerberos_hive_table.sql
-
-sleep 20
+touch /mnt/SUCCESS
 
 tail -f /dev/null

docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh

@@ -23,12 +23,26 @@ mkdir -p /etc/hadoop-init.d/
 cp /etc/trino/conf/* /keytabs/
 /usr/local/hadoop-run.sh &
 
-sleep 30
+# check healthy hear
+echo "Waiting for hadoop to be healthy"
+
+for i in {1..10}; do
+    if /usr/local/health.sh; then
+        echo "Hadoop is healthy"
+        break
+    fi
+    echo "Hadoop is not healthy yet. Retrying in 20 seconds..."
+    sleep 20
+done
+
+if [ $i -eq 10 ]; then
+    echo "Hadoop did not become healthy after 120 attempts. Exiting."
+    exit 1
+fi
 
 echo "Init kerberos test data"
 kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM
 hive  -f /usr/local/sql/create_kerberos_hive_table.sql
-
-sleep 20
+touch /mnt/SUCCESS
 
 tail -f /dev/null

docker/thirdparties/docker-compose/kerberos/health-checks/hadoop-health-check.sh

@@ -32,6 +32,7 @@ fi
 FAILED=$(supervisorctl status | grep -v RUNNING || true)
 
 if [ "$FAILED" == "" ]; then
+  echo "All services are running"
   exit 0
 else
   echo "Some of the services are failing: ${FAILED}"

docker/thirdparties/docker-compose/kerberos/health-checks/health.sh

@@ -32,3 +32,4 @@ if test -d "${HEALTH_D}"; then
         "${health_script}" &>> /var/log/container-health.log || exit 1
     done
 fi
+exit 0

docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM
+beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/hadoop-master-2@OTHERREALM.COM" -e "show databases;"

docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM
+beeline -u "jdbc:hive2://localhost:10000/default;principal=hive/hadoop-master@LABS.TERADATA.COM" -e "show databases;"

docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl

@@ -24,12 +24,17 @@ services:
       - ./sql:/usr/local/sql
       - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh
       - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh
+      - ./health-checks/health.sh:/usr/local/health.sh
       - ./health-checks/hadoop-health-check.sh:/etc/health.d/hadoop-health-check.sh
+      - ./health-checks/hive-health-check.sh:/etc/health.d/hive-health-check.sh
       - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh
     hostname: hadoop-master
     entrypoint: /usr/local/entrypoint-hive-master.sh
     healthcheck:
-      test: ./health-checks/health.sh
+      test: ["CMD", "ls", "/mnt/SUCCESS"]
+      interval: 20s
+      timeout: 60s
+      retries: 120
     ports:
       - "5806:5006"
       - "8820:8020"
@@ -50,11 +55,16 @@ services:
       - ./sql:/usr/local/sql
       - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh
       - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh
+      - ./health-checks/health.sh:/usr/local/health.sh
       - ./health-checks/hadoop-health-check.sh:/etc/health.d/hadoop-health-check.sh
+      - ./health-checks/hive-health-check-2.sh:/etc/health.d/hive-health-check-2.sh
       - ./entrypoint-hive-master-2.sh:/usr/local/entrypoint-hive-master-2.sh
     entrypoint: /usr/local/entrypoint-hive-master-2.sh
     healthcheck:
-      test: ./health-checks/health.sh
+      test: ["CMD", "ls", "/mnt/SUCCESS"]
+      interval: 20s
+      timeout: 60s
+      retries: 120
     ports:
       - "15806:5006"
       - "18820:8020"

docker/thirdparties/run-thirdparties-docker.sh

@@ -709,4 +709,6 @@ for compose in "${!pids[@]}"; do
     fi
 done
 
+echo "docker started"
+docker ps -a --format "{{.ID}} | {{.Image}} | {{.Status}}"
 echo "all dockers started successfully"