From 64851da632ddd297fcf000fa828df8bf4a966ac6 Mon Sep 17 00:00:00 2001 From: Yuki Iwai Date: Sun, 13 Aug 2023 21:39:17 +0900 Subject: [PATCH] E2E: Add additional check to verify if the components are ready Signed-off-by: Yuki Iwai --- .../v1beta1/scripts/gh-actions/setup-katib.sh | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh b/test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh index 97c322b3db9..f65b88bb5f2 100755 --- a/test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh +++ b/test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh @@ -65,10 +65,40 @@ echo "Deploying Katib" cd ../../../../../ && WITH_DATABASE_TYPE=$WITH_DATABASE_TYPE make deploy && cd - # Wait until all Katib pods is running. +MAX_RETRY_COUNT=10 +COMPONENTS_NUM=3 +if "${DEPLOY_KATIB_UI}"; then + COMPONENTS_NUM=4 +fi +until [ "$(kubectl get pods -n kubeflow -l "katib.kubeflow.org/component in ($WITH_DATABASE_TYPE,controller,ui,db-manager)" -oname | wc -l)" = "${COMPONENTS_NUM}" ]; do + echo -e "Waiting for katib components to be created\n" + kubectl get pods -n kubeflow || : + + sleep 2 + MAX_RETRY_COUNT=$((MAX_RETRY_COUNT - 1)) + if [ "${MAX_RETRY_COUNT}" = "0" ]; then + echo "Failed to create katib components" + exit 1 + fi +done + TIMEOUT=120s kubectl wait --for=condition=ready --timeout=${TIMEOUT} -l "katib.kubeflow.org/component in ($WITH_DATABASE_TYPE,controller,db-manager,ui)" -n kubeflow pod || (kubectl get pods -n kubeflow && kubectl describe pods -n kubeflow && exit 1) +MAX_RETRY_COUNT=10 +until [ "$(kubectl get pod -n kubeflow -l katib.kubeflow.org/component=controller -ojson | jq '.items[].status.conditions[] | select(.type=="ContainersReady").status')" = "True" ]; do + echo -e "Waiting for Katib controller to be ready\n" + kubectl get pods -n kubeflow || : + + sleep 2 + MAX_RETRY_COUNT=$((MAX_RETRY_COUNT - 1)) + if [ "${MAX_RETRY_COUNT}" = "0" ]; then + echo "Failed to set up Katib controller" + exit 1 + fi +done + echo "All Katib components are running." echo "Katib deployments" kubectl -n kubeflow get deploy