From 38e87178fdff8cb87df86bb27028553f7ecbc31e Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 30 Oct 2024 12:02:43 +0800 Subject: [PATCH 1/4] print correct logs for debug Signed-off-by: chensuyue --- .github/workflows/_manifest-e2e.yml | 6 +++++- ChatQnA/tests/test_manifest_on_xeon.sh | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_manifest-e2e.yml b/.github/workflows/_manifest-e2e.yml index 77386afa89..135e865474 100644 --- a/.github/workflows/_manifest-e2e.yml +++ b/.github/workflows/_manifest-e2e.yml @@ -90,11 +90,15 @@ jobs: echo "Validate ${{ inputs.example }} successful!" else echo "Validate ${{ inputs.example }} failure!!!" - .github/workflows/scripts/k8s-utils.sh dump_all_pod_logs $NAMESPACE exit 1 fi fi + - name: Dump logs when e2e test failed + if: failure() + run: | + .github/workflows/scripts/k8s-utils.sh dump_all_pod_logs $NAMESPACE + - name: Kubectl uninstall if: always() run: | diff --git a/ChatQnA/tests/test_manifest_on_xeon.sh b/ChatQnA/tests/test_manifest_on_xeon.sh index d913421a63..b175a939dd 100755 --- a/ChatQnA/tests/test_manifest_on_xeon.sh +++ b/ChatQnA/tests/test_manifest_on_xeon.sh @@ -40,7 +40,7 @@ function get_end_point() { function validate_chatqna() { local ns=$1 local log=$2 - max_retry=20 + max_retry=10 # make sure microservice retriever-usvc is ready # try to curl retriever-svc for max_retry times test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") @@ -111,7 +111,7 @@ function _cleanup_ns() { function install_and_validate_chatqna_guardrail() { echo "Testing manifests chatqna_guardrils" - local ns=${NAMESPACE}-gaurdrails + local ns=${NAMESPACE} _cleanup_ns $ns kubectl create namespace $ns # install guardrail From fdba86110c6df3dc56e2ec60c8cb583760326882 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 30 Oct 2024 12:03:53 +0800 Subject: [PATCH 2/4] for test Signed-off-by: chensuyue --- .github/workflows/pr-manifest-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-manifest-e2e.yml b/.github/workflows/pr-manifest-e2e.yml index 7a9d2379b5..526dd2dac6 100644 --- a/.github/workflows/pr-manifest-e2e.yml +++ b/.github/workflows/pr-manifest-e2e.yml @@ -4,7 +4,7 @@ name: E2E test with manifests on: - pull_request_target: + pull_request: branches: ["main", "*rc"] types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped paths: From 9ebf18f3ede5399538e9de60fefae453cc612aef Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 30 Oct 2024 21:24:35 +0800 Subject: [PATCH 3/4] update test scripts Signed-off-by: chensuyue --- .github/workflows/_manifest-e2e.yml | 3 ++- ChatQnA/tests/test_manifest_on_gaudi.sh | 8 ++------ ChatQnA/tests/test_manifest_on_xeon.sh | 6 +----- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/_manifest-e2e.yml b/.github/workflows/_manifest-e2e.yml index 135e865474..dedefe3efb 100644 --- a/.github/workflows/_manifest-e2e.yml +++ b/.github/workflows/_manifest-e2e.yml @@ -90,6 +90,7 @@ jobs: echo "Validate ${{ inputs.example }} successful!" else echo "Validate ${{ inputs.example }} failure!!!" + echo "Check the logs in 'Dump logs when e2e test failed' step!!!" exit 1 fi fi @@ -107,4 +108,4 @@ jobs: kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS fi - fi + fi \ No newline at end of file diff --git a/ChatQnA/tests/test_manifest_on_gaudi.sh b/ChatQnA/tests/test_manifest_on_gaudi.sh index 2716efa4b0..30b5ef0ce5 100755 --- a/ChatQnA/tests/test_manifest_on_gaudi.sh +++ b/ChatQnA/tests/test_manifest_on_gaudi.sh @@ -111,7 +111,7 @@ function _cleanup_ns() { function install_and_validate_chatqna_guardrail() { echo "Testing manifests chatqna_guardrils" - local ns=${NAMESPACE}-gaurdrails + local ns=${NAMESPACE} _cleanup_ns $ns kubectl create namespace $ns # install guardrail @@ -119,10 +119,9 @@ function install_and_validate_chatqna_guardrail() { # Sleep enough time for chatqna_guardrail to be ready sleep 60 if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then - echo "Waiting for cahtqna_guardrail pod ready done!" + echo "Waiting for chatqna_guardrail pod ready done!" else echo "Timeout waiting for chatqna_guardrail pod ready!" - _cleanup_ns $ns exit 1 fi @@ -130,10 +129,8 @@ function install_and_validate_chatqna_guardrail() { validate_chatqna $ns chatqna-guardrails local ret=$? if [ $ret -ne 0 ]; then - _cleanup_ns $ns exit 1 fi - _cleanup_ns $ns } if [ $# -eq 0 ]; then @@ -162,7 +159,6 @@ case "$1" in exit $ret fi pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifests - set +e install_and_validate_chatqna_guardrail popd ;; diff --git a/ChatQnA/tests/test_manifest_on_xeon.sh b/ChatQnA/tests/test_manifest_on_xeon.sh index b175a939dd..7c23fb5b92 100755 --- a/ChatQnA/tests/test_manifest_on_xeon.sh +++ b/ChatQnA/tests/test_manifest_on_xeon.sh @@ -119,10 +119,9 @@ function install_and_validate_chatqna_guardrail() { # Sleep enough time for chatqna_guardrail to be ready sleep 60 if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then - echo "Waiting for cahtqna_guardrail pod ready done!" + echo "Waiting for chatqna_guardrail pod ready done!" else echo "Timeout waiting for chatqna_guardrail pod ready!" - _cleanup_ns $ns exit 1 fi @@ -130,10 +129,8 @@ function install_and_validate_chatqna_guardrail() { validate_chatqna $ns chatqna-guardrails local ret=$? if [ $ret -ne 0 ]; then - _cleanup_ns $ns exit 1 fi - _cleanup_ns $ns } if [ $# -eq 0 ]; then @@ -162,7 +159,6 @@ case "$1" in exit $ret fi pushd ChatQnA/kubernetes/intel/cpu/xeon/manifests - set +e install_and_validate_chatqna_guardrail popd ;; From 63625fc00614f435294368c3c5e31a5bbc319141 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:28:51 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/_manifest-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_manifest-e2e.yml b/.github/workflows/_manifest-e2e.yml index dedefe3efb..fc414490da 100644 --- a/.github/workflows/_manifest-e2e.yml +++ b/.github/workflows/_manifest-e2e.yml @@ -108,4 +108,4 @@ jobs: kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS fi - fi \ No newline at end of file + fi