From 224fa8e4e659f9492eb8f75cc3aafaec37358201 Mon Sep 17 00:00:00 2001 From: wangyang0918 Date: Fri, 7 May 2021 16:22:24 +0800 Subject: [PATCH 1/2] [hotfix][e2e] Output and collect the logs for Kubernetes IT cases --- .../test-scripts/test_kubernetes_itcases.sh | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_itcases.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_itcases.sh index 510a6bc19f1c6..c6d044bcabdab 100755 --- a/flink-end-to-end-tests/test-scripts/test_kubernetes_itcases.sh +++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_itcases.sh @@ -24,11 +24,28 @@ start_kubernetes # Set the ITCASE_KUBECONFIG environment since it is required to run the ITCases export ITCASE_KUBECONFIG=~/.kube/config +if [ -z "$DEBUG_FILES_OUTPUT_DIR"] ; then + export DEBUG_FILES_OUTPUT_DIR="${TEST_DATA_DIR}/log" +fi +LOG4J_PROPERTIES=${END_TO_END_DIR}/../tools/ci/log4j.properties +MVN_LOGGING_OPTIONS="-Dlog.dir=${DEBUG_FILES_OUTPUT_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES" + +function run_mvn_test { + local test_class=$1 + run_mvn test $MVN_LOGGING_OPTIONS -Dtest=${test_class} + + EXIT_CODE=$? + if [ $EXIT_CODE != 0 ]; then + echo "Failed to run Kubernetes ITCase $test_class" + exit $EXIT_CODE + fi +} + cd $END_TO_END_DIR/../flink-kubernetes # Run the ITCases -run_mvn test -Dtest=org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClientITCase -run_mvn test -Dtest=org.apache.flink.kubernetes.kubeclient.resources.KubernetesLeaderElectorITCase -run_mvn test -Dtest=org.apache.flink.kubernetes.highavailability.KubernetesLeaderElectionAndRetrievalITCase -run_mvn test -Dtest=org.apache.flink.kubernetes.highavailability.KubernetesStateHandleStoreITCase -run_mvn test -Dtest=org.apache.flink.kubernetes.highavailability.KubernetesHighAvailabilityRecoverFromSavepointITCase +run_mvn_test org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClientITCase +run_mvn_test org.apache.flink.kubernetes.kubeclient.resources.KubernetesLeaderElectorITCase +run_mvn_test org.apache.flink.kubernetes.highavailability.KubernetesLeaderElectionAndRetrievalITCase +run_mvn_test org.apache.flink.kubernetes.highavailability.KubernetesStateHandleStoreITCase +run_mvn_test org.apache.flink.kubernetes.highavailability.KubernetesHighAvailabilityRecoverFromSavepointITCase From e2e703b430cbdfd387de4e2437a981307731027b Mon Sep 17 00:00:00 2001 From: wangyang0918 Date: Tue, 10 Nov 2020 13:58:14 +0800 Subject: [PATCH 2/2] [FLINK-17857][test] Make K8s e2e tests could run on Mac --- .../test-scripts/common_kubernetes.sh | 35 +++++++------------ 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/flink-end-to-end-tests/test-scripts/common_kubernetes.sh b/flink-end-to-end-tests/test-scripts/common_kubernetes.sh index 9cf5904492669..57e8a1bad71ed 100755 --- a/flink-end-to-end-tests/test-scripts/common_kubernetes.sh +++ b/flink-end-to-end-tests/test-scripts/common_kubernetes.sh @@ -25,7 +25,6 @@ MINIKUBE_START_RETRIES=3 MINIKUBE_START_BACKOFF=5 RESULT_HASH="e682ec6622b5e83f2eb614617d5ab2cf" MINIKUBE_VERSION="v1.8.2" -MINIKUBE_PATH="/usr/local/bin/minikube-$MINIKUBE_VERSION" NON_LINUX_ENV_NOTE="****** Please start/stop minikube manually in non-linux environment. ******" @@ -43,16 +42,18 @@ function setup_kubernetes_for_linux { curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$version/bin/linux/$arch/kubectl && \ chmod +x kubectl && sudo mv kubectl /usr/local/bin/ fi - # Download minikube. - if ! [ -x "$(command -v minikube)" ] || ! [[ $(minikube version) =~ "$MINIKUBE_VERSION" ]]; then - echo "Installing minikube to $MINIKUBE_PATH ..." + # Download minikube when it is not installed beforehand. + if ! [ -x "$(command -v minikube)" ]; then + echo "Installing minikube $MINIKUBE_VERSION ..." curl -Lo minikube https://storage.googleapis.com/minikube/releases/$MINIKUBE_VERSION/minikube-linux-$arch && \ - chmod +x minikube && sudo mv minikube $MINIKUBE_PATH + chmod +x minikube && sudo mv minikube /usr/bin/minikube fi + # conntrack is required for minikube 1.9 and later + sudo apt-get install conntrack } function check_kubernetes_status { - $MINIKUBE_PATH status + minikube status return $? } @@ -75,7 +76,7 @@ function start_kubernetes_if_not_running { # here. # Similarly, the kubelets are marking themself as "low disk space", # causing Flink to avoid this node (again, failing the test) - sudo CHANGE_MINIKUBE_NONE_USER=true $MINIKUBE_PATH start --vm-driver=none \ + sudo CHANGE_MINIKUBE_NONE_USER=true minikube start --vm-driver=none \ --extra-config=kubelet.image-gc-high-threshold=99 \ --extra-config=kubelet.image-gc-low-threshold=98 \ --extra-config=kubelet.minimum-container-ttl-duration=120m \ @@ -83,7 +84,7 @@ function start_kubernetes_if_not_running { --extra-config=kubelet.eviction-soft="memory.available<5Mi,nodefs.available<2Mi,imagefs.available<2Mi" \ --extra-config=kubelet.eviction-soft-grace-period="memory.available=2h,nodefs.available=2h,imagefs.available=2h" # Fix the kubectl context, as it's often stale. - $MINIKUBE_PATH update-context + minikube update-context fi check_kubernetes_status @@ -99,6 +100,7 @@ function start_kubernetes { # Mount Flink dist into minikube virtual machine because we need to mount hostPath as usrlib minikube mount $FLINK_DIR:$FLINK_DIR & export minikube_mount_pid=$! + echo "The mounting process is running with pid $minikube_mount_pid" else setup_kubernetes_for_linux if ! retry_times ${MINIKUBE_START_RETRIES} ${MINIKUBE_START_BACKOFF} start_kubernetes_if_not_running; then @@ -106,16 +108,17 @@ function start_kubernetes { exit 1 fi fi - eval $($MINIKUBE_PATH docker-env) + eval $(minikube docker-env) } function stop_kubernetes { if [[ "${OS_TYPE}" != "linux" ]]; then echo "$NON_LINUX_ENV_NOTE" + echo "Killing mounting process $minikube_mount_pid" kill $minikube_mount_pid 2> /dev/null else echo "Stopping minikube ..." - stop_command="sudo $MINIKUBE_PATH stop" + stop_command="sudo minikube stop" if ! retry_times ${MINIKUBE_START_RETRIES} ${MINIKUBE_START_BACKOFF} "${stop_command}"; then echo "Could not stop minikube. Aborting..." exit 1 @@ -178,18 +181,6 @@ function cleanup { stop_kubernetes } -function setConsoleLogging { - cat >> $FLINK_DIR/conf/log4j.properties <