[KYUUBI #3953] [IT][Test][K8S] Fix Kyuubi with Spark On Kubernetes Fl…

…aky Test ### _Why are the changes needed?_ To close #3952 & #3176 ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #3953 from zwangsheng/test/fix_spark_on_kubernetes_flaky_test. Closes #3953 d884f0d [zwangsheng] fix sty;e 9923407 [zwangsheng] using toApplicationState format e219401 [zwangsheng] ifx 471745d [zwangsheng] Test Authored-by: zwangsheng <2213335496@qq.com> Signed-off-by: Cheng Pan <chengpan@apache.org>
apache · Dec 12, 2022 · b4e0a97 · b4e0a97
1 parent 3bb594a
commit b4e0a97
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 7 deletions.
diff --git a/.../src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala b/.../src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
@@ -183,6 +183,8 @@ class KyuubiOperationKubernetesClusterClusterModeSuite
         s" invalid. must conform https://kubernetes.io/docs/concepts/overview/" +
         "working-with-objects/names/#dns-subdomain-names and the value length <= 237")
     }
+    // clean test conf
+    conf.unset(KUBERNETES_EXECUTOR_POD_NAME_PREFIX)
   }
 
   test("Spark Cluster Mode On Kubernetes Kyuubi KubernetesApplicationOperation Suite") {
@@ -223,6 +225,5 @@ class KyuubiOperationKubernetesClusterClusterModeSuite
 
     val failKillResponse = k8sOperation.killApplicationByTag(sessionHandle.identifier.toString)
     assert(!failKillResponse._1)
-    assert(failKillResponse._2 === ApplicationOperation.NOT_FOUND)
   }
 }
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
@@ -23,8 +23,8 @@ import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable
 
 import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.engine.ApplicationState.{ApplicationState, FAILED, FINISHED, PENDING, RUNNING}
-import org.apache.kyuubi.engine.KubernetesApplicationOperation.SPARK_APP_ID_LABEL
+import org.apache.kyuubi.engine.ApplicationState.{ApplicationState, FAILED, FINISHED, PENDING, RUNNING, UNKNOWN}
+import org.apache.kyuubi.engine.KubernetesApplicationOperation.{toApplicationState, SPARK_APP_ID_LABEL}
 import org.apache.kyuubi.util.KubernetesUtils
 
 class KubernetesApplicationOperation extends ApplicationOperation with Logging {
@@ -61,12 +61,19 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
         val operation = findDriverPodByTag(tag)
         val podList = operation.list().getItems
         if (podList.size() != 0) {
-          (
-            operation.delete(),
-            s"Operation of deleted appId: " + s"${podList.get(0).getMetadata.getName} is completed")
+          toApplicationState(podList.get(0).getStatus.getPhase) match {
+            case FAILED | UNKNOWN =>
+              (
+                false,
+                s"Target Pod ${podList.get(0).getMetadata.getName} is in FAILED or UNKNOWN status")
+            case _ =>
+              (
+                operation.delete(),
+                s"Operation of deleted appId: ${podList.get(0).getMetadata.getName} is completed")
+          }
         } else {
           // client mode
-          return jpsOperation.killApplicationByTag(tag)
+          jpsOperation.killApplicationByTag(tag)
         }
       } catch {
         case e: Exception =>