[KYUUBI #3627] Support vanilla Jetty for Spark packaged by sbt

### _Why are the changes needed?_

Fix #3627.

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [x] Add screenshots for manual tests if appropriate

manual test, and the class not found error is gone.

- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #3630 from pan3793/jetty.

Closes #3627

bf9e4f9 [Cheng Pan] private
f6c496f [Cheng Pan] nit
25a7c8c [Cheng Pan] [KYUUBI #3627] Support vanilla Jetty for Spark packaged by sbt

Authored-by: Cheng Pan <chengpan@apache.org>
Signed-off-by: Cheng Pan <chengpan@apache.org>
(cherry picked from commit 33c573c)
Signed-off-by: Cheng Pan <chengpan@apache.org>

Author: pan3793

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineTab.scala

@@ -26,6 +26,7 @@ import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.engine.spark.SparkSQLEngine
 import org.apache.kyuubi.engine.spark.events.EngineEventsStore
 import org.apache.kyuubi.service.ServiceState
+import org.apache.kyuubi.util.ClassUtils
 
 /**
  * Note that [[SparkUITab]] is private for Spark
@@ -61,26 +62,43 @@ case class EngineTab(
 
   sparkUI.foreach { ui =>
     try {
-      // Spark shade the jetty package so here we use reflect
+      // Spark shade the jetty package so here we use reflection
+      val sparkServletContextHandlerClz = loadSparkServletContextHandler
       Class.forName("org.apache.spark.ui.SparkUI")
-        .getMethod("attachHandler", classOf[org.sparkproject.jetty.servlet.ServletContextHandler])
+        .getMethod("attachHandler", sparkServletContextHandlerClz)
         .invoke(
           ui,
           Class.forName("org.apache.spark.ui.JettyUtils")
             .getMethod(
               "createRedirectHandler",
               classOf[String],
               classOf[String],
-              classOf[(HttpServletRequest) => Unit],
+              classOf[HttpServletRequest => Unit],
               classOf[String],
               classOf[scala.collection.immutable.Set[String]])
             .invoke(null, "/kyuubi/stop", "/kyuubi", handleKillRequest _, "", Set("GET", "POST")))
     } catch {
-      case NonFatal(e) =>
-        warn(
-          "Failed to attach handler using SparkUI, please check the Spark version. " +
-            s"So the config '${KyuubiConf.ENGINE_UI_STOP_ENABLED.key}' does not work.",
-          e)
+      case NonFatal(cause) => reportInstallError(cause)
+      case cause: NoClassDefFoundError => reportInstallError(cause)
+    }
+  }
+
+  private def reportInstallError(cause: Throwable): Unit = {
+    warn(
+      "Failed to attach handler using SparkUI, please check the Spark version. " +
+        s"So the config '${KyuubiConf.ENGINE_UI_STOP_ENABLED.key}' does not work.",
+      cause)
+  }
+
+  private def loadSparkServletContextHandler: Class[_] = {
+    // [KYUUBI #3627]: the official spark release uses the shaded and relocated jetty classes,
+    // but if use sbt to build for testing, e.g. docker image, it still uses vanilla jetty classes.
+    val shaded = "org.sparkproject.jetty.servlet.ServletContextHandler"
+    val vanilla = "org.eclipse.jetty.servlet.ServletContextHandler"
+    if (ClassUtils.classIsLoadable(shaded)) {
+      Class.forName(shaded)
+    } else {
+      Class.forName(vanilla)
     }
   }