[KYUUBI #2433] HiveSQLEngine load required jars from HIVE_HADOOP_CLASSPATH

### _Why are the changes needed?_

We submit a hive job to yarn, when we don't set `kyuubi.engine.hive.extra.classpath`, HiveSQLEngine will throw an error.

My improvement is supporting set `HIVE_HADOOP_CLASSPATH`

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #2446 from deadwind4/hive-cp.

Closes #2433

7caa7fb [Ada Wang] update template
45598cb [Ada Wang] update template
c7d3409 [Ada Wang] [KYUUBI #2433] Load jars from HIVE_HADOOP_CLASSPATH

Authored-by: Ada Wang <wang4luning@gmail.com>
Signed-off-by: Kent Yao <yao@apache.org>

Author: a49a

conf/kyuubi-env.sh.template

@@ -46,6 +46,10 @@
 # - FLINK_HOME              Flink distribution which you would like to use in Kyuubi.
 # - FLINK_CONF_DIR          Optional directory where the Flink configuration lives.
 #                           (Default: $FLINK_HOME/conf)
+# - HIVE_HOME               Hive distribution which you would like to use in Kyuubi.
+# - HIVE_CONF_DIR           Optional directory where the Hive configuration lives.
+#                           (Default: $HIVE_HOME/conf)
+# - HIVE_HADOOP_CLASSPATH   Required Hadoop jars when you use the Kyuubi Hive engine.
 #
 
 
@@ -54,6 +58,8 @@
 # export JAVA_HOME=/usr/jdk64/jdk1.8.0_152
 # export SPARK_HOME=/opt/spark
 # export FLINK_HOME=/opt/flink
+# export HIVE_HOME=/opt/hive
+# export HIVE_HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/common/lib/commons-collections-3.2.2.jar:${HADOOP_HOME}/share/hadoop/client/hadoop-client-runtime-3.1.0.jar:${HADOOP_HOME}/share/hadoop/client/hadoop-client-api-3.1.0.jar:${HADOOP_HOME}/share/hadoop/common/lib/htrace-core4-4.1.0-incubating.jar
 # export HADOOP_CONF_DIR=/usr/ndp/current/mapreduce_client/conf
 # export YARN_CONF_DIR=/usr/ndp/current/yarn/conf
 # export KYUUBI_JAVA_OPTS="-Xmx10g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark -XX:MaxDirectMemorySize=1024m  -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./logs -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:./logs/kyuubi-server-gc-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=5M -XX:NewRatio=3 -XX:MetaspaceSize=512m"

docs/deployment/settings.md

@@ -81,6 +81,10 @@ You can configure the environment variables in `$KYUUBI_HOME/conf/kyuubi-env.sh`
 # - FLINK_HOME              Flink distribution which you would like to use in Kyuubi.
 # - FLINK_CONF_DIR          Optional directory where the Flink configuration lives.
 #                           (Default: $FLINK_HOME/conf)
+# - HIVE_HOME               Hive distribution which you would like to use in Kyuubi.
+# - HIVE_CONF_DIR           Optional directory where the Hive configuration lives.
+#                           (Default: $HIVE_HOME/conf)
+# - HIVE_HADOOP_CLASSPATH   Required Hadoop jars when you use the Kyuubi Hive engine.
 #
 
 
@@ -89,6 +93,8 @@ You can configure the environment variables in `$KYUUBI_HOME/conf/kyuubi-env.sh`
 # export JAVA_HOME=/usr/jdk64/jdk1.8.0_152
 # export SPARK_HOME=/opt/spark
 # export FLINK_HOME=/opt/flink
+# export HIVE_HOME=/opt/hive
+# export HIVE_HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/common/lib/commons-collections-3.2.2.jar:${HADOOP_HOME}/share/hadoop/client/hadoop-client-runtime-3.1.0.jar:${HADOOP_HOME}/share/hadoop/client/hadoop-client-api-3.1.0.jar:${HADOOP_HOME}/share/hadoop/common/lib/htrace-core4-4.1.0-incubating.jar
 # export HADOOP_CONF_DIR=/usr/ndp/current/mapreduce_client/conf
 # export YARN_CONF_DIR=/usr/ndp/current/yarn/conf
 # export KYUUBI_JAVA_OPTS="-Xmx10g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark -XX:MaxDirectMemorySize=1024m  -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./logs -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:./logs/kyuubi-server-gc-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=5M -XX:NewRatio=3 -XX:MetaspaceSize=512m"

kyuubi-server/src/main/scala/org/apache/kyuubi/engine/hive/HiveProcessBuilder.scala

@@ -39,6 +39,8 @@ class HiveProcessBuilder(
 
   private val hiveHome: String = getEngineHome("hive")
 
+  private val HIVE_HADOOP_CLASSPATH: String = "HIVE_HADOOP_CLASSPATH"
+
   override protected def module: String = "kyuubi-hive-sql-engine"
 
   override protected def mainClass: String = "org.apache.kyuubi.engine.hive.HiveSQLEngine"
@@ -77,18 +79,21 @@ class HiveProcessBuilder(
     env.get("YARN_CONF_DIR").foreach(classpathEntries.add)
     // jars from hive distribution
     classpathEntries.add(s"$hiveHome${File.separator}lib${File.separator}*")
+    val hadoopCp = env.get(HIVE_HADOOP_CLASSPATH)
+    hadoopCp.foreach(classpathEntries.add)
     val extraCp = conf.get(ENGINE_HIVE_EXTRA_CLASSPATH)
     extraCp.foreach(classpathEntries.add)
-    if (extraCp.isEmpty) {
-      warn(s"The conf of kyuubi.engine.hive.extra.classpath is empty.")
+    if (hadoopCp.isEmpty && extraCp.isEmpty) {
+      warn(s"The conf of ${HIVE_HADOOP_CLASSPATH} or ${ENGINE_HIVE_EXTRA_CLASSPATH} is empty.")
+      debug("Detected development environment")
       mainResource.foreach { path =>
         val devHadoopJars = Paths.get(path).getParent
           .resolve(s"scala-$SCALA_COMPILE_VERSION")
           .resolve("jars")
         if (!Files.exists(devHadoopJars)) {
-          throw new KyuubiException(s"The path $devHadoopJars does not exists. Please set " +
-            s"kyuubi.engine.hive.extra.classpath for configuring location of " +
-            s"hadoop client jars, etc")
+          throw new KyuubiException(s"The path $devHadoopJars does not exists. " +
+            s"Please set ${HIVE_HADOOP_CLASSPATH} or ${ENGINE_HIVE_EXTRA_CLASSPATH} for " +
+            s"configuring location of hadoop client jars, etc")
         }
         classpathEntries.add(s"$devHadoopJars${File.separator}*")
       }