#155 fix class loader problem when query

apache · Jun 15, 2020 · bf9d2d1 · bf9d2d1
1 parent 6396252
commit bf9d2d1
Show file tree

Hide file tree

Showing 11 changed files with 113 additions and 109 deletions.
diff --git a/build/deploy/context.xml b/build/deploy/context.xml
@@ -33,6 +33,6 @@
     <Valve className="org.apache.catalina.valves.CometConnectionManagerValve" />
     -->
 
-    <Loader loaderClass="org.apache.kylin.ext.CustomizedWebappClassloader"/>
+    <Loader loaderClass="org.apache.kylin.spark.classloader.DebugTomcatClassLoader"/>
 
 </Context>
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
@@ -72,33 +72,6 @@ fi
 echo "version ${version}"
 export version
 
-cp tomcat-ext/target/kylin-tomcat-ext-${version}.jar build/tomcat/lib/kylin-tomcat-ext-${version}.jar
-chmod 644 build/tomcat/lib/kylin-tomcat-ext-${version}.jar
+cp kylin-spark-project/kylin-spark-classloader/target/kylin-spark-classloader-${version}.jar build/tomcat/lib/kylin-spark-classloader-${version}.jar
+chmod 644 build/tomcat/lib/kylin-spark-classloader-${version}.jar
 
-# add ROOT application
-mkdir -p build/tomcat/webapps/ROOT/WEB-INF/
-cat > build/tomcat/webapps/ROOT/index.html <<EOL
-<html>
-  <head>
-    <meta http-equiv="refresh" content="1;url=kylin">
-  </head>
-</html>
-EOL
-
-cat > build/tomcat/webapps/ROOT/WEB-INF/web.xml <<EOL
-<?xml version="1.0" encoding="UTF-8"?>
-<web-app xmlns="http://java.sun.com/xml/ns/javaee"
-           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-           xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
-		  http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
-           version="2.5">
-
-    <display-name>ROOT</display-name>
-    <absolute-ordering />
-    <welcome-file-list>
-        <welcome-file>/index.html</welcome-file>
-    </welcome-file-list>
-
-    <distributable />
-</web-app>
-EOL
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
@@ -342,6 +342,10 @@ kylin.engine.spark-conf.spark.hadoop.yarn.timeline-service.enabled=false
 kylin.engine.spark-conf-mergedict.spark.executor.memory=6G
 kylin.engine.spark-conf-mergedict.spark.memory.fraction=0.2
 
+### Spark conf overwrite for query engine
+kylin.query.spark-conf.spark.executor.cores=5
+kylin.query.spark-conf.spark.executor.instances=4
+
 # manually upload spark-assembly jar to HDFS and then set this property will avoid repeatedly uploading jar at runtime
 #kylin.engine.spark-conf.spark.yarn.archive=hdfs://namenode:8020/kylin/spark/spark-libs.jar
 #kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec

diff --git a/kylin-spark-project/kylin-spark-classloader/pom.xml b/kylin-spark-project/kylin-spark-classloader/pom.xml
@@ -1,4 +1,22 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

diff --git a/...-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/ClassLoaderUtils.java b/...-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/ClassLoaderUtils.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,

diff --git a/...-classloader/src/main/java/org/apache/kylin/spark/classloader/DebugTomcatClassLoader.java b/...-classloader/src/main/java/org/apache/kylin/spark/classloader/DebugTomcatClassLoader.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -101,7 +101,7 @@ public Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundExce
             throw new ClassNotFoundException();
         }
 
-        if (name.startsWith("io.kyligence.kap.ext")) {
+        if (name.startsWith("org.apache.kylin.spark.classloader")) {
             return parent.loadClass(name);
         }
         if (sparkClassLoader.classNeedPreempt(name)) {

diff --git a/...park-classloader/src/main/java/org/apache/kylin/spark/classloader/KylinItClassLoader.java b/...park-classloader/src/main/java/org/apache/kylin/spark/classloader/KylinItClassLoader.java
@@ -7,14 +7,15 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.kylin.spark.classloader;
 
 import java.io.File;
@@ -34,7 +35,7 @@ public class KylinItClassLoader extends URLClassLoader {
             "com.sun.", "launcher.", "javax.", "org.ietf", "java", "org.omg", "org.w3c", "org.xml", "sunw.",
             // logging
             "org.slf4j", "org.apache.commons.logging", "org.apache.log4j", "sun", "org.apache.catalina",
-            "org.apache.tomcat",};
+            "org.apache.tomcat", };
     private static final String[] THIS_CL_PRECEDENT_CLASS = new String[] {"io.kyligence", "org.apache.kylin",
             "org.apache.calcite"};
     private static final String[] CODE_GEN_CLASS = new String[] {"org.apache.spark.sql.catalyst.expressions.Object"};
@@ -78,9 +79,9 @@ public void init() {
                 e.printStackTrace();
             }
         }
-        String spark_home = System.getenv("SPARK_HOME");
+        String sparkHome = System.getenv("SPARK_HOME");
         try {
-            File sparkJar = findFile(spark_home + "/jars", "spark-yarn_.*.jar");
+            File sparkJar = findFile(sparkHome + "/jars", "spark-yarn_.*.jar");
             addURL(sparkJar.toURI().toURL());
             addURL(new File("../examples/test_case_data/sandbox").toURI().toURL());
         } catch (MalformedURLException e) {
@@ -94,7 +95,7 @@ public Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundExce
         if (isCodeGen(name)) {
             throw new ClassNotFoundException();
         }
-        if (name.startsWith("io.kyligence.kap.ext")) {
+        if (name.startsWith("org.apache.kylin.spark.classloader")) {
             return parent.loadClass(name);
         }
         if (isThisCLPrecedent(name)) {
@@ -124,7 +125,6 @@ public Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundExce
                 return clasz;
             }
         }
-        //交换位置 为了让codehua 被父类加载
         if (isParentCLPrecedent(name)) {
             logger.debug("Skipping exempt class " + name + " - delegating directly to parent");
             return parent.loadClass(name);

diff --git a/...classloader/src/main/java/org/apache/kylin/spark/classloader/KylinItSparkClassLoader.java b/...classloader/src/main/java/org/apache/kylin/spark/classloader/KylinItSparkClassLoader.java
@@ -1,25 +1,19 @@
 /*
- * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * http://kyligence.io
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * This software is the confidential and proprietary information of
- * Kyligence Inc. ("Confidential Information"). You shall not disclose
- * such Confidential Information and shall use it only in accordance
- * with the terms of the license agreement you entered into with
- * Kyligence Inc.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package org.apache.kylin.spark.classloader;
@@ -73,15 +67,15 @@ protected KylinItSparkClassLoader(ClassLoader parent) throws IOException {
     }
 
     public void init() throws MalformedURLException {
-        String spark_home = System.getenv("SPARK_HOME");
-        if (spark_home == null) {
-            spark_home = System.getProperty("SPARK_HOME");
-            if (spark_home == null) {
+        String sparkHome = System.getenv("SPARK_HOME");
+        if (sparkHome == null) {
+            sparkHome = System.getProperty("SPARK_HOME");
+            if (sparkHome == null) {
                 throw new RuntimeException(
                         "Spark home not found; set it explicitly or use the SPARK_HOME environment variable.");
             }
         }
-        File file = new File(spark_home + "/jars");
+        File file = new File(sparkHome + "/jars");
         File[] jars = file.listFiles();
         for (File jar : jars) {
             addURL(jar.toURI().toURL());

diff --git a/...-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/SparkClassLoader.java b/...-spark-classloader/src/main/java/org/apache/kylin/spark/classloader/SparkClassLoader.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,23 +18,20 @@
 
 package org.apache.kylin.spark.classloader;
 
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Method;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.HashSet;
 import java.util.Set;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.kylin.spark.classloader.ClassLoaderUtils.findFile;
 
 public class SparkClassLoader extends URLClassLoader {
     //preempt these classes from parent
@@ -61,25 +58,25 @@ public class SparkClassLoader extends URLClassLoader {
     private static Logger logger = LoggerFactory.getLogger(SparkClassLoader.class);
 
     static {
-        String sparkclassloader_spark_cl_preempt_classes = System.getenv("SPARKCLASSLOADER_SPARK_CL_PREEMPT_CLASSES");
-        if (!StringUtils.isEmpty(sparkclassloader_spark_cl_preempt_classes)) {
-            SPARK_CL_PREEMPT_CLASSES = StringUtils.split(sparkclassloader_spark_cl_preempt_classes, ",");
+        String sparkClassLoaderSparkClPreemptClasses = System.getenv("SPARKCLASSLOADER_SPARK_CL_PREEMPT_CLASSES");
+        if (!StringUtils.isEmpty(sparkClassLoaderSparkClPreemptClasses)) {
+            SPARK_CL_PREEMPT_CLASSES = StringUtils.split(sparkClassLoaderSparkClPreemptClasses, ",");
         }
 
-        String sparkclassloader_spark_cl_preempt_files = System.getenv("SPARKCLASSLOADER_SPARK_CL_PREEMPT_FILES");
-        if (!StringUtils.isEmpty(sparkclassloader_spark_cl_preempt_files)) {
-            SPARK_CL_PREEMPT_FILES = StringUtils.split(sparkclassloader_spark_cl_preempt_files, ",");
+        String sparkClassLoaderSparkClPreemptFiles = System.getenv("SPARKCLASSLOADER_SPARK_CL_PREEMPT_FILES");
+        if (!StringUtils.isEmpty(sparkClassLoaderSparkClPreemptFiles)) {
+            SPARK_CL_PREEMPT_FILES = StringUtils.split(sparkClassLoaderSparkClPreemptFiles, ",");
         }
 
-        String sparkclassloader_this_cl_precedent_classes = System.getenv("SPARKCLASSLOADER_THIS_CL_PRECEDENT_CLASSES");
-        if (!StringUtils.isEmpty(sparkclassloader_this_cl_precedent_classes)) {
-            THIS_CL_PRECEDENT_CLASSES = StringUtils.split(sparkclassloader_this_cl_precedent_classes, ",");
+        String sparkClassLoaderThisClPrecedentClasses = System.getenv("SPARKCLASSLOADER_THIS_CL_PRECEDENT_CLASSES");
+        if (!StringUtils.isEmpty(sparkClassLoaderThisClPrecedentClasses)) {
+            THIS_CL_PRECEDENT_CLASSES = StringUtils.split(sparkClassLoaderThisClPrecedentClasses, ",");
         }
 
-        String sparkclassloader_parent_cl_precedent_classes = System
+        String sparkClassLoaderParentClPrecedentClasses = System
                 .getenv("SPARKCLASSLOADER_PARENT_CL_PRECEDENT_CLASSES");
-        if (!StringUtils.isEmpty(sparkclassloader_parent_cl_precedent_classes)) {
-            PARENT_CL_PRECEDENT_CLASSES = StringUtils.split(sparkclassloader_parent_cl_precedent_classes, ",");
+        if (!StringUtils.isEmpty(sparkClassLoaderParentClPrecedentClasses)) {
+            PARENT_CL_PRECEDENT_CLASSES = StringUtils.split(sparkClassLoaderParentClPrecedentClasses, ",");
         }
 
         try {
@@ -111,20 +108,20 @@ protected SparkClassLoader(ClassLoader parent) throws IOException {
     }
 
     public void init() throws MalformedURLException {
-        String spark_home = System.getenv("SPARK_HOME");
-        if (spark_home == null) {
-            spark_home = System.getProperty("SPARK_HOME");
-            if (spark_home == null) {
+        String sparkHome = System.getenv("SPARK_HOME");
+        if (sparkHome == null) {
+            sparkHome = System.getProperty("SPARK_HOME");
+            if (sparkHome == null) {
                 throw new RuntimeException(
                         "Spark home not found; set it explicitly or use the SPARK_HOME environment variable.");
             }
         }
-        File file = new File(spark_home + "/jars");
+        File file = new File(sparkHome + "/jars");
         File[] jars = file.listFiles();
         for (File jar : jars) {
             addURL(jar.toURI().toURL());
         }
-        if (System.getenv("KYLIN_HOME") != null) {
+        /*if (System.getenv("KYLIN_HOME") != null) {
             // for prod
             String kylin_home = System.getenv("KYLIN_HOME");
             File sparkJar = findFile(kylin_home + "/lib", "kylin-udf-.*-SNAPSHOT.jar");
@@ -139,7 +136,7 @@ public void init() throws MalformedURLException {
             //  for debugtomcat
             logger.info("Add kylin UDF classes to spark classloader");
             addURL(new File("../udf/target/classes").toURI().toURL());
-        }
+        }*/
 
     }