Enable EmbeddedZkServer for KyuubiServer

.gitignore

@@ -43,3 +43,4 @@ kyuubi-server/kyuubi-kdc/
 metrics/report.json
 metrics/.report.json.crc
 /kyuubi-ha/embedded_zookeeper/
+embedded_zookeeper/

bin/load-kyuubi-env.sh

@@ -47,7 +47,7 @@ if [[ -z ${JAVA_HOME} ]]; then
    fi
 fi
 
-export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.11"}"
+export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.12"}"
 
 # Print essential environment variables to console
 echo "JAVA_HOME: ${JAVA_HOME}"

kyuubi-assembly/pom.xml

@@ -44,6 +44,88 @@
             <artifactId>kyuubi-hive-thrift</artifactId>
             <version>${project.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-ha</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-main</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-test</artifactId>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-framework</artifactId>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <scope>compile</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.code.gson</groupId>
+                    <artifactId>gson</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-cli</groupId>
+                    <artifactId>commons-cli</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.mortbay.jetty</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet.jsp</groupId>
+                    <artifactId>jsp-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-logging</groupId>
+                    <artifactId>commons-logging</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.htrace</groupId>
+                    <artifactId>htrace-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.avro</groupId>
+                    <artifactId>avro</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
 </project>

kyuubi-common/src/main/resources/log4j-defaults.properties

@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n

kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala

@@ -17,10 +17,37 @@
 
 package org.apache.kyuubi
 
-import org.slf4j.LoggerFactory
+import org.apache.log4j.{Level, LogManager, PropertyConfigurator}
+import org.slf4j.{Logger, LoggerFactory}
+import org.slf4j.impl.StaticLoggerBinder
 
+/**
+ * Simple version of logging adopted from Apache Spark.
+ */
 trait Logging {
-  lazy val logger = LoggerFactory.getLogger(this.getClass)
+
+  @transient private var log_ : Logger = _
+
+  // Method to get the logger name for this object
+  protected def loggerName: String = {
+    // Ignore trailing $'s in the class names for Scala objects
+    this.getClass.getName.stripSuffix("$")
+  }
+
+  // Method to get or create the logger for this object
+  protected def logger: Logger = {
+    if (log_ == null) {
+      if (!Logging.initialized) {
+        Logging.initLock.synchronized {
+          if (!Logging.initialized) {
+            initializeLogging()
+          }
+        }
+      }
+      log_ = LoggerFactory.getLogger(loggerName)
+    }
+    log_
+  }
 
   def debug(message: => Any): Unit = {
     if (logger.isDebugEnabled) {
@@ -57,4 +84,46 @@ trait Logging {
       logger.error(message.toString)
     }
   }
+
+  private def initializeLogging(): Unit = {
+    if (Logging.isLog4j12) {
+      val log4j12Initialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements
+      // scalastyle:off println
+      if (!log4j12Initialized) {
+        Logging.useDefault = true
+        val defaultLogProps = "log4j-defaults.properties"
+        Option(Thread.currentThread().getContextClassLoader.getResource(defaultLogProps)) match {
+          case Some(url) =>
+            PropertyConfigurator.configure(url)
+          case None =>
+            System.err.println(s"Missing $defaultLogProps")
+        }
+      }
+
+      val rootLogger = LogManager.getRootLogger
+      if (Logging.defaultRootLevel == null) {
+        Logging.defaultRootLevel = rootLogger.getLevel
+      }
+      // scalastyle:on println
+    }
+    Logging.initialized = true
+
+    // Force a call into slf4j to initialize it. Avoids this happening from multiple threads
+    // and triggering this: http://mailman.qos.ch/pipermail/slf4j-dev/2010-April/002956.html
+    logger
+  }
+}
+
+object Logging {
+  @volatile private var useDefault = false
+  @volatile private var defaultRootLevel: Level = _
+  @volatile private var initialized = false
+  val initLock = new Object()
+  private def isLog4j12: Boolean = {
+    // This distinguishes the log4j 1.2 binding, currently
+    // org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently
+    // org.apache.logging.slf4j.Log4jLoggerFactory
+    val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr
+    "org.slf4j.impl.Log4jLoggerFactory".equals(binderClass)
+  }
 }

kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala

@@ -47,6 +47,7 @@ private[kyuubi] object Utils extends Logging {
 
   def getPropertiesFromFile(file: Option[File]): Map[String, String] = {
     file.map { f =>
+      info(s"Loading Kyuubi properties from ${f.getAbsolutePath}")
       val reader = new InputStreamReader(f.toURI.toURL.openStream(), StandardCharsets.UTF_8)
       try {
         val properties = new Properties()

kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

@@ -19,23 +19,29 @@ package org.apache.kyuubi.config
 
 import java.util.concurrent.ConcurrentHashMap
 
-import org.apache.kyuubi.Utils
+import org.apache.kyuubi.{Logging, Utils}
 
-case class KyuubiConf(loadSysDefault: Boolean = true) {
+case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
   private val settings = new ConcurrentHashMap[String, String]()
   private lazy val reader: ConfigProvider = new ConfigProvider(settings)
 
   if (loadSysDefault) {
-    loadSysProps()
+    loadFromMap()
   }
 
-  private def loadSysProps(): KyuubiConf = {
-    for ((key, value) <- Utils.getSystemProperties if key.startsWith(KYUUBI_PREFIX)) {
+  private def loadFromMap(props: Map[String, String] = Utils.getSystemProperties): KyuubiConf = {
+    for ((key, value) <- props if key.startsWith(KYUUBI_PREFIX)) {
       set(key, value)
     }
     this
   }
 
+  def loadFileDefaults(): KyuubiConf = {
+    val maybeConfigFile = Utils.getDefaultPropertiesFile()
+    loadFromMap(Utils.getPropertiesFromFile(maybeConfigFile))
+    this
+  }
+
   def set[T](entry: ConfigEntry[T], value: T): KyuubiConf = {
     settings.put(entry.key, entry.strConverter(value))
     this

kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/server/EmbeddedZkServer.scala

@@ -21,6 +21,7 @@ import java.io.File
 
 import org.apache.curator.test.TestingServer
 
+import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.service.AbstractService
 
@@ -32,7 +33,7 @@ import org.apache.kyuubi.service.AbstractService
  *
  * @param name the service name
  */
-class EmbeddedZkServer private(name: String) extends AbstractService(name) {
+class EmbeddedZkServer private(name: String) extends AbstractService(name) with Logging {
 
   def this() = this(classOf[EmbeddedZkServer].getSimpleName)
 
@@ -51,6 +52,7 @@ class EmbeddedZkServer private(name: String) extends AbstractService(name) {
 
   override def start(): Unit = {
     server.start()
+    info(s"$getName is started at $getConnectString")
     super.start()
   }
 

kyuubi-main/pom.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>kyuubi</artifactId>
+        <groupId>org.apache.kyuubi</groupId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>kyuubi-main</artifactId>
+    <name>Kyuubi Project Main Service</name>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-ha</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+
+</project>

kyuubi-common/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala → kyuubi-main/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala

@@ -17,10 +17,40 @@
 
 package org.apache.kyuubi.server
 
+import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.ha.server.EmbeddedZkServer
+import org.apache.kyuubi.service.CompositeService
+
 object KyuubiServer {
 
   def main(args: Array[String]): Unit = {
+    val conf = new KyuubiConf().loadFileDefaults()
+    val server = new KyuubiServer()
+    server.initialize(conf)
+    server.start()
+
     Thread.sleep(10000)
     print("Hello Kyuubi")
   }
 }
+
+class KyuubiServer(name: String) extends CompositeService(name) {
+
+  def this() = this(classOf[KyuubiServer].getName)
+
+  override def initialize(conf: KyuubiConf): Unit = {
+    this.conf = conf
+    val zkServer = new EmbeddedZkServer()
+    addService(zkServer)
+    super.initialize(conf)
+  }
+
+  override def start(): Unit = {
+    super.start()
+  }
+
+  override def stop(): Unit = {
+    super.stop()
+  }
+
+}