Skip to content

Commit

Permalink
Introduce ServiceDiscovery
Browse files Browse the repository at this point in the history
  • Loading branch information
yaooqinn committed Jun 17, 2020
1 parent 5100d8a commit e1d55f8
Show file tree
Hide file tree
Showing 17 changed files with 764 additions and 34 deletions.
64 changes: 64 additions & 0 deletions kyuubi-common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -42,17 +42,81 @@
<artifactId>slf4j-api</artifactId>
</dependency>

<!-- ut -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
</dependency>

<dependency>
<groupId>org.apache.directory.server</groupId>
<artifactId>apacheds-service</artifactId>
</dependency>
</dependencies>

<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
<resources>
<resource>
<directory>${project.basedir}/src/main/resources</directory>
</resource>
<resource>
<!-- Include the properties file to provide the build information. -->
<directory>${project.build.directory}/extra-resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>build-info</id>
<phase>generate-resources</phase>
<configuration>
<target>
<exec executable="bash">
<arg value="${project.basedir}/../build/kyuubi-build-info"/>
<arg value="${project.build.directory}/extra-resources"/>
<arg value="${project.version}"/>
<arg value="${spark.version}"/>
</exec>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
<execution>
<id>build-info-test</id>
<phase>generate-test-resources</phase>
<configuration>
<target>
<exec executable="bash">
<arg value="${project.basedir}/../build/kyuubi-build-info"/>
<arg value="${project.build.testOutputDirectory}"/>
<arg value="${project.version}"/>
<arg value="${spark.version}"/>
</exec>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ package org.apache.kyuubi
class KyuubiException(message: String, cause: Throwable) extends Exception(message, cause) {

def this(message: String) = this(message, null)

def this(cause: Throwable) = this(cause.getMessage, cause)
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ case class KyuubiConf(loadSysDefault: Boolean = true) {
this
}

def set[T](entry: OptionalConfigEntry[T], value: T): KyuubiConf = {
set(entry.key, entry.rawStrConverter(value))
this
}

def set(key: String, value: String): KyuubiConf = {
require(key != null)
require(value != null)
Expand All @@ -52,6 +57,15 @@ case class KyuubiConf(loadSysDefault: Boolean = true) {
config.readFrom(reader)
}

/** unset a parameter from the configuration */
def unset(key: String): KyuubiConf = {
settings.remove(key)
this
}

def unset(entry: ConfigEntry[_]): KyuubiConf = {
unset(entry.key)
}
}

object KyuubiConf {
Expand All @@ -64,24 +78,66 @@ object KyuubiConf {

def buildConf(key: String): ConfigBuilder = ConfigBuilder(KYUUBI_PREFIX + key)

val EMBEDDED_ZK_PORT: ConfigEntry[Int] =
buildConf("embedded.zk.port")
.doc("The port of the embedded zookeeper server")
.version("1.0.0")
.intConf.checkValue(_ >= 0, s"The value of $EMBEDDED_ZK_PORT must be >= 0")
.createWithDefault(2181)
val EMBEDDED_ZK_PORT: ConfigEntry[Int] = buildConf("embedded.zookeeper.port")
.doc("The port of the embedded zookeeper server")
.version("1.0.0")
.intConf
.createWithDefault(2181)

val EMBEDDED_ZK_TEMP_DIR: ConfigEntry[String] =
buildConf("embedded.zk.directory")
val EMBEDDED_ZK_TEMP_DIR: ConfigEntry[String] = buildConf("embedded.zookeeper.directory")
.doc("The temporary directory for the embedded zookeeper server")
.version("1.0.0")
.stringConf
.createWithDefault(Utils.resolveURI("embedded_zookeeper").getRawPath)

val HA_ZK_QUORUM: OptionalConfigEntry[Seq[String]] =
buildConf("ha.zk.quorum")
.version("1.0.0")
.stringConf
.toSequence
.createOptional
val HA_ZK_QUORUM: ConfigEntry[String] = buildConf("ha.zookeeper.quorum")
.doc("The connection string for the zookeeper ensemble")
.version("1.0.0")
.stringConf
.createWithDefault("")

val HA_ZK_NAMESPACE: ConfigEntry[String] = buildConf("ha.zookeeper.namespace")
.doc("The connection string for the zookeeper ensemble")
.version("1.0.0")
.stringConf
.createWithDefault("")

val HA_ZK_CONNECTION_MAX_RETRIES: ConfigEntry[Int] =
buildConf("ha.zookeeper.connection.max.retries")
.doc("Max retry times for connecting to the zookeeper ensemble")
.version("1.0.0")
.intConf
.createWithDefault(3)

val HA_ZK_CONNECTION_RETRY_WAIT: ConfigEntry[Int] =
buildConf("ha.zookeeper.connection.retry.wait")
.doc("Initial amount of time to wait between retries to the zookeeper ensemble")
.version("1.0.0")
.intConf
.createWithDefault(1000)

val HA_ZK_CONNECTION_TIMEOUT: ConfigEntry[Int] = buildConf("ha.zookeeper.connection.timeout")
.doc("The timeout(ms) of creating the connection to the zookeeper ensemble")
.version("1.0.0")
.intConf
.createWithDefault(60 * 1000)

val HA_ZK_SESSION_TIMEOUT: ConfigEntry[Int] = buildConf("ha.zookeeper.session.timeout")
.doc("The timeout(ms) of a connected session to be idled")
.version("1.0.0")
.intConf
.createWithDefault(60 * 1000)

val SERVER_PRINCIPAL: OptionalConfigEntry[String] = buildConf("server.principal")
.doc("")
.version("1.0.0")
.stringConf
.createOptional

val SERVER_KEYTAB: OptionalConfigEntry[String] = buildConf("server.keytab")
.doc("")
.version("1.0.0")
.stringConf
.createOptional

}
58 changes: 58 additions & 0 deletions kyuubi-common/src/main/scala/org/apache/kyuubi/package.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache

import java.util.Properties

import scala.util.Try

package object kyuubi {
private object BuildInfo {
private val buildFile = "kyuubi-version-info.properties"
private val buildFileStream =
Thread.currentThread().getContextClassLoader.getResourceAsStream(buildFile)
private val unknown = "<unknown>"
private val props = new Properties()

try {
props.load(buildFileStream)
} catch {
case e: Exception => throw new KyuubiException(e)
} finally {
Try(buildFileStream.close())
}

val version: String = props.getProperty("kyuubi_version", unknown)
val sparkVersion: String = props.getProperty("spark_version", unknown)
val branch: String = props.getProperty("branch", unknown)
val jar: String = props.getProperty("jar", unknown)
val revision: String = props.getProperty("revision", unknown)
val user: String = props.getProperty("user", unknown)
val repoUrl: String = props.getProperty("url", unknown)
val buildDate: String = props.getProperty("date", unknown)
}

val KYUUBI_VERSION: String = BuildInfo.version
val SPARK_COMPILE_VERSION: String = BuildInfo.sparkVersion
val BRANCH: String = BuildInfo.branch
val KYUUBI_JAR_NAME: String = BuildInfo.jar
val REVISION: String = BuildInfo.revision
val BUILD_USER: String = BuildInfo.user
val REPO_URL: String = BuildInfo.repoUrl
val BUILD_DATE: String = BuildInfo.buildDate
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.kyuubi

import java.io.{File, IOException}

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.minikdc.MiniKdc
import org.apache.hadoop.security.UserGroupInformation

trait KerberizedTestHelper {
var kdc: MiniKdc = _
val baseDir: File = Utils.createTempDir(
this.getClass.getProtectionDomain.getCodeSource.getLocation.getPath, "kyuubi-kdc")

try {
val kdcConf = MiniKdc.createConf()
kdcConf.setProperty(MiniKdc.INSTANCE, "KyuubiKrbServer")
kdcConf.setProperty(MiniKdc.ORG_NAME, "KYUUBI")
kdcConf.setProperty(MiniKdc.ORG_DOMAIN, "COM")

if (kdc == null) {
kdc = new MiniKdc(kdcConf, baseDir)
kdc.start()
}
} catch {
case e: IOException =>
throw new AssertionError("unable to create temporary directory: " + e.getMessage)
}

def tryWithSecurityEnabled(block: => Unit): Unit = {
val conf = new Configuration()
assert(!UserGroupInformation.isSecurityEnabled)
val authType = "hadoop.security.authentication"
try {
conf.set(authType, "KERBEROS")
System.setProperty("java.security.krb5.realm", kdc.getRealm)
UserGroupInformation.setConfiguration(conf)
assert(UserGroupInformation.isSecurityEnabled)
block
} finally {
conf.unset(authType)
System.clearProperty("java.security.krb5.realm")
UserGroupInformation.setConfiguration(conf)
assert(!UserGroupInformation.isSecurityEnabled)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,6 @@ class KyuubiConfSuite extends KyuubiFunSuite {
val conf = new KyuubiConf()
assert(conf.get(EMBEDDED_ZK_PORT) === 2181)
assert(conf.get(EMBEDDED_ZK_TEMP_DIR).endsWith("embedded_zookeeper"))
assert(conf.get(HA_ZK_QUORUM) === None)
assert(conf.get(HA_ZK_QUORUM).isEmpty)
}
}
22 changes: 22 additions & 0 deletions kyuubi-ha/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,35 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>

<!-- ut -->
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-common</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<type>test-jar</type>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
</dependency>

<dependency>
<groupId>org.apache.directory.server</groupId>
<artifactId>apacheds-service</artifactId>
</dependency>
</dependencies>

<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
</build>

</project>

0 comments on commit e1d55f8

Please sign in to comment.