Skip to content

Commit 33b70cf

Browse files
zhaomin1423zhaomin1423
authored
[KYUUBI #3107] [Subtask] DorisSQLEngine - Add process builder (#3123)
* fix conflict * redact password * add jars to cp * update javaOptions
1 parent f36508b commit 33b70cf

File tree

5 files changed

+229
-1
lines changed

5 files changed

+229
-1
lines changed

‎docs/deployment/settings.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,9 @@ kyuubi.engine.jdbc.connection.provider|<undefined>|The connection provider
231231
kyuubi.engine.jdbc.connection.url|<undefined>|The server url that engine will connect to|string|1.6.0
232232
kyuubi.engine.jdbc.connection.user|<undefined>|The user is used for connecting to server|string|1.6.0
233233
kyuubi.engine.jdbc.driver.class|<undefined>|The driver class for jdbc engine connection|string|1.6.0
234+
kyuubi.engine.jdbc.extra.classpath|<undefined>|The extra classpath for the jdbc query engine, for configuring location of jdbc driver, etc|string|1.6.0
235+
kyuubi.engine.jdbc.java.options|<undefined>|The extra java options for the jdbc query engine|string|1.6.0
236+
kyuubi.engine.jdbc.memory|1g|The heap memory for the jdbc query engine|string|1.6.0
234237
kyuubi.engine.jdbc.type|<undefined>|The short name of jdbc type|string|1.6.0
235238
kyuubi.engine.operation.convert.catalog.database.enabled|true|When set to true, The engine converts the JDBC methods of set/get Catalog and set/get Schema to the implementation of different engines|boolean|1.6.0
236239
kyuubi.engine.operation.log.dir.root|engine_operation_logs|Root directory for query operation log at engine-side.|string|1.4.0

‎kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1930,4 +1930,26 @@ object KyuubiConf {
19301930
"Use kyuubi.ha.zookeeper.auth.type and kyuubi.ha.zookeeper.engine.auth.type instead"))
19311931
Map(configs.map { cfg => cfg.key -> cfg }: _*)
19321932
}
1933+
1934+
val ENGINE_JDBC_MEMORY: ConfigEntry[String] =
1935+
buildConf("kyuubi.engine.jdbc.memory")
1936+
.doc("The heap memory for the jdbc query engine")
1937+
.version("1.6.0")
1938+
.stringConf
1939+
.createWithDefault("1g")
1940+
1941+
val ENGINE_JDBC_JAVA_OPTIONS: OptionalConfigEntry[String] =
1942+
buildConf("kyuubi.engine.jdbc.java.options")
1943+
.doc("The extra java options for the jdbc query engine")
1944+
.version("1.6.0")
1945+
.stringConf
1946+
.createOptional
1947+
1948+
val ENGINE_JDBC_EXTRA_CLASSPATH: OptionalConfigEntry[String] =
1949+
buildConf("kyuubi.engine.jdbc.extra.classpath")
1950+
.doc("The extra classpath for the jdbc query engine, for configuring location" +
1951+
" of jdbc driver, etc")
1952+
.version("1.6.0")
1953+
.stringConf
1954+
.createOptional
19331955
}

‎kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,11 @@ import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiSQLException, Logging, Utils}
2929
import org.apache.kyuubi.config.KyuubiConf
3030
import org.apache.kyuubi.config.KyuubiConf._
3131
import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_ENGINE_SUBMIT_TIME_KEY
32-
import org.apache.kyuubi.engine.EngineType.{EngineType, FLINK_SQL, HIVE_SQL, SPARK_SQL, TRINO}
32+
import org.apache.kyuubi.engine.EngineType.{EngineType, FLINK_SQL, HIVE_SQL, JDBC, SPARK_SQL, TRINO}
3333
import org.apache.kyuubi.engine.ShareLevel.{CONNECTION, GROUP, SERVER, ShareLevel}
3434
import org.apache.kyuubi.engine.flink.FlinkProcessBuilder
3535
import org.apache.kyuubi.engine.hive.HiveProcessBuilder
36+
import org.apache.kyuubi.engine.jdbc.JdbcProcessBuilder
3637
import org.apache.kyuubi.engine.spark.SparkProcessBuilder
3738
import org.apache.kyuubi.engine.trino.TrinoProcessBuilder
3839
import org.apache.kyuubi.ha.HighAvailabilityConf.{HA_ENGINE_REF_ID, HA_NAMESPACE}
@@ -181,6 +182,8 @@ private[kyuubi] class EngineRef(
181182
new TrinoProcessBuilder(appUser, conf, engineRefId, extraEngineLog)
182183
case HIVE_SQL =>
183184
new HiveProcessBuilder(appUser, conf, engineRefId, extraEngineLog)
185+
case JDBC =>
186+
new JdbcProcessBuilder(appUser, conf, engineRefId, extraEngineLog)
184187
}
185188

186189
MetricsSystem.tracing(_.incCount(ENGINE_TOTAL))

‎kyuubi-server/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilder.scala

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.kyuubi.engine.jdbc
19+
20+
import java.io.File
21+
import java.nio.file.Paths
22+
import java.util
23+
24+
import scala.collection.JavaConverters._
25+
import scala.collection.mutable.ArrayBuffer
26+
27+
import com.google.common.annotations.VisibleForTesting
28+
29+
import org.apache.kyuubi.{Logging, SCALA_COMPILE_VERSION, Utils}
30+
import org.apache.kyuubi.Utils.REDACTION_REPLACEMENT_TEXT
31+
import org.apache.kyuubi.config.KyuubiConf
32+
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_JDBC_JAVA_OPTIONS, ENGINE_JDBC_MEMORY}
33+
import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY
34+
import org.apache.kyuubi.engine.ProcBuilder
35+
import org.apache.kyuubi.operation.log.OperationLog
36+
37+
class JdbcProcessBuilder(
38+
override val proxyUser: String,
39+
override val conf: KyuubiConf,
40+
val engineRefId: String,
41+
val extraEngineLog: Option[OperationLog] = None)
42+
extends ProcBuilder with Logging {
43+
44+
@VisibleForTesting
45+
def this(proxyUser: String, conf: KyuubiConf) {
46+
this(proxyUser, conf, "")
47+
}
48+
49+
/**
50+
* The short name of the engine process builder, we use this for form the engine jar paths now
51+
* see `mainResource`
52+
*/
53+
override def shortName: String = "jdbc"
54+
55+
override protected def module: String = "kyuubi-jdbc-engine"
56+
57+
/**
58+
* The class containing the main method
59+
*/
60+
override protected def mainClass: String = "org.apache.kyuubi.engine.jdbc.JdbcSQLEngine"
61+
62+
override protected val commands: Array[String] = {
63+
require(
64+
conf.get(ENGINE_JDBC_CONNECTION_URL).nonEmpty,
65+
s"Jdbc server url can not be null! Please set ${ENGINE_JDBC_CONNECTION_URL.key}")
66+
val buffer = new ArrayBuffer[String]()
67+
buffer += executable
68+
69+
val memory = conf.get(ENGINE_JDBC_MEMORY)
70+
buffer += s"-Xmx$memory"
71+
72+
val javaOptions = conf.get(ENGINE_JDBC_JAVA_OPTIONS)
73+
javaOptions.foreach(buffer += _)
74+
75+
buffer += "-cp"
76+
val classpathEntries = new util.LinkedHashSet[String]
77+
mainResource.foreach(classpathEntries.add)
78+
mainResource.foreach { path =>
79+
val parent = Paths.get(path).getParent
80+
if (Utils.isTesting) {
81+
// add dev classpath
82+
val jdbcDeps = parent
83+
.resolve(s"scala-$SCALA_COMPILE_VERSION")
84+
.resolve("jars")
85+
classpathEntries.add(s"$jdbcDeps${File.separator}*")
86+
} else {
87+
// add prod classpath
88+
classpathEntries.add(s"$parent${File.separator}*")
89+
}
90+
}
91+
92+
val extraCp = conf.get(ENGINE_JDBC_EXTRA_CLASSPATH)
93+
extraCp.foreach(classpathEntries.add)
94+
buffer += classpathEntries.asScala.mkString(File.pathSeparator)
95+
buffer += mainClass
96+
97+
buffer += "--conf"
98+
buffer += s"$KYUUBI_SESSION_USER_KEY=$proxyUser"
99+
100+
for ((k, v) <- conf.getAll) {
101+
buffer += "--conf"
102+
buffer += s"$k=$v"
103+
}
104+
buffer.toArray
105+
}
106+
107+
override def toString: String = {
108+
if (commands == null) {
109+
super.toString()
110+
} else {
111+
Utils.redactCommandLineArgs(conf, commands).map {
112+
case arg if arg.contains(ENGINE_JDBC_CONNECTION_PASSWORD.key) =>
113+
s"${ENGINE_JDBC_CONNECTION_PASSWORD.key}=$REDACTION_REPLACEMENT_TEXT"
114+
case arg => arg
115+
}.mkString("\n")
116+
}
117+
}
118+
}

‎kyuubi-server/src/test/scala/org/apache/kyuubi/engine/jdbc/JdbcProcessBuilderSuite.scala

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.kyuubi.engine.jdbc
18+
19+
import org.apache.kyuubi.KyuubiFunSuite
20+
import org.apache.kyuubi.config.KyuubiConf
21+
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_PASSWORD, ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_EXTRA_CLASSPATH, ENGINE_JDBC_JAVA_OPTIONS, ENGINE_JDBC_MEMORY}
22+
23+
class JdbcProcessBuilderSuite extends KyuubiFunSuite {
24+
25+
test("jdbc process builder") {
26+
val conf = KyuubiConf().set("kyuubi.on", "off")
27+
.set(ENGINE_JDBC_CONNECTION_URL.key, "")
28+
.set(ENGINE_JDBC_CONNECTION_PASSWORD.key, "123456")
29+
val builder = new JdbcProcessBuilder("kyuubi", conf)
30+
val commands = builder.toString.split("\n")
31+
assert(commands.head.endsWith("bin/java"), "wrong exec")
32+
assert(builder.toString.contains("--conf\nkyuubi.session.user=kyuubi"))
33+
assert(commands.exists(ss => ss.contains("kyuubi-jdbc-engine")), "wrong classpath")
34+
assert(builder.toString.contains("--conf\nkyuubi.on=off"))
35+
assert(builder.toString.contains(
36+
"--conf\nkyuubi.engine.jdbc.connection.password=*********(redacted)"))
37+
}
38+
39+
test("capture error from jdbc process builder") {
40+
val e1 = intercept[IllegalArgumentException](
41+
new JdbcProcessBuilder("kyuubi", KyuubiConf()).processBuilder)
42+
assert(e1.getMessage contains
43+
s"Jdbc server url can not be null! Please set ${ENGINE_JDBC_CONNECTION_URL.key}")
44+
}
45+
46+
test("default engine memory") {
47+
val conf = KyuubiConf()
48+
.set(ENGINE_JDBC_CONNECTION_URL.key, "")
49+
val builder = new JdbcProcessBuilder("kyuubi", conf)
50+
val commands = builder.toString.split("\n")
51+
assert(commands.contains("-Xmx1g"))
52+
}
53+
54+
test("set engine memory") {
55+
val conf = KyuubiConf()
56+
.set(ENGINE_JDBC_MEMORY, "5g")
57+
.set(ENGINE_JDBC_CONNECTION_URL.key, "")
58+
val builder = new JdbcProcessBuilder("kyuubi", conf)
59+
val commands = builder.toString.split("\n")
60+
assert(commands.contains("-Xmx5g"))
61+
}
62+
63+
test("set engine java options") {
64+
val conf = KyuubiConf()
65+
.set(ENGINE_JDBC_CONNECTION_URL.key, "")
66+
.set(
67+
ENGINE_JDBC_JAVA_OPTIONS,
68+
"-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005")
69+
val builder = new JdbcProcessBuilder("kyuubi", conf)
70+
val commands = builder.toString.split("\n")
71+
assert(commands.contains("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"))
72+
}
73+
74+
test("set extra classpath") {
75+
val conf = KyuubiConf()
76+
.set(ENGINE_JDBC_CONNECTION_URL.key, "")
77+
.set(ENGINE_JDBC_EXTRA_CLASSPATH, "/dummy_classpath/*")
78+
val builder = new JdbcProcessBuilder("kyuubi", conf)
79+
val commands = builder.toString
80+
assert(commands.contains("/dummy_classpath/*"))
81+
}
82+
}

0 commit comments

Comments
 (0)