Skip to content

Commit 6811530

Browse files
committed
[KYUUBI #1536] Add session conf advisor
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> closes #1536 ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1892 from ulysses-you/KYUUBI-1536. Closes #1536 f0c450a [ulysses-you] codecov 35fac4f [ulysses-you] fix f34cfbf [ulysses-you] comment 78a54bc [ulysses-you] address comment 5cb574b [ulysses-you] fix test 1033909 [ulysses-you] fix 3c925e5 [ulysses-you] address comment a74988c [ulysses-you] style 649f02d [ulysses-you] Add session conf advisor debd3ad [ulysses-you] Add session conf advisor Authored-by: ulysses-you <ulyssesyou18@gmail.com> Signed-off-by: ulysses-you <ulyssesyou@apache.org>
1 parent d90c664 commit 6811530

File tree

14 files changed

+261
-8
lines changed

14 files changed

+261
-8
lines changed

.github/workflows/style.yml

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,12 @@ jobs:
4141
with:
4242
distribution: zulu
4343
java-version: 8
44+
- name: Install
45+
run: >-
46+
build/mvn clean install -V -Pflink-provided,spark-provided -Dorg.slf4j.simpleLogger.defaultLogLevel=warn \
47+
-Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -DskipTests \
48+
-Pflink-provided,spark-provided \
49+
-pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
4450
- name: Scalastyle with Maven
4551
run: build/mvn scalastyle:check ${{ matrix.profiles }}
4652
- name: Upload scalastyle report
@@ -49,9 +55,4 @@ jobs:
4955
- name: JavaStyle with Maven
5056
run: build/mvn spotless:check ${{ matrix.profiles }}
5157
- name: Check dependency list
52-
run: >-
53-
build/mvn clean install -V -Pflink-provided,spark-provided -Dorg.slf4j.simpleLogger.defaultLogLevel=warn \
54-
-Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -DskipTests \
55-
-Pflink-provided,spark-provided \
56-
-pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
57-
build/dependency.sh
58+
run: build/dependency.sh

dev/kyuubi-codecov/pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,12 @@
6868
<version>${project.version}</version>
6969
</dependency>
7070

71+
<dependency>
72+
<groupId>org.apache.kyuubi</groupId>
73+
<artifactId>kyuubi-server-plugin</artifactId>
74+
<version>${project.version}</version>
75+
</dependency>
76+
7177
<dependency>
7278
<groupId>org.apache.kyuubi</groupId>
7379
<artifactId>kyuubi-spark-sql-engine_${scala.binary.version}</artifactId>

docs/deployment/settings.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,7 @@ Key | Default | Meaning | Type | Since
313313
Key | Default | Meaning | Type | Since
314314
--- | --- | --- | --- | ---
315315
<code>kyuubi.session.check.interval</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT5M</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>The check interval for session timeout.</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.0.0</div>
316+
<code>kyuubi.session.conf.advisor</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>&lt;undefined&gt;</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>A config advisor plugin for Kyuubi Server. This plugin can provide some custom configs for different user or session configs and overwrite the session configs before open a new session. This config value should be a class which is a child of 'org.apache.kyuubi.plugin.SessionConfAdvisor' which has zero-arg constructor.</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.5.0</div>
316317
<code>kyuubi.session.conf.ignore.list</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'></div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>A comma separated list of ignored keys. If the client connection contains any of them, the key and the corresponding value will be removed silently during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but will not forbid users to set dynamic configurations via SET syntax.</div>|<div style='width: 30pt'>seq</div>|<div style='width: 20pt'>1.2.0</div>
317318
<code>kyuubi.session.conf.restrict.list</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'></div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>A comma separated list of restricted keys. If the client connection contains any of them, the connection will be rejected explicitly during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but will not forbid users to set dynamic configurations via SET syntax.</div>|<div style='width: 30pt'>seq</div>|<div style='width: 20pt'>1.2.0</div>
318319
<code>kyuubi.session.engine.check.interval</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT1M</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>The check interval for engine timeout</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.0.0</div>

kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1172,6 +1172,16 @@ object KyuubiConf {
11721172
.checkValues(OperationLanguages.values.map(_.toString))
11731173
.createWithDefault(OperationLanguages.SQL.toString)
11741174

1175+
val SESSION_CONF_ADVISOR: OptionalConfigEntry[String] =
1176+
buildConf("session.conf.advisor")
1177+
.doc("A config advisor plugin for Kyuubi Server. This plugin can provide some custom " +
1178+
"configs for different user or session configs and overwrite the session configs before " +
1179+
"open a new session. This config value should be a class which is a child of " +
1180+
"'org.apache.kyuubi.plugin.SessionConfAdvisor' which has zero-arg constructor.")
1181+
.version("1.5.0")
1182+
.stringConf
1183+
.createOptional
1184+
11751185
val SERVER_NAME: OptionalConfigEntry[String] =
11761186
buildConf("server.name")
11771187
.doc("The name of Kyuubi Server.")

kyuubi-server-plugin/pom.xml

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<project xmlns="http://maven.apache.org/POM/4.0.0"
20+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
22+
<parent>
23+
<artifactId>kyuubi-parent</artifactId>
24+
<groupId>org.apache.kyuubi</groupId>
25+
<version>1.5.0-SNAPSHOT</version>
26+
</parent>
27+
<modelVersion>4.0.0</modelVersion>
28+
29+
<artifactId>kyuubi-server-plugin</artifactId>
30+
<name>Kyuubi Project Server Plugin</name>
31+
<packaging>jar</packaging>
32+
33+
<build>
34+
<plugins>
35+
<plugin>
36+
<groupId>net.alchim31.maven</groupId>
37+
<artifactId>scala-maven-plugin</artifactId>
38+
<configuration>
39+
<skip>true</skip>
40+
</configuration>
41+
</plugin>
42+
</plugins>
43+
</build>
44+
</project>
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.kyuubi.plugin;
19+
20+
public class DefaultSessionConfAdvisor implements SessionConfAdvisor {}
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.kyuubi.plugin;
19+
20+
import java.util.Collections;
21+
import java.util.Map;
22+
23+
/** Provide the session configuration according to the user and session configuration. */
24+
public interface SessionConfAdvisor {
25+
/** The returned conf will overwrite the session conf */
26+
@SuppressWarnings("unchecked")
27+
default Map<String, String> getConfOverlay(String user, Map<String, String> sessionConf) {
28+
return Collections.EMPTY_MAP;
29+
}
30+
}

kyuubi-server/pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,12 @@
3737
<version>${project.version}</version>
3838
</dependency>
3939

40+
<dependency>
41+
<groupId>org.apache.kyuubi</groupId>
42+
<artifactId>kyuubi-server-plugin</artifactId>
43+
<version>${project.version}</version>
44+
</dependency>
45+
4046
<dependency>
4147
<groupId>org.apache.kyuubi</groupId>
4248
<artifactId>kyuubi-ha_${scala.binary.version}</artifactId>
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.kyuubi.plugin
19+
20+
import scala.util.control.NonFatal
21+
22+
import org.apache.kyuubi.KyuubiException
23+
import org.apache.kyuubi.config.KyuubiConf
24+
25+
private[kyuubi] object PluginLoader {
26+
27+
def loadSessionConfAdvisor(conf: KyuubiConf): SessionConfAdvisor = {
28+
val advisorClass = conf.get(KyuubiConf.SESSION_CONF_ADVISOR)
29+
if (advisorClass.isEmpty) {
30+
return new DefaultSessionConfAdvisor()
31+
}
32+
33+
try {
34+
Class.forName(advisorClass.get).getConstructor().newInstance()
35+
.asInstanceOf[SessionConfAdvisor]
36+
} catch {
37+
case _: ClassCastException =>
38+
throw new KyuubiException(
39+
s"Class ${advisorClass.get} is not a child of '${classOf[SessionConfAdvisor].getName}'.")
40+
case NonFatal(e) =>
41+
throw new IllegalArgumentException(s"Error while instantiating '${advisorClass.get}':", e)
42+
}
43+
}
44+
}

kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionImpl.scala

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.kyuubi.session
1919

20+
import scala.collection.JavaConverters._
21+
2022
import com.codahale.metrics.MetricRegistry
2123
import org.apache.hive.service.rpc.thrift._
2224

@@ -45,8 +47,20 @@ class KyuubiSessionImpl(
4547
extends AbstractSession(protocol, user, password, ipAddress, conf, sessionManager) {
4648
override val handle: SessionHandle = SessionHandle(protocol)
4749

50+
private[kyuubi] val optimizedConf: Map[String, String] = {
51+
val confOverlay = sessionManager.sessionConfAdvisor.getConfOverlay(
52+
user,
53+
normalizedConf.asJava)
54+
if (confOverlay != null) {
55+
normalizedConf ++ confOverlay.asScala
56+
} else {
57+
warn(s"the server plugin return null value for user: $user, ignore it")
58+
normalizedConf
59+
}
60+
}
61+
4862
// TODO: needs improve the hardcode
49-
normalizedConf.foreach {
63+
optimizedConf.foreach {
5064
case ("use:database", _) =>
5165
case ("kyuubi.engine.pool.size.threshold", _) =>
5266
case (key, value) => sessionConf.set(key, value)
@@ -90,7 +104,7 @@ class KyuubiSessionImpl(
90104
Option(password).filter(_.nonEmpty).getOrElse("anonymous")
91105
}
92106
_client = KyuubiSyncThriftClient.createClient(user, passwd, host, port, sessionConf)
93-
_engineSessionHandle = _client.openSession(protocol, user, passwd, normalizedConf)
107+
_engineSessionHandle = _client.openSession(protocol, user, passwd, optimizedConf)
94108
logSessionInfo(s"Connected to engine [$host:$port] with ${_engineSessionHandle}")
95109
sessionEvent.openedTime = System.currentTimeMillis()
96110
sessionEvent.remoteSessionId = _engineSessionHandle.identifier.toString

0 commit comments

Comments
 (0)