Skip to content

Commit c363311

Browse files
wForgetulysses-you
authored andcommitted
[KYUUBI #1528] Record the kyuubi server ip address in event log
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> Record the kyuubi server ip address in event log. For details: #1528 ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [X] Add screenshots for manual tests if appropriate - [X] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1530 from wForget/KYUUBI-1528. Closes #1528 6bc1857 [Wang Zhen] [KYUUBI-1528] Record the kyuubi server ip address in event log, set serverIpAddress in SessionEvent.apply 5f0307d [Wang Zhen] [KYUUBI-1528] Record the kyuubi server ip address in event log Authored-by: Wang Zhen <wangzhen07@qiyi.com> Signed-off-by: ulysses-you <ulyssesyou@apache.org>
1 parent c30d580 commit c363311

File tree

6 files changed

+22
-11
lines changed

6 files changed

+22
-11
lines changed

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/events/SessionEvent.scala

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import org.apache.spark.sql.types.StructType
2222

2323
import org.apache.kyuubi.Utils
2424
import org.apache.kyuubi.engine.spark.KyuubiSparkUtil
25-
import org.apache.kyuubi.session.Session
25+
import org.apache.kyuubi.engine.spark.session.SparkSessionImpl
2626

2727
/**
2828
* Event Tracking for user sessions
@@ -31,13 +31,15 @@ import org.apache.kyuubi.session.Session
3131
* @param startTime Start time
3232
* @param endTime End time
3333
* @param ip Client IP address
34+
* @param serverIp Kyuubi Server IP address
3435
* @param totalOperations how many queries and meta calls
3536
*/
3637
case class SessionEvent(
3738
sessionId: String,
3839
engineId: String,
3940
username: String,
4041
ip: String,
42+
serverIp: String,
4143
startTime: Long,
4244
var endTime: Long = -1L,
4345
var totalOperations: Int = 0) extends KyuubiSparkEvent {
@@ -56,12 +58,13 @@ case class SessionEvent(
5658
}
5759

5860
object SessionEvent {
59-
def apply(session: Session): SessionEvent = {
61+
def apply(session: SparkSessionImpl): SessionEvent = {
6062
new SessionEvent(
6163
session.handle.identifier.toString,
6264
KyuubiSparkUtil.engineId,
6365
session.user,
6466
session.ipAddress,
67+
session.serverIpAddress,
6568
session.createTime)
6669
}
6770
}

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ class SparkSQLSessionManager private (name: String, spark: SparkSession)
7979
protocol,
8080
user,
8181
password,
82+
ipAddress,
8283
clientIp,
8384
conf,
8485
this,

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class SparkSessionImpl(
3030
protocol: TProtocolVersion,
3131
user: String,
3232
password: String,
33+
serverIpAddress: String,
3334
ipAddress: String,
3435
conf: Map[String, String],
3536
sessionManager: SessionManager,
@@ -47,6 +48,8 @@ class SparkSessionImpl(
4748

4849
private val sessionEvent = SessionEvent(this)
4950

51+
def serverIpAddress(): String = serverIpAddress
52+
5053
override def open(): Unit = {
5154
normalizedConf.foreach {
5255
case ("use:database", database) => spark.catalog.setCurrentDatabase(database)

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") {
240240
Seq(
241241
("User", true, None),
242242
("Client IP", true, None),
243+
("Server IP", true, None),
243244
("Session ID", true, None),
244245
("Start Time", true, None),
245246
("Finish Time", true, None),
@@ -264,6 +265,7 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") {
264265
<tr>
265266
<td> {session.username} </td>
266267
<td> {session.ip} </td>
268+
<td> {session.serverIp} </td>
267269
<td> <a href={sessionLink}> {session.sessionId} </a> </td>
268270
<td> {formatDate(session.startTime)} </td>
269271
<td> {if (session.endTime > 0) formatDate(session.endTime)} </td>
@@ -402,6 +404,7 @@ private class SessionStatsTableDataSource(
402404
val ordering: Ordering[SessionEvent] = sortColumn match {
403405
case "User" => Ordering.by(_.username)
404406
case "Client IP" => Ordering.by(_.ip)
407+
case "Server IP" => Ordering.by(_.serverIp)
405408
case "Session ID" => Ordering.by(_.sessionId)
406409
case "Start Time" => Ordering.by(_.startTime)
407410
case "Finish Time" => Ordering.by(_.endTime)

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ case class EngineSessionPage(parent: EngineTab)
4545
<h4>
4646
User {sessionStat.username},
4747
IP {sessionStat.ip},
48+
Server {sessionStat.serverIp},
4849
Session created at {formatDate(sessionStat.startTime)},
4950
Total run {sessionStat.totalOperations} SQL
5051
</h4> ++

externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/events/EngineEventsStoreSuite.scala

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ class EngineEventsStoreSuite extends KyuubiFunSuite {
2626
test("ensure that the sessions are stored in order") {
2727
val store = new EngineEventsStore(KyuubiConf())
2828

29-
val s1 = SessionEvent("a", "ea", "test1", "1.1.1.1", 1L)
30-
val s2 = SessionEvent("c", "ea", "test2", "1.1.1.1", 3L)
31-
val s3 = SessionEvent("b", "ea", "test3", "1.1.1.1", 2L)
29+
val s1 = SessionEvent("a", "ea", "test1", "1.1.1.1", "1.1.1.2", 1L)
30+
val s2 = SessionEvent("c", "ea", "test2", "1.1.1.1", "1.1.1.2", 3L)
31+
val s3 = SessionEvent("b", "ea", "test3", "1.1.1.1", "1.1.1.2", 2L)
3232

3333
store.saveSession(s1)
3434
store.saveSession(s2)
@@ -45,7 +45,7 @@ class EngineEventsStoreSuite extends KyuubiFunSuite {
4545

4646
val store = new EngineEventsStore(conf)
4747
for (i <- 1 to 5) {
48-
val s = SessionEvent(s"b$i", "ea", s"test$i", "1.1.1.1", 2L)
48+
val s = SessionEvent(s"b$i", "ea", s"test$i", "1.1.1.1", "1.1.1.2", 2L)
4949
store.saveSession(s)
5050
}
5151

@@ -58,18 +58,18 @@ class EngineEventsStoreSuite extends KyuubiFunSuite {
5858

5959
val store = new EngineEventsStore(conf)
6060

61-
store.saveSession(SessionEvent("s1", "ea", "test1", "1.1.1.1", 1L, -1L))
62-
store.saveSession(SessionEvent("s2", "ea", "test1", "1.1.1.1", 2L, -1L))
63-
store.saveSession(SessionEvent("s3", "ea", "test1", "1.1.1.1", 3L, 1L))
64-
store.saveSession(SessionEvent("s4", "ea", "test1", "1.1.1.1", 4L, -1L))
61+
store.saveSession(SessionEvent("s1", "ea", "test1", "1.1.1.1", "1.1.1.2", 1L, -1L))
62+
store.saveSession(SessionEvent("s2", "ea", "test1", "1.1.1.1", "1.1.1.2", 2L, -1L))
63+
store.saveSession(SessionEvent("s3", "ea", "test1", "1.1.1.1", "1.1.1.2", 3L, 1L))
64+
store.saveSession(SessionEvent("s4", "ea", "test1", "1.1.1.1", "1.1.1.2", 4L, -1L))
6565

6666
assert(store.getSessionList.size == 3)
6767
assert(store.getSessionList(2).sessionId == "s4")
6868
}
6969

7070
test("test check session after update session") {
7171
val store = new EngineEventsStore(KyuubiConf())
72-
val s = SessionEvent("abc", "ea", "test3", "1.1.1.1", 2L)
72+
val s = SessionEvent("abc", "ea", "test3", "1.1.1.1", "1.1.1.2", 2L)
7373
store.saveSession(s)
7474

7575
val finishTimestamp: Long = 456L

0 commit comments

Comments
 (0)