-
Notifications
You must be signed in to change notification settings - Fork 903
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[KYUUBI #3098] Unify the event log code path
### _Why are the changes needed?_ close #3098 (comment) ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #3090 from zhaomin1423/event_spi. Closes #3098 322e48d [Min] clear unnecessary variable 22b448d [Min] fix review a452321 [Min] [KYUUBI #3098] Unify the event log code path Authored-by: Min <zhaomin1423@163.com> Signed-off-by: ulysses-you <ulyssesyou@apache.org>
- Loading branch information
1 parent
6aa898e
commit d086525
Showing
9 changed files
with
257 additions
and
92 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
...engine/src/main/scala/org/apache/kyuubi/engine/hive/events/HiveEventHandlerRegister.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kyuubi.engine.hive.events | ||
|
|
||
| import java.net.InetAddress | ||
|
|
||
| import org.apache.kyuubi.config.KyuubiConf | ||
| import org.apache.kyuubi.config.KyuubiConf.ENGINE_EVENT_JSON_LOG_PATH | ||
| import org.apache.kyuubi.engine.hive.events.handler.HiveJsonLoggingEventHandler | ||
| import org.apache.kyuubi.events.{EventHandlerRegister, KyuubiEvent} | ||
| import org.apache.kyuubi.events.handler.EventHandler | ||
| import org.apache.kyuubi.util.KyuubiHadoopUtils | ||
|
|
||
| object HiveEventHandlerRegister extends EventHandlerRegister { | ||
|
|
||
| override protected def createJsonEventHandler( | ||
| kyuubiConf: KyuubiConf): EventHandler[KyuubiEvent] = { | ||
| val hadoopConf = KyuubiHadoopUtils.newHadoopConf(kyuubiConf) | ||
| val hostName = InetAddress.getLocalHost.getCanonicalHostName | ||
| HiveJsonLoggingEventHandler( | ||
| s"Hive-$hostName", | ||
| ENGINE_EVENT_JSON_LOG_PATH, | ||
| hadoopConf, | ||
| kyuubiConf) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
45 changes: 45 additions & 0 deletions
45
...gine/src/main/scala/org/apache/kyuubi/engine/spark/events/SparkEventHandlerRegister.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kyuubi.engine.spark.events | ||
|
|
||
| import org.apache.spark.sql.SparkSession | ||
|
|
||
| import org.apache.kyuubi.config.KyuubiConf | ||
| import org.apache.kyuubi.config.KyuubiConf.ENGINE_EVENT_JSON_LOG_PATH | ||
| import org.apache.kyuubi.engine.spark.events.handler.{SparkHistoryLoggingEventHandler, SparkJsonLoggingEventHandler} | ||
| import org.apache.kyuubi.events.{EventHandlerRegister, KyuubiEvent} | ||
| import org.apache.kyuubi.events.handler.EventHandler | ||
|
|
||
| class SparkEventHandlerRegister(spark: SparkSession) extends EventHandlerRegister { | ||
|
|
||
| override protected def createSparkEventHandler(kyuubiConf: KyuubiConf) | ||
| : EventHandler[KyuubiEvent] = { | ||
| new SparkHistoryLoggingEventHandler(spark.sparkContext) | ||
| } | ||
|
|
||
| override protected def createJsonEventHandler(kyuubiConf: KyuubiConf) | ||
| : EventHandler[KyuubiEvent] = { | ||
| SparkJsonLoggingEventHandler( | ||
| spark.sparkContext.applicationAttemptId | ||
| .map(id => s"${spark.sparkContext.applicationId}_$id") | ||
| .getOrElse(spark.sparkContext.applicationId), | ||
| ENGINE_EVENT_JSON_LOG_PATH, | ||
| spark.sparkContext.hadoopConfiguration, | ||
| kyuubiConf) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
...ngine/src/main/scala/org/apache/kyuubi/engine/trino/event/TrinoEventHandlerRegister.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kyuubi.engine.trino.event | ||
|
|
||
| import java.net.InetAddress | ||
|
|
||
| import org.apache.kyuubi.config.KyuubiConf | ||
| import org.apache.kyuubi.config.KyuubiConf.ENGINE_EVENT_JSON_LOG_PATH | ||
| import org.apache.kyuubi.engine.trino.event.handler.TrinoJsonLoggingEventHandler | ||
| import org.apache.kyuubi.events.{EventHandlerRegister, KyuubiEvent} | ||
| import org.apache.kyuubi.events.handler.EventHandler | ||
| import org.apache.kyuubi.util.KyuubiHadoopUtils | ||
|
|
||
| object TrinoEventHandlerRegister extends EventHandlerRegister { | ||
|
|
||
| override protected def createJsonEventHandler(kyuubiConf: KyuubiConf) | ||
| : EventHandler[KyuubiEvent] = { | ||
| val hadoopConf = KyuubiHadoopUtils.newHadoopConf(kyuubiConf) | ||
| val hostName = InetAddress.getLocalHost.getCanonicalHostName | ||
| TrinoJsonLoggingEventHandler( | ||
| s"Trino-$hostName", | ||
| ENGINE_EVENT_JSON_LOG_PATH, | ||
| hadoopConf, | ||
| kyuubiConf) | ||
| } | ||
| } |
74 changes: 74 additions & 0 deletions
74
kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventHandlerRegister.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kyuubi.events | ||
|
|
||
| import org.apache.kyuubi.{KyuubiException, Logging} | ||
| import org.apache.kyuubi.config.KyuubiConf | ||
| import org.apache.kyuubi.config.KyuubiConf.{ENGINE_EVENT_LOGGERS, SERVER_EVENT_LOGGERS} | ||
| import org.apache.kyuubi.events.EventLoggerType.EventLoggerType | ||
| import org.apache.kyuubi.events.handler.EventHandler | ||
|
|
||
| trait EventHandlerRegister extends Logging { | ||
|
|
||
| def registerEngineEventLoggers(conf: KyuubiConf): Unit = { | ||
| val loggers = conf.get(ENGINE_EVENT_LOGGERS) | ||
| register(loggers, conf) | ||
| } | ||
|
|
||
| def registerServerEventLoggers(conf: KyuubiConf): Unit = { | ||
| val loggers = conf.get(SERVER_EVENT_LOGGERS) | ||
| register(loggers, conf) | ||
| } | ||
|
|
||
| private def register(loggers: Seq[String], conf: KyuubiConf): Unit = { | ||
| loggers | ||
| .map(EventLoggerType.withName) | ||
| .foreach { logger => | ||
| EventBus.register(loadEventHandler(logger, conf)) | ||
| } | ||
| } | ||
|
|
||
| protected def createSparkEventHandler(kyuubiConf: KyuubiConf): EventHandler[KyuubiEvent] = { | ||
| throw new KyuubiException(s"Unsupported spark event logger.") | ||
| } | ||
|
|
||
| protected def createJsonEventHandler(kyuubiConf: KyuubiConf): EventHandler[KyuubiEvent] = { | ||
| throw new KyuubiException(s"Unsupported json event logger.") | ||
| } | ||
|
|
||
| protected def createJdbcEventHandler(kyuubiConf: KyuubiConf): EventHandler[KyuubiEvent] = { | ||
| throw new KyuubiException(s"Unsupported jdbc event logger.") | ||
| } | ||
|
|
||
| private def loadEventHandler( | ||
| eventLoggerType: EventLoggerType, | ||
| kyuubiConf: KyuubiConf): EventHandler[KyuubiEvent] = { | ||
| eventLoggerType match { | ||
| case EventLoggerType.SPARK => | ||
| createSparkEventHandler(kyuubiConf) | ||
|
|
||
| case EventLoggerType.JSON => | ||
| createJsonEventHandler(kyuubiConf) | ||
|
|
||
| case EventLoggerType.JDBC => | ||
| createJdbcEventHandler(kyuubiConf) | ||
|
|
||
| case other => | ||
| throw new KyuubiException(s"Unsupported event logger: ${other.toString}") | ||
| } | ||
| } | ||
| } |
38 changes: 38 additions & 0 deletions
38
kyuubi-server/src/main/scala/org/apache/kyuubi/events/ServerEventHandlerRegister.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.kyuubi.events | ||
|
|
||
| import java.net.InetAddress | ||
|
|
||
| import org.apache.kyuubi.config.KyuubiConf | ||
| import org.apache.kyuubi.config.KyuubiConf.SERVER_EVENT_JSON_LOG_PATH | ||
| import org.apache.kyuubi.events.handler.{EventHandler, ServerJsonLoggingEventHandler} | ||
| import org.apache.kyuubi.util.KyuubiHadoopUtils | ||
|
|
||
| object ServerEventHandlerRegister extends EventHandlerRegister { | ||
|
|
||
| override protected def createJsonEventHandler(kyuubiConf: KyuubiConf) | ||
| : EventHandler[KyuubiEvent] = { | ||
| val hadoopConf = KyuubiHadoopUtils.newHadoopConf(kyuubiConf) | ||
| val hostName = InetAddress.getLocalHost.getCanonicalHostName | ||
| ServerJsonLoggingEventHandler( | ||
| s"server-$hostName", | ||
| SERVER_EVENT_JSON_LOG_PATH, | ||
| hadoopConf, | ||
| kyuubiConf) | ||
| } | ||
| } |
Oops, something went wrong.