-
Notifications
You must be signed in to change notification settings - Fork 28.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
SPARK-1729. Update logging in Spark Sink.
- Loading branch information
1 parent
8c00289
commit 1edc806
Showing
6 changed files
with
163 additions
and
23 deletions.
There are no files selected for viewing
135 changes: 135 additions & 0 deletions
135
external/flume-sink/src/main/scala/org/apache/spark/flume/sink/Logging.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.spark.flume.sink | ||
| import org.apache.log4j.{LogManager, PropertyConfigurator} | ||
| import org.slf4j.{Logger, LoggerFactory} | ||
| import org.slf4j.impl.StaticLoggerBinder | ||
|
|
||
| trait Logging { | ||
| // Make the log field transient so that objects with Logging can | ||
| // be serialized and used on another machine | ||
| @transient private var log_ : Logger = null | ||
|
|
||
| // Method to get or create the logger for this object | ||
| protected def log: Logger = { | ||
| if (log_ == null) { | ||
| initializeIfNecessary() | ||
| var className = this.getClass.getName | ||
| // Ignore trailing $'s in the class names for Scala objects | ||
| if (className.endsWith("$")) { | ||
| className = className.substring(0, className.length - 1) | ||
| } | ||
| log_ = LoggerFactory.getLogger(className) | ||
| } | ||
| log_ | ||
| } | ||
|
|
||
| // Log methods that take only a String | ||
| protected def logInfo(msg: => String) { | ||
| if (log.isInfoEnabled) log.info(msg) | ||
| } | ||
|
|
||
| protected def logDebug(msg: => String) { | ||
| if (log.isDebugEnabled) log.debug(msg) | ||
| } | ||
|
|
||
| protected def logTrace(msg: => String) { | ||
| if (log.isTraceEnabled) log.trace(msg) | ||
| } | ||
|
|
||
| protected def logWarning(msg: => String) { | ||
| if (log.isWarnEnabled) log.warn(msg) | ||
| } | ||
|
|
||
| protected def logError(msg: => String) { | ||
| if (log.isErrorEnabled) log.error(msg) | ||
| } | ||
|
|
||
| // Log methods that take Throwables (Exceptions/Errors) too | ||
| protected def logInfo(msg: => String, throwable: Throwable) { | ||
| if (log.isInfoEnabled) log.info(msg, throwable) | ||
| } | ||
|
|
||
| protected def logDebug(msg: => String, throwable: Throwable) { | ||
| if (log.isDebugEnabled) log.debug(msg, throwable) | ||
| } | ||
|
|
||
| protected def logTrace(msg: => String, throwable: Throwable) { | ||
| if (log.isTraceEnabled) log.trace(msg, throwable) | ||
| } | ||
|
|
||
| protected def logWarning(msg: => String, throwable: Throwable) { | ||
| if (log.isWarnEnabled) log.warn(msg, throwable) | ||
| } | ||
|
|
||
| protected def logError(msg: => String, throwable: Throwable) { | ||
| if (log.isErrorEnabled) log.error(msg, throwable) | ||
| } | ||
|
|
||
| protected def isTraceEnabled(): Boolean = { | ||
| log.isTraceEnabled | ||
| } | ||
|
|
||
| private def initializeIfNecessary() { | ||
| if (!Logging.initialized) { | ||
| Logging.initLock.synchronized { | ||
| if (!Logging.initialized) { | ||
| initializeLogging() | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private def initializeLogging() { | ||
| // If Log4j is being used, but is not initialized, load a default properties file | ||
| val binder = StaticLoggerBinder.getSingleton | ||
| val usingLog4j = binder.getLoggerFactoryClassStr.endsWith("Log4jLoggerFactory") | ||
| val log4jInitialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements | ||
| if (!log4jInitialized && usingLog4j) { | ||
| val defaultLogProps = "org/apache/spark/log4j-defaults.properties" | ||
| Option(getClass.getClassLoader.getResource(defaultLogProps)) match { | ||
| case Some(url) => | ||
| PropertyConfigurator.configure(url) | ||
| log.info(s"Using Spark's default log4j profile: $defaultLogProps") | ||
| case None => | ||
| System.err.println(s"Spark was unable to load $defaultLogProps") | ||
| } | ||
| } | ||
| Logging.initialized = true | ||
|
|
||
| // Force a call into slf4j to initialize it. Avoids this happening from mutliple threads | ||
| // and triggering this: http://mailman.qos.ch/pipermail/slf4j-dev/2010-April/002956.html | ||
| log | ||
| } | ||
| } | ||
|
|
||
| private object Logging { | ||
| @volatile private var initialized = false | ||
| val initLock = new Object() | ||
| try { | ||
| // We use reflection here to handle the case where users remove the | ||
| // slf4j-to-jul bridge order to route their logs to JUL. | ||
| val bridgeClass = Class.forName("org.slf4j.bridge.SLF4JBridgeHandler") | ||
| bridgeClass.getMethod("removeHandlersForRootLogger").invoke(null) | ||
| val installed = bridgeClass.getMethod("isInstalled").invoke(null).asInstanceOf[Boolean] | ||
| if (!installed) { | ||
| bridgeClass.getMethod("install").invoke(null) | ||
| } | ||
| } catch { | ||
| case e: ClassNotFoundException => // can't log anything yet so just fail silently | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters