From 12d896c9d9b326c8539d11376c1d28187a343057 Mon Sep 17 00:00:00 2001 From: mihir6692 Date: Tue, 26 Apr 2016 13:55:45 +0530 Subject: [PATCH 1/4] [SPARK-14754][CORE]Metrics as logs are not coming through slf4j --- .../main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala index 773e074336cb0..ee32d30636c96 100644 --- a/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala +++ b/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala @@ -25,6 +25,9 @@ import com.codahale.metrics.{MetricRegistry, Slf4jReporter} import org.apache.spark.SecurityManager import org.apache.spark.metrics.MetricsSystem +import org.slf4j.Logger +import org.slf4j.LoggerFactory + private[spark] class Slf4jSink( val property: Properties, val registry: MetricRegistry, @@ -49,6 +52,7 @@ private[spark] class Slf4jSink( MetricsSystem.checkMinimalPollingPeriod(pollUnit, pollPeriod) val reporter: Slf4jReporter = Slf4jReporter.forRegistry(registry) + .outputTo(LoggerFactory.getLogger("org.apache.spark.metrics")) .convertDurationsTo(TimeUnit.MILLISECONDS) .convertRatesTo(TimeUnit.SECONDS) .build() From 5a437a12fb0c6cfdd7119d63bebc3f3f2c935d5f Mon Sep 17 00:00:00 2001 From: mihir6692 Date: Tue, 26 Apr 2016 13:57:26 +0530 Subject: [PATCH 2/4] [SPARK-14754][CORE]Metrics as logs are not coming through slf4j --- conf/log4j.properties.template | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/conf/log4j.properties.template b/conf/log4j.properties.template index ec1aa187dfb32..ba97bf1b7d53b 100644 --- a/conf/log4j.properties.template +++ b/conf/log4j.properties.template @@ -38,3 +38,14 @@ log4j.logger.parquet=ERROR # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR + +# SPARK-14754: Metrics as logs are not coming through slf4j. +#log4j.logger.org.apache.spark.metrics=INFO, metricFileAppender +#log4j.additivity.org.apache.spark.metrics=true + +#log4j.appender.metricFileAppender=org.apache.log4j.RollingFileAppender +#log4j.appender.metricFileAppender.File=${logFilePath} +#log4j.appender.metricFileAppender.MaxFileSize=10MB +#log4j.appender.metricFileAppender.MaxBackupIndex=10 +#log4j.appender.metricFileAppender.layout=org.apache.log4j.PatternLayout +#log4j.appender.metricFileAppender.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n From 032422b66b60b558a5830a414f5e6a5948d560da Mon Sep 17 00:00:00 2001 From: mihir6692 Date: Tue, 26 Apr 2016 15:48:29 +0530 Subject: [PATCH 3/4] [SPARK-14754][CORE]Metrics as logs are not coming through slf4j Updated import style as per conventions --- .../scala/org/apache/spark/metrics/sink/Slf4jSink.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala index ee32d30636c96..8b1398face1ba 100644 --- a/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala +++ b/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala @@ -22,12 +22,12 @@ import java.util.concurrent.TimeUnit import com.codahale.metrics.{MetricRegistry, Slf4jReporter} -import org.apache.spark.SecurityManager -import org.apache.spark.metrics.MetricsSystem - import org.slf4j.Logger import org.slf4j.LoggerFactory +import org.apache.spark.SecurityManager +import org.apache.spark.metrics.MetricsSystem + private[spark] class Slf4jSink( val property: Properties, val registry: MetricRegistry, From 7f961e8a11fa26ab95848325bc61c09b12a3747d Mon Sep 17 00:00:00 2001 From: mihir6692 Date: Tue, 26 Apr 2016 15:53:42 +0530 Subject: [PATCH 4/4] [SPARK-14754][CORE]Metrics as logs are not coming through slf4j Update class name for logger --- .../main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala index 8b1398face1ba..40338acf89bf1 100644 --- a/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala +++ b/core/src/main/scala/org/apache/spark/metrics/sink/Slf4jSink.scala @@ -52,7 +52,7 @@ private[spark] class Slf4jSink( MetricsSystem.checkMinimalPollingPeriod(pollUnit, pollPeriod) val reporter: Slf4jReporter = Slf4jReporter.forRegistry(registry) - .outputTo(LoggerFactory.getLogger("org.apache.spark.metrics")) + .outputTo(LoggerFactory.getLogger("org.apache.spark.metrics.sink.Slf4jSink")) .convertDurationsTo(TimeUnit.MILLISECONDS) .convertRatesTo(TimeUnit.SECONDS) .build()