From 46f830122aaa4d1f153d2ae5ef66602d81f518cd Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Sun, 22 Jan 2017 12:16:15 +0800 Subject: [PATCH 1/2] fix message format --- .../jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java | 4 ++++ .../jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java | 3 +-- .../java/org/apache/eagle/jpm/analyzer/publisher/Result.java | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java index 103de7a4a5..4007747543 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java @@ -20,6 +20,7 @@ import org.apache.eagle.jpm.analyzer.Processor; import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.util.Constants; import org.apache.eagle.jpm.util.jobcounter.JobCounters; import java.util.ArrayList; @@ -45,9 +46,11 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) long mapGCTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS); long mapCPUTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS); + String message = "%s GC_MILLISECONDS is: %s, and CPU_MILLISECONDS is: %s. "; if (mapGCTime > mapCPUTime * 0.1) { setting = String.format("-D%s", MAP_JAVA_OPTS); optSettings.add(setting); + sb.append(String.format(message, Constants.TaskType.MAP, mapGCTime, mapCPUTime)); sb.append("Map GC_TIME_MILLIS took too long. Please increase mapper memory via ").append(setting); sb.append(", or optimize your mapper class.\n"); } @@ -58,6 +61,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) if (reduceGCTime > reduceCPUTime * 0.1) { setting = String.format("-D%s", REDUCE_JAVA_OPTS); optSettings.add(setting); + sb.append(String.format(message, Constants.TaskType.REDUCE, reduceGCTime, reduceCPUTime)); sb.append("Reduce GC_TIME_MILLIS took too long. Please increase memory for reduce via ").append(setting); sb.append(", or optimize your reducer class.\n"); } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java index 835b382ac9..96be2d5033 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java @@ -82,7 +82,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) long heapSize = getMaxHeapSize(context.getJobconf().get(MAP_JAVA_OPTS)); if (heapSize < 3 * minMapSpillMemMB) { long expectedHeapSizeMB = (minMapSpillMemMB * 3 + 1024) / 1024 * 1024; - setting = String.format(" -D%s=-Xmx%sM", MAP_JAVA_OPTS, expectedHeapSizeMB); + setting = String.format("-D%s=-Xmx%sM", MAP_JAVA_OPTS, expectedHeapSizeMB); sb.append(" ").append(setting); optSettings.add(setting); } @@ -90,7 +90,6 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) sb.append(" to avoid spilled records.\n"); } - long reduceInputRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS); spillRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS); if (reduceInputRecords < spillRecords) { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java index a9f51324fb..7d7442b6ae 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java @@ -63,9 +63,11 @@ public Map> getAlertEntities() { } private void normalizeResult(ProcessorResult processorResult) { + String settingList = ""; if (processorResult.getSettings() != null && !processorResult.getSettings().isEmpty()) { - processorResult.setSettingList(StringUtils.join(processorResult.getSettings(), "\n")); + settingList = StringUtils.join(processorResult.getSettings(), "\n"); } + processorResult.setSettingList(settingList); } /** From bbd33bf28713c5bc853c4ca76ed1dadaab10b057 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Sun, 22 Jan 2017 14:08:45 +0800 Subject: [PATCH 2/2] disable job performance email publisher --- .../apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index e32a37c0f4..57e1765d19 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -48,7 +48,7 @@ public MRJobPerformanceAnalyzer(Config config) { evaluators.add(new JobSuggestionEvaluator(config)); publishers.add(new EagleStorePublisher(config)); - publishers.add(new EmailPublisher(config)); + //publishers.add(new EmailPublisher(config)); } @Override