From 89ba5befd38615c1b35b15f75b610fa0a1e978f0 Mon Sep 17 00:00:00 2001 From: Qingwen Zhao Date: Thu, 12 Jan 2017 11:00:12 +0800 Subject: [PATCH 01/22] rename method analysis to analyze --- .../java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java | 6 +++--- .../eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java | 2 +- .../org/apache/eagle/jpm/mr/running/parser/MRJobParser.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java index 6cda1cdc26..7cc5c2e44a 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java @@ -18,11 +18,11 @@ package org.apache.eagle.jpm.analyzer; /** - * Each JobAnalyzer contains one or more Evaluators to analysis each job. + * Each JobAnalyzer contains one or more Evaluators to analyze each job. * Each Evaluator is a group of Processors - * Each Processor implements an algorithm or a model to analysis one dimension of a job + * Each Processor implements an algorithm or a model to analyze one dimension of a job * */ public interface JobAnalyzer { - void analysis(AnalyzerEntity analyzerEntity) throws Exception; + void analyze(AnalyzerEntity analyzerEntity) throws Exception; } \ No newline at end of file diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index e0e579a951..e6d0d09099 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -51,7 +51,7 @@ public MRJobPerformanceAnalyzer(Config config) { } @Override - public void analysis(AnalyzerEntity analyzerJobEntity) throws Exception { + public void analyze(AnalyzerEntity analyzerJobEntity) throws Exception { Result result = new Result(); for (Evaluator evaluator : evaluators) { diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java index c21eaf1540..0a8bcf38fe 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java @@ -173,7 +173,7 @@ private void fetchMRRunningInfo() throws Exception { break; } } - mrJobPerformanceAnalyzer.analysis(convertToAnalysisEntity(mrJobEntityMap.get(jobId))); + mrJobPerformanceAnalyzer.analyze(convertToAnalysisEntity(mrJobEntityMap.get(jobId))); } } From f085fce32f4c42b7031bf455e42fdaa7604b6bf8 Mon Sep 17 00:00:00 2001 From: Qingwen Zhao Date: Thu, 12 Jan 2017 14:41:27 +0800 Subject: [PATCH 02/22] fix a typo --- eagle-jpm/eagle-jpm-mr-history/pom.xml | 5 +++++ .../eagle/jpm/mr/history/parser/JHFEventReaderBase.java | 2 +- .../jpm/mr/history/parser/JobEntityCreationPublisher.java | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/eagle-jpm/eagle-jpm-mr-history/pom.xml b/eagle-jpm/eagle-jpm-mr-history/pom.xml index 349c489c91..2e3a1a82cc 100644 --- a/eagle-jpm/eagle-jpm-mr-history/pom.xml +++ b/eagle-jpm/eagle-jpm-mr-history/pom.xml @@ -89,6 +89,11 @@ 1.6 test + + org.apache.eagle + eagle-jpm-analyzer + ${project.version} + diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index d89937e937..5d8d1ba8bc 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -332,7 +332,7 @@ private void entityCreated(JobBaseAPIEntity entity) throws Exception { } } - super.notifiyListeners(entity); + super.notifyListeners(entity); } protected abstract JobCounters parseCounters(Object value) throws IOException; diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationPublisher.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationPublisher.java index a80462d912..f2730fdc1c 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationPublisher.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationPublisher.java @@ -29,7 +29,7 @@ public void addListener(HistoryJobEntityCreationListener l) { listeners.add(l); } - public void notifiyListeners(JobBaseAPIEntity entity) throws Exception { + public void notifyListeners(JobBaseAPIEntity entity) throws Exception { for (HistoryJobEntityCreationListener l : listeners) { l.jobEntityCreated(entity); } From 0d95e1f02b09e18c16795681742fbad72430a826 Mon Sep 17 00:00:00 2001 From: Qingwen Zhao Date: Thu, 12 Jan 2017 15:47:27 +0800 Subject: [PATCH 03/22] add job suggestion --- .../eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java | 5 ++++- .../apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java | 5 +++++ .../jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java | 5 +++++ .../apache/eagle/jpm/mr/history/parser/JHFParserFactory.java | 2 ++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index e6d0d09099..b772245970 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -55,7 +55,10 @@ public void analyze(AnalyzerEntity analyzerJobEntity) throws Exception { Result result = new Result(); for (Evaluator evaluator : evaluators) { - result.addEvaluatorResult(evaluator.getClass(), evaluator.evaluate(analyzerJobEntity)); + Result.EvaluatorResult evaluatorResult = evaluator.evaluate(analyzerJobEntity); + if (evaluatorResult != null) { + result.addEvaluatorResult(evaluator.getClass(), evaluatorResult); + } } for (Publisher publisher : publishers) { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java index f10b68d3e7..f8c2dcb26a 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java @@ -26,6 +26,7 @@ import org.apache.eagle.jpm.analyzer.Processor; import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.analyzer.util.Utils; +import org.apache.eagle.jpm.util.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,10 @@ public SLAJobEvaluator(Config config) { @Override public Result.EvaluatorResult evaluate(AnalyzerEntity analyzerJobEntity) { + if (!analyzerJobEntity.getCurrentState().equalsIgnoreCase(Constants.JobState.RUNNING.toString())) { + return null; + } + Result.EvaluatorResult result = new Result.EvaluatorResult(); List jobMetaEntities = Utils.getJobMeta(config, analyzerJobEntity.getJobDefId()); diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java index 79f5318d7e..4f088dc9cb 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java @@ -21,6 +21,7 @@ import org.apache.eagle.jpm.analyzer.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.Evaluator; import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.util.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +38,10 @@ public JobSuggestionEvaluator(Config config) { @Override public Result.EvaluatorResult evaluate(AnalyzerEntity mrJobEntity) { + if (mrJobEntity.getCurrentState().equalsIgnoreCase(Constants.JobState.RUNNING.toString())) { + return null; + } + Result.EvaluatorResult result = new Result.EvaluatorResult(); //TODO return result; diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFParserFactory.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFParserFactory.java index ca49d9cdc3..1d176405db 100755 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFParserFactory.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFParserFactory.java @@ -38,11 +38,13 @@ public static JHFParserBase getParser(Map baseTags, MRHistoryJobConfig.EagleServiceConfig eagleServiceConfig = appConfig.getEagleServiceConfig(); JHFMRVer2EventReader reader2 = new JHFMRVer2EventReader(baseTags, configuration, filter, appConfig); + // add HistoryJobEntityCreationListener reader2.addListener(new JobEntityCreationEagleServiceListener(appConfig)); reader2.addListener(new TaskFailureListener(eagleServiceConfig)); reader2.addListener(new TaskAttemptCounterListener(eagleServiceConfig)); reader2.addListener(new JobConfigurationCreationServiceListener(eagleServiceConfig)); + // add HistoryJobEntityLifecycleListener reader2.register(new JobEntityLifecycleAggregator()); JHFParserBase parser = new JHFMRVer2Parser(reader2); return parser; From 69b9646fbdd9e7c4bacf461e9a72305a391e375d Mon Sep 17 00:00:00 2001 From: Qingwen Zhao Date: Fri, 13 Jan 2017 21:08:12 +0800 Subject: [PATCH 04/22] add JobSuggestionListner --- .../eagle/jpm/analyzer/AnalyzerEntity.java | 6 +- .../eagle/jpm/analyzer/JobAnalyzer.java | 4 +- .../analyzer/mr/MRJobPerformanceAnalyzer.java | 5 +- .../analyzer/publisher/EmailPublisher.java | 2 - .../TaskAttemptExecutionAPIEntity.java | 46 ++++- .../DefaultJHFInputStreamCallback.java | 3 +- .../mr/history/parser/JHFEventReaderBase.java | 30 +++- .../jpm/mr/history/parser/JHFFormat.java | 1 + .../history/parser/JHFMRVer2EventReader.java | 11 +- .../history/parser/JobSuggestionListener.java | 170 ++++++++++++++++++ 10 files changed, 254 insertions(+), 24 deletions(-) create mode 100644 eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java index f9b7af0d2b..98e0f97ed1 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java @@ -5,9 +5,9 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java index 7cc5c2e44a..b0955ebfd0 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java @@ -23,6 +23,6 @@ * Each Processor implements an algorithm or a model to analyze one dimension of a job * */ -public interface JobAnalyzer { - void analyze(AnalyzerEntity analyzerEntity) throws Exception; +public interface JobAnalyzer { + void analyze(T analyzerEntity) throws Exception; } \ No newline at end of file diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index b772245970..54f12d621c 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -20,6 +20,7 @@ import com.typesafe.config.Config; import org.apache.eagle.jpm.analyzer.*; import org.apache.eagle.jpm.analyzer.Evaluator; +import org.apache.eagle.jpm.analyzer.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.mr.sla.SLAJobEvaluator; import org.apache.eagle.jpm.analyzer.mr.suggestion.JobSuggestionEvaluator; import org.apache.eagle.jpm.analyzer.publisher.EagleStorePublisher; @@ -33,7 +34,7 @@ import java.util.ArrayList; import java.util.List; -public class MRJobPerformanceAnalyzer implements JobAnalyzer, Serializable { +public class MRJobPerformanceAnalyzer implements JobAnalyzer, Serializable { private static final Logger LOG = LoggerFactory.getLogger(MRJobPerformanceAnalyzer.class); private List evaluators = new ArrayList<>(); @@ -51,7 +52,7 @@ public MRJobPerformanceAnalyzer(Config config) { } @Override - public void analyze(AnalyzerEntity analyzerJobEntity) throws Exception { + public void analyze(T analyzerJobEntity) throws Exception { Result result = new Result(); for (Evaluator evaluator : evaluators) { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java index 4e490942b5..9b6543384a 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java @@ -21,7 +21,6 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.eagle.app.service.ApplicationEmailService; import org.apache.eagle.common.DateTimeUtil; -import org.apache.eagle.common.mail.AlertEmailConstants; import org.apache.eagle.common.mail.AlertEmailContext; import org.apache.eagle.jpm.analyzer.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.dedup.AlertDeduplicator; @@ -31,7 +30,6 @@ import org.slf4j.LoggerFactory; import java.io.Serializable; -import java.net.URLEncoder; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java index c284341df3..7b0256e069 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java @@ -46,6 +46,40 @@ public class TaskAttemptExecutionAPIEntity extends JobBaseAPIEntity { private String error; @Column("f") private JobCounters jobCounters; + // new added + @Column("g") + private long shuffleFinishTime; + @Column("h") + private long sortFinishTime; + @Column("i") + private long mapFinishTime; + + public long getShuffleFinishTime() { + return shuffleFinishTime; + } + + public void setShuffleFinishTime(long shuffleFinishTime) { + this.shuffleFinishTime = shuffleFinishTime; + valueChanged("shuffleFinishTime"); + } + + public long getSortFinishTime() { + return sortFinishTime; + } + + public void setSortFinishTime(long sortFinishTime) { + this.sortFinishTime = sortFinishTime; + valueChanged("sortFinishTime"); + } + + public long getMapFinishTime() { + return mapFinishTime; + } + + public void setMapFinishTime(long mapFinishTime) { + this.mapFinishTime = mapFinishTime; + valueChanged("mapFinishTime"); + } public String getTaskStatus() { return taskStatus; @@ -53,7 +87,7 @@ public String getTaskStatus() { public void setTaskStatus(String taskStatus) { this.taskStatus = taskStatus; - pcs.firePropertyChange("taskStatus", null, null); + valueChanged("taskStatus"); } public long getStartTime() { @@ -62,7 +96,7 @@ public long getStartTime() { public void setStartTime(long startTime) { this.startTime = startTime; - pcs.firePropertyChange("startTime", null, null); + valueChanged("startTime"); } public long getEndTime() { @@ -71,7 +105,7 @@ public long getEndTime() { public void setEndTime(long endTime) { this.endTime = endTime; - pcs.firePropertyChange("endTime", null, null); + valueChanged("endTime"); } public long getDuration() { @@ -80,7 +114,7 @@ public long getDuration() { public void setDuration(long duration) { this.duration = duration; - pcs.firePropertyChange("duration", null, null); + valueChanged("duration"); } public String getError() { @@ -89,7 +123,7 @@ public String getError() { public void setError(String error) { this.error = error; - pcs.firePropertyChange("error", null, null); + valueChanged("error"); } public JobCounters getJobCounters() { @@ -98,6 +132,6 @@ public JobCounters getJobCounters() { public void setJobCounters(JobCounters jobCounters) { this.jobCounters = jobCounters; - pcs.firePropertyChange("jobCounters", null, null); + valueChanged("jobCounters"); } } diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/DefaultJHFInputStreamCallback.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/DefaultJHFInputStreamCallback.java index 262054e9f6..28ebf4efe0 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/DefaultJHFInputStreamCallback.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/DefaultJHFInputStreamCallback.java @@ -21,6 +21,7 @@ import org.apache.eagle.jpm.mr.history.MRHistoryJobConfig; import org.apache.eagle.jpm.mr.history.parser.JHFParserBase; import org.apache.eagle.jpm.mr.history.parser.JHFParserFactory; +import org.apache.eagle.jpm.util.MRJobTagName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +47,7 @@ public void onInputStream(InputStream jobFileInputStream, org.apache.hadoop.conf @SuppressWarnings("serial") Map baseTags = new HashMap() { { - put("site", appConfig.getJobHistoryEndpointConfig().site); + put(MRJobTagName.SITE.toString(), appConfig.getJobHistoryEndpointConfig().site); } }; diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index 5d8d1ba8bc..c15bb31c1a 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -81,6 +81,7 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl private long sumReduceTaskDuration; private JobCounterMetricsGenerator jobCounterMetricsGenerator; + private JobSuggestionListener jobSuggestionListener; private MRHistoryJobConfig appConfig; @@ -127,6 +128,10 @@ public JHFEventReaderBase(Map baseTags, Configuration configurat this.appConfig = appConfig; this.jobCounterMetricsGenerator = new JobCounterMetricsGenerator(appConfig.getEagleServiceConfig()); + this.jobSuggestionListener = new JobSuggestionListener(); + if (this.configuration != null) { + this.jobSuggestionListener.jobConfigCreated(configuration); + } } public void register(HistoryJobEntityLifecycleListener lifecycleListener) { @@ -179,7 +184,7 @@ private void setJobType(String jobType) { this.jobType = jobType; } - protected void handleJob(EventType eventType, Map values, Object totalCounters) throws Exception { + protected void handleJob(EventType eventType, Map values, Object totalCounters, Object mapCounters, Object reduceCounters) throws Exception { String id = values.get(Keys.JOBID); if (jobId == null) { @@ -300,8 +305,15 @@ protected void handleJob(EventType eventType, Map values, Object t this.jobCounterMetricsGenerator.setBaseTags(jobExecutionEntity.getTags()); formatDiagnostics(values.get(Keys.DIAGNOSTICS)); - entityCreated(jobExecutionEntity); + jobSuggestionListener.jobEntityCreated(jobExecutionEntity); + + if (configuration != null && totalCounters != null) { + JobCounters parsedTotalCounters = parseCounters(totalCounters); + JobCounters parsedMapCounters = parseCounters(mapCounters); + JobCounters parsedReduceCounters = parseCounters(reduceCounters); + jobSuggestionListener.jobCountersCreated(parsedTotalCounters, parsedMapCounters, parsedReduceCounters); + } } } @@ -415,6 +427,7 @@ protected void handleTask(RecordTypes recType, EventType eventType, final Map> groups = new HashMap<>(); JhCounters counters = (JhCounters) value; diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java new file mode 100644 index 0000000000..a7cca181c2 --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.mr.history.parser; + +import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.mr.historyentity.JobBaseAPIEntity; +import org.apache.eagle.jpm.mr.historyentity.JobExecutionAPIEntity; +import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; +import org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity; +import org.apache.eagle.jpm.util.MRJobTagName; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; +import org.apache.eagle.jpm.mr.history.parser.JHFEventReaderBase.Keys; +import org.apache.hadoop.conf.Configuration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * JobEventCounterListener provides an interface to add job/task counter analyzers + */ +public class JobSuggestionListener implements HistoryJobEntityCreationListener { + private static final Logger LOG = LoggerFactory.getLogger(JobSuggestionListener.class); + + private MapReduceAnalyzerEntity info = new MapReduceAnalyzerEntity(); + private Configuration jobConf; + + @Override + public void jobEntityCreated(JobBaseAPIEntity entity) throws Exception { + if (entity instanceof TaskExecutionAPIEntity) { + info.getTasksMap().put(entity.getTags().get(Keys.TASKID), (TaskExecutionAPIEntity) entity); + } else if (entity instanceof TaskAttemptExecutionAPIEntity) { + info.getCompletedTaskAttemptsMap().put(entity.getTags().get(Keys.TASK_ATTEMPT_ID), (TaskAttemptExecutionAPIEntity) entity); + } else if (entity instanceof JobExecutionAPIEntity) { + JobExecutionAPIEntity jobExecutionAPIEntity = (JobExecutionAPIEntity) entity; + info.setCurrentState(jobExecutionAPIEntity.getCurrentState()); + info.setStartTime(jobExecutionAPIEntity.getStartTime()); + info.setEndTime(jobExecutionAPIEntity.getEndTime()); + info.setDurationTime(jobExecutionAPIEntity.getDurationTime()); + info.setUserId(jobExecutionAPIEntity.getTags().get(MRJobTagName.USER.toString())); + info.setJobId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_ID.toString())); + info.setJobDefId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOD_DEF_ID.toString())); + info.setSiteId(jobExecutionAPIEntity.getTags().get(MRJobTagName.SITE.toString())); + info.jobName = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_NAME.toString()); + info.jobQueueName = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_QUEUE.toString()); + info.jobType = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_TYPE.toString()); + info.finishedMaps = jobExecutionAPIEntity.getNumFinishedMaps(); + info.finishedReduces = jobExecutionAPIEntity.getNumFinishedReduces(); + info.failedReduces = jobExecutionAPIEntity.getNumFailedReduces(); + info.failedMaps = jobExecutionAPIEntity.getNumFailedMaps(); + info.totalMaps = jobExecutionAPIEntity.getNumTotalMaps(); + info.totalReduces = jobExecutionAPIEntity.getNumTotalReduces(); + } + } + + public void jobConfigCreated(Configuration configuration) { + this.jobConf = configuration; + } + + public void jobCountersCreated(JobCounters totalCounters, JobCounters mapCounters, JobCounters reduceCounters) { + info.totalCounters = totalCounters; + info.reduceCounters = reduceCounters; + info.mapCounters = mapCounters; + } + + @Override + public void flush() throws Exception { + + } + + public static class MapReduceAnalyzerEntity extends AnalyzerEntity { + String jobName; + String jobQueueName; + String jobType; + int totalMaps; + int totalReduces; + int failedMaps; + int failedReduces; + int finishedMaps; + int finishedReduces; + JobCounters totalCounters; + JobCounters mapCounters; + JobCounters reduceCounters; + Map tasksMap; + Map completedTaskAttemptsMap; + + public MapReduceAnalyzerEntity() { + this.setEndTime(-1); + this.setStartTime(-1); + finishedMaps = finishedReduces = 0; + jobName = jobQueueName = ""; + tasksMap = new HashMap<>(); + completedTaskAttemptsMap = new HashMap<>(); + } + + public String getJobName() { + return jobName; + } + + public String getJobQueueName() { + return jobQueueName; + } + + public String getJobType() { + return jobType; + } + + public int getTotalMaps() { + return totalMaps; + } + + public int getTotalReduces() { + return totalReduces; + } + + public int getFailedMaps() { + return failedMaps; + } + + public int getFailedReduces() { + return failedReduces; + } + + public int getFinishedMaps() { + return finishedMaps; + } + + public int getFinishedReduces() { + return finishedReduces; + } + + public JobCounters getTotalCounters() { + return totalCounters; + } + + public JobCounters getMapCounters() { + return mapCounters; + } + + public JobCounters getReduceCounters() { + return reduceCounters; + } + + public Map getTasksMap() { + return tasksMap; + } + + public Map getCompletedTaskAttemptsMap() { + return completedTaskAttemptsMap; + } + } + +} From 5e9c2e0a22e3ff2fabcf28580391064b4b7170b5 Mon Sep 17 00:00:00 2001 From: Qingwen Zhao Date: Tue, 17 Jan 2017 14:34:02 +0800 Subject: [PATCH 05/22] add job suggestion processors --- eagle-jpm/eagle-jpm-analyzer/pom.xml | 5 + .../eagle/jpm/analyzer/AnalyzerEntity.java | 2 +- .../apache/eagle/jpm/analyzer/Evaluator.java | 4 +- .../apache/eagle/jpm/analyzer/Processor.java | 4 +- .../meta/model/MapReduceAnalyzerEntity.java | 166 +++++++++++++ .../mr/suggestion/JobSuggestionEvaluator.java | 35 ++- .../MapReduceCompressionSettingAnalyzer.java | 82 +++++++ .../suggestion/MapReduceDataSkewAnalyzer.java | 62 +++++ .../suggestion/MapReduceGCTimeAnalyzer.java | 66 ++++++ .../MapReduceJobSuggestionContext.java | 221 ++++++++++++++++++ .../MapReduceQueueResourceAnalyzer.java | 83 +++++++ .../mr/suggestion/MapReduceSpillAnalyzer.java | 115 +++++++++ .../MapReduceSplitSettingAnalyzer.java | 43 ++++ .../suggestion/MapReduceTaskNumAnalyzer.java | 163 +++++++++++++ .../history/parser/JobSuggestionListener.java | 112 ++------- 15 files changed, 1055 insertions(+), 108 deletions(-) create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java diff --git a/eagle-jpm/eagle-jpm-analyzer/pom.xml b/eagle-jpm/eagle-jpm-analyzer/pom.xml index d6383b685a..1f7a69d8bd 100644 --- a/eagle-jpm/eagle-jpm-analyzer/pom.xml +++ b/eagle-jpm/eagle-jpm-analyzer/pom.xml @@ -29,6 +29,11 @@ eagle-jpm-util ${project.version} + + org.apache.eagle + eagle-jpm-entity + ${project.version} + org.apache.eagle eagle-app-base diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java index 98e0f97ed1..a600f79ee3 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java @@ -22,7 +22,7 @@ /** * will refactor later if other types of job needs this. - * AnalyzerEntity for each job needed to be analysised + * AnalyzerEntity for each job needed to be analyzed */ public class AnalyzerEntity { private String jobDefId; diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java index 6617916ff5..17fc119a23 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java @@ -19,6 +19,6 @@ import org.apache.eagle.jpm.analyzer.publisher.Result; -public interface Evaluator { - Result.EvaluatorResult evaluate(AnalyzerEntity analyzerEntity); +public interface Evaluator { + Result.EvaluatorResult evaluate(T analyzerEntity); } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java index d5a8a74ea1..0e8b417757 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java @@ -19,6 +19,6 @@ import org.apache.eagle.jpm.analyzer.publisher.Result; -public interface Processor { - Result.ProcessorResult process(AnalyzerEntity jobAnalysisEntity); +public interface Processor { + Result.ProcessorResult process(T jobAnalysisEntity); } \ No newline at end of file diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java new file mode 100644 index 0000000000..2dc26fc27d --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.meta.model; + +import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; +import org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; + +import java.util.HashMap; +import java.util.Map; + +public class MapReduceAnalyzerEntity extends AnalyzerEntity { + private String jobName; + private String jobQueueName; + private String jobType; + private int totalMaps; + private int totalReduces; + private int failedMaps; + private int failedReduces; + private int finishedMaps; + private int finishedReduces; + private JobCounters totalCounters; + private JobCounters mapCounters; + private JobCounters reduceCounters; + private Map tasksMap; + private Map completedTaskAttemptsMap; + + public MapReduceAnalyzerEntity() { + this.setEndTime(-1); + this.setStartTime(-1); + finishedMaps = finishedReduces = 0; + jobName = jobQueueName = ""; + tasksMap = new HashMap<>(); + completedTaskAttemptsMap = new HashMap<>(); + } + + public String getJobName() { + return jobName; + } + + public String getJobQueueName() { + return jobQueueName; + } + + public String getJobType() { + return jobType; + } + + public int getTotalMaps() { + return totalMaps; + } + + public int getTotalReduces() { + return totalReduces; + } + + public int getFailedMaps() { + return failedMaps; + } + + public int getFailedReduces() { + return failedReduces; + } + + public int getFinishedMaps() { + return finishedMaps; + } + + public int getFinishedReduces() { + return finishedReduces; + } + + public JobCounters getTotalCounters() { + return totalCounters; + } + + public JobCounters getMapCounters() { + return mapCounters; + } + + public JobCounters getReduceCounters() { + return reduceCounters; + } + + public Map getTasksMap() { + return tasksMap; + } + + public Map getCompletedTaskAttemptsMap() { + return completedTaskAttemptsMap; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public void setJobQueueName(String jobQueueName) { + this.jobQueueName = jobQueueName; + } + + public void setJobType(String jobType) { + this.jobType = jobType; + } + + public void setTotalMaps(int totalMaps) { + this.totalMaps = totalMaps; + } + + public void setTotalReduces(int totalReduces) { + this.totalReduces = totalReduces; + } + + public void setFailedMaps(int failedMaps) { + this.failedMaps = failedMaps; + } + + public void setFailedReduces(int failedReduces) { + this.failedReduces = failedReduces; + } + + public void setFinishedMaps(int finishedMaps) { + this.finishedMaps = finishedMaps; + } + + public void setFinishedReduces(int finishedReduces) { + this.finishedReduces = finishedReduces; + } + + public void setTotalCounters(JobCounters totalCounters) { + this.totalCounters = totalCounters; + } + + public void setMapCounters(JobCounters mapCounters) { + this.mapCounters = mapCounters; + } + + public void setReduceCounters(JobCounters reduceCounters) { + this.reduceCounters = reduceCounters; + } + + public void setTasksMap(Map tasksMap) { + this.tasksMap = tasksMap; + } + + public void setCompletedTaskAttemptsMap(Map completedTaskAttemptsMap) { + this.completedTaskAttemptsMap = completedTaskAttemptsMap; + } + + +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java index 4f088dc9cb..8afeadbad7 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java @@ -20,14 +20,18 @@ import com.typesafe.config.Config; import org.apache.eagle.jpm.analyzer.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.Evaluator; +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.util.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; -public class JobSuggestionEvaluator implements Evaluator, Serializable { +public class JobSuggestionEvaluator implements Evaluator, Serializable { private static final Logger LOG = LoggerFactory.getLogger(JobSuggestionEvaluator.class); private Config config; @@ -36,14 +40,37 @@ public JobSuggestionEvaluator(Config config) { this.config = config; } + private List loadProcessors(MapReduceJobSuggestionContext context) { + List processors = new ArrayList<>(); + processors.add(new MapReduceCompressionSettingAnalyzer(context)); + processors.add(new MapReduceSplitSettingAnalyzer(context)); + processors.add(new MapReduceDataSkewAnalyzer(context)); + processors.add(new MapReduceGCTimeAnalyzer(context)); + processors.add(new MapReduceSpillAnalyzer(context)); + processors.add(new MapReduceTaskNumAnalyzer(context)); + processors.add(new MapReduceQueueResourceAnalyzer(context)); + + return processors; + } + @Override - public Result.EvaluatorResult evaluate(AnalyzerEntity mrJobEntity) { - if (mrJobEntity.getCurrentState().equalsIgnoreCase(Constants.JobState.RUNNING.toString())) { + public Result.EvaluatorResult evaluate(MapReduceAnalyzerEntity analyzerEntity) { + if (analyzerEntity.getCurrentState().equalsIgnoreCase(Constants.JobState.RUNNING.toString())) { + return null; + } + + MapReduceJobSuggestionContext jobContext = MapReduceJobSuggestionContext.getInstance().buildContext(); + if (jobContext.getNumMaps() == 0) { return null; } Result.EvaluatorResult result = new Result.EvaluatorResult(); - //TODO + for (Processor processor : loadProcessors(jobContext)) { + Result.ProcessorResult processorResult = processor.process(analyzerEntity); + if (processorResult != null) { + result.addProcessorResult(processor.getClass(), processorResult); + } + } return result; } } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java new file mode 100644 index 0000000000..d8870b7bd9 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; + +import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_OUTPUT_COMPRESS; +import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC; +import static org.apache.hadoop.mapreduce.MRJobConfig.NUM_REDUCES; +import static org.apache.hadoop.mapreduce.MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR; + +public class MapReduceCompressionSettingAnalyzer implements Processor { + + private MapReduceJobSuggestionContext context; + + public MapReduceCompressionSettingAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + StringBuilder sb = new StringBuilder(); + JobConf jobconf = new JobConf(context.getJobconf()); + if (jobconf.getLong(NUM_REDUCES, 0) > 0) { + if (jobconf.getCompressMapOutput() == false) { + sb.append("Please set " + MAP_OUTPUT_COMPRESS + + " to true to reduce network IO.\n"); + } else { + String codecClassName = jobconf + .get(MAP_OUTPUT_COMPRESS_CODEC); + if (!(codecClassName.endsWith("LzoCodec") || codecClassName + .endsWith("SnappyCodec"))) { + sb.append("Best practice: use LzoCodec or SnappyCodec for " + + MAP_OUTPUT_COMPRESS_CODEC); + sb.append("\n"); + } + } + } + + if (!jobconf.getBoolean(FileOutputFormat.COMPRESS, false)) { + sb.append("Please set " + FileOutputFormat.COMPRESS + + " to true to reduce disk usage and network IO.\n"); + } else { + String codecName = jobconf.get(FileOutputFormat.COMPRESS_CODEC, ""); + String outputFileFormat = jobconf.get(OUTPUT_FORMAT_CLASS_ATTR, ""); + + if ((codecName.endsWith("GzipCodec") || + codecName.endsWith("SnappyCodec") || + codecName.endsWith("DefaultCodec")) + && outputFileFormat.endsWith("TextOutputFormat")) { + sb.append("Best practice: don't use Gzip/Snappy/DefaultCodec with TextOutputFormat"); + sb.append(" as this will cause the output files to be unsplittable. "); + sb.append("Please use LZO instead or "); + sb.append("use a container file format such as SequenceFileOutputFormat.\n"); + } + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + } + return null; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java new file mode 100644 index 0000000000..ebe2f4f170 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; + +public class MapReduceDataSkewAnalyzer implements Processor { + private MapReduceJobSuggestionContext context; + public MapReduceDataSkewAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + TaskAttemptExecutionAPIEntity worstReduce = context.getWorstReduce(); + if (context.getNumReduces() == 0 || worstReduce == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + try { + long worstTime = (worstReduce.getEndTime() - worstReduce + .getShuffleFinishTime()) / 1000; + if (worstTime - context.getAvgReduceTimeInSec() > 30 * 60 ) { + long avgInputs = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS) + / context.getNumReduces(); + long worstInputs = worstReduce.getJobCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS); + + if (worstInputs / 5 > avgInputs) { + sb.append("Data skew detected in reducers. The average reduce time is " + + context.getAvgReduceTimeInSec()); + sb.append(" seconds, the worst reduce time is " + worstTime); + sb.append(" seconds. Please investigate this problem to improve your job performance.\n"); + } + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + } + } catch (NullPointerException e) { + // When job failed there may not have counters, so just ignore it + } + return null; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java new file mode 100644 index 0000000000..6edfda4bd3 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; + +import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_JAVA_OPTS; +import static org.apache.hadoop.mapreduce.MRJobConfig.REDUCE_JAVA_OPTS; + +public class MapReduceGCTimeAnalyzer implements Processor { + private MapReduceJobSuggestionContext context; + + public MapReduceGCTimeAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + StringBuilder sb = new StringBuilder(); + try { + long mapGCTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS); + long mapCPUTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS); + + if (mapGCTime > mapCPUTime * 0.1) { + sb.append("Map GC_TIME_MILLIS took too long. Please increase mapper memory via -D" + + MAP_JAVA_OPTS); + sb.append(", or optimize your mapper class.\n"); + } + + if (context.getNumReduces() > 0) { + long reduceGCTime = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS); + long reduceCPUTime = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS); + if (reduceGCTime > reduceCPUTime * 0.1) { + sb.append("Reduce GC_TIME_MILLIS took too long. Please increase memory for reduce via -D" + + REDUCE_JAVA_OPTS); + sb.append(", or optimize your reducer class.\n"); + } + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + } + } catch (NullPointerException e) { + // When job failed there may not have counters, so just ignore it + } + return null; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java new file mode 100644 index 0000000000..1c87f60eb5 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.common.DateTimeUtil; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; +import org.apache.eagle.jpm.util.MRJobTagName; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.TaskType; + +import java.util.regex.Pattern; + +import static org.apache.eagle.jpm.util.jobcounter.JobCounters.CounterName.MAP_OUTPUT_BYTES; +import static org.apache.eagle.jpm.util.jobcounter.JobCounters.CounterName.MAP_OUTPUT_RECORDS; +import static org.apache.hadoop.mapreduce.MRJobConfig.NUM_MAPS; +import static org.apache.hadoop.mapreduce.MRJobConfig.NUM_REDUCES; + +public class MapReduceJobSuggestionContext { + + private JobConf jobconf; + private MapReduceAnalyzerEntity job; + + private long numMaps; + private long numReduces; + + private long avgMapTimeInSec; + private long avgReduceTimeInSec; + private long avgShuffleTimeInSec; + private TaskAttemptExecutionAPIEntity worstMap; + private TaskAttemptExecutionAPIEntity worstReduce; + private TaskAttemptExecutionAPIEntity worstShuffle; + private TaskAttemptExecutionAPIEntity lastMap; + private TaskAttemptExecutionAPIEntity lastReduce; + private TaskAttemptExecutionAPIEntity lastShuffle; + private TaskAttemptExecutionAPIEntity firstMap; + private TaskAttemptExecutionAPIEntity firstReduce; + private TaskAttemptExecutionAPIEntity firstShuffle; + + private long minMapSpillMemBytes; + + public static final Pattern MAX_HEAP_PATTERN = Pattern.compile("-Xmx([0-9]+)([kKmMgG]?)"); + public final long _1MB = 1024 * 1024l; + public final long _256MB = 256 * _1MB; + public final long _512MB = 512 * _1MB; + public final long _1GB = 1024 * _1MB; + + private static Boolean initialized = false; + + private MapReduceJobSuggestionContext() {} + private static MapReduceJobSuggestionContext instance; + public static MapReduceJobSuggestionContext getInstance() { + if (instance == null) { + synchronized (MapReduceJobSuggestionContext.class) { + if (instance == null) { + instance = new MapReduceJobSuggestionContext(); + } + } + } + return instance; + } + + public MapReduceJobSuggestionContext buildContext() { + if (!initialized) { + avgMapTimeInSec = avgReduceTimeInSec = avgShuffleTimeInSec = 0; + numMaps = jobconf.getLong(NUM_MAPS, 0); + numReduces = jobconf.getLong(NUM_REDUCES, 0); + + for (TaskAttemptExecutionAPIEntity attempt : job.getCompletedTaskAttemptsMap().values()) { + if (TaskType.MAP.toString().equalsIgnoreCase(getTaskType(attempt))) { + long mapTime = attempt.getEndTime() - attempt.getStartTime(); + avgMapTimeInSec += mapTime; + if (firstMap == null || firstMap.getStartTime() > attempt.getStartTime()) { + firstMap = attempt; + } + if (lastMap == null || lastMap.getEndTime() < attempt.getEndTime()) { + lastMap = attempt; + } + if (worstMap == null || (worstMap.getEndTime() - worstMap.getStartTime()) < mapTime) { + worstMap = attempt; + } + long tmpMem = getMinimumIOSortMemory(attempt); + if (tmpMem > minMapSpillMemBytes) { + minMapSpillMemBytes = tmpMem; + } + } else if (TaskType.REDUCE.toString().equalsIgnoreCase(getTaskType(attempt))) { + long shuffleTime = attempt.getShuffleFinishTime() - attempt.getStartTime(); + avgShuffleTimeInSec += shuffleTime; + if (firstShuffle == null || firstShuffle.getStartTime() > attempt.getStartTime()) { + firstShuffle = attempt; + } + if (lastShuffle == null || lastShuffle.getShuffleFinishTime() < attempt.getShuffleFinishTime()) { + lastShuffle = attempt; + } + if (worstShuffle == null || (worstShuffle.getShuffleFinishTime() - worstShuffle.getStartTime()) < shuffleTime) { + worstShuffle = attempt; + } + + long reduceTime = attempt.getEndTime() - attempt.getShuffleFinishTime(); + avgReduceTimeInSec += reduceTime; + if (firstReduce == null || firstReduce.getStartTime() > attempt.getStartTime()) { + firstReduce = attempt; + } + if (lastReduce == null || lastReduce.getEndTime() < attempt.getEndTime()) { + lastReduce = attempt; + } + if (worstReduce == null || (worstReduce.getEndTime() - worstReduce.getShuffleFinishTime()) < reduceTime) { + worstReduce = attempt; + } + } + } + if (numMaps > 0) { + avgMapTimeInSec = avgMapTimeInSec / numMaps / DateTimeUtil.ONESECOND; + } + if (numReduces > 0) { + avgReduceTimeInSec = avgReduceTimeInSec / numReduces / DateTimeUtil.ONESECOND; + avgShuffleTimeInSec = avgShuffleTimeInSec / numReduces / DateTimeUtil.ONESECOND; + } + initialized = true; + } + return this; + } + + private String getTaskType(TaskAttemptExecutionAPIEntity taskAttemptInfo) { + return taskAttemptInfo.getTags().get(MRJobTagName.TASK_TYPE); + } + + /** + * 16 is the default index size + * @param attempt + * @return + */ + private long getMinimumIOSortMemory(TaskAttemptExecutionAPIEntity attempt) { + long records = attempt.getJobCounters().getCounterValue(MAP_OUTPUT_RECORDS); + long outputBytes = attempt.getJobCounters().getCounterValue(MAP_OUTPUT_BYTES); + return outputBytes + records * 16; + } + + public JobConf getJobconf() { + return jobconf; + } + + public MapReduceAnalyzerEntity getJob() { + return job; + } + + public long getNumMaps() { + return numMaps; + } + + public long getNumReduces() { + return numReduces; + } + + public long getAvgMapTimeInSec() { + return avgMapTimeInSec; + } + + public long getAvgReduceTimeInSec() { + return avgReduceTimeInSec; + } + + public long getAvgShuffleTimeInSec() { + return avgShuffleTimeInSec; + } + + public TaskAttemptExecutionAPIEntity getWorstMap() { + return worstMap; + } + + public TaskAttemptExecutionAPIEntity getWorstReduce() { + return worstReduce; + } + + public TaskAttemptExecutionAPIEntity getWorstShuffle() { + return worstShuffle; + } + + public TaskAttemptExecutionAPIEntity getLastMap() { + return lastMap; + } + + public TaskAttemptExecutionAPIEntity getLastReduce() { + return lastReduce; + } + + public TaskAttemptExecutionAPIEntity getLastShuffle() { + return lastShuffle; + } + + public TaskAttemptExecutionAPIEntity getFirstMap() { + return firstMap; + } + + public TaskAttemptExecutionAPIEntity getFirstReduce() { + return firstReduce; + } + + public TaskAttemptExecutionAPIEntity getFirstShuffle() { + return firstShuffle; + } + + public long getMinMapSpillMemBytes() { + return minMapSpillMemBytes; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java new file mode 100644 index 0000000000..e2d15a3e51 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.common.DateTimeUtil; +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Criterion: (TimeElapsed / (numTasks / 500 * avgTaskTime)) > 20 + */ +public class MapReduceQueueResourceAnalyzer implements Processor { + private static final Logger LOG = LoggerFactory.getLogger(MapReduceQueueResourceAnalyzer.class); + + private MapReduceJobSuggestionContext context; + + public MapReduceQueueResourceAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + try { + String userName = context.getJob().getUserId(); + TaskAttemptExecutionAPIEntity lastMap = context.getLastMap(); + TaskAttemptExecutionAPIEntity firstMap = context.getFirstMap(); + TaskAttemptExecutionAPIEntity lastReduce = context.getLastReduce(); + TaskAttemptExecutionAPIEntity firstShuffle = context.getFirstShuffle(); + + if (checkBatchUser(userName) && lastMap != null && firstMap != null) { + StringBuilder sb = new StringBuilder(); + + long tasksPerTime = 500; // better get it from RM + long mapPhaseTimeInSec = (lastMap.getEndTime() - firstMap.getStartTime()) / DateTimeUtil.ONESECOND; + if (mapPhaseTimeInSec > context.getAvgMapTimeInSec() + * ((context.getNumMaps() + tasksPerTime - 1) / tasksPerTime) * 20) { + sb.append("There appears to have been resource contention during the map phase of your job. Please ask for more resources if your job is SLA-bound,"); + sb.append(" or submit your job when the cluster is less busy.\n"); + } + + if (context.getNumReduces() > 0 && lastReduce != null && firstShuffle != null) { + long reducePhaseTimeInSec = (lastReduce.getEndTime() - firstShuffle.getStartTime()) / DateTimeUtil.ONESECOND; + if (reducePhaseTimeInSec > context.getAvgReduceTimeInSec() + * ((context.getNumReduces() + tasksPerTime - 1) / tasksPerTime) * 20) { + sb.append("Seems there was resource contention when your job in reduce phase, please ask for more resource if your job is SLA enabled,"); + sb.append(" or submit your job when the cluster is less busy.\n"); + } + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + } + } + } catch (Exception e) { + LOG.warn(e.getMessage(), e); + } + return null; + } + + protected boolean checkBatchUser(String userName) { + return userName.startsWith("b_"); + } +} \ No newline at end of file diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java new file mode 100644 index 0000000000..68ed88114c --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.commons.io.FileUtils; +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; + +import java.util.regex.Matcher; + +import static org.apache.eagle.jpm.analyzer.mr.suggestion.MapReduceJobSuggestionContext.MAX_HEAP_PATTERN; +import static org.apache.hadoop.mapreduce.MRJobConfig.IO_SORT_MB; +import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_JAVA_OPTS; +import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_SORT_SPILL_PERCENT; + +/** + * Check whether spilled more than once, if true, find out the minimum value of the memory to hold all the data, + * based on that value, find out how much memory need for heap size. + */ +public class MapReduceSpillAnalyzer implements Processor { + + private MapReduceJobSuggestionContext context; + + public MapReduceSpillAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + StringBuilder sb = new StringBuilder(); + long outputRecords = 0L; // Map output records + long spillRecords = 0L; // Spilled Records + try { + outputRecords = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.MAP_OUTPUT_RECORDS); + spillRecords = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS); + + if (outputRecords < spillRecords) { + sb.append("Total Map output records: " + outputRecords); + sb.append(" Total Spilled Records: " + spillRecords); + sb.append(". Please set"); + + long minMapSpillMemBytes = context.getMinMapSpillMemBytes(); + double spillPercent = context.getJobconf().getDouble(MAP_SORT_SPILL_PERCENT, 0.8); + if (minMapSpillMemBytes > context._512MB * spillPercent) { + if (Math.abs(1.0 - spillPercent) > 0.001) { + sb.append(" -D" + MAP_SORT_SPILL_PERCENT + "=1"); + } + } else { + minMapSpillMemBytes /= spillPercent; + } + + minMapSpillMemBytes = (minMapSpillMemBytes / FileUtils.ONE_MB + 10) / 10 * 10; + if (minMapSpillMemBytes >= 2047 ) { + sb.append(" Please reduce the block size of the input files and make sure they are splittable."); + } else { + sb.append(" -D" + IO_SORT_MB + "=" + minMapSpillMemBytes); + long heapSize = getMaxHeapSize(context.getJobconf().get(MAP_JAVA_OPTS)); + if (heapSize < 3 * minMapSpillMemBytes) { + sb.append(" -D" + MAP_JAVA_OPTS + "=-Xmx" + + (long) (minMapSpillMemBytes * 3 + 1024) / 1024 * 1024 + "M"); + } + } + sb.append(" to avoid spilled records.\n"); + } + + + long reduceInputRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS); + spillRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS); + if (reduceInputRecords < spillRecords) { + sb.append("Please add more memory (mapreduce.reduce.java.opts) to avoid spilled records."); + sb.append(" Total Reduce input records: " + reduceInputRecords); + sb.append(" Total Spilled Records: " + spillRecords); + sb.append("\n"); + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + } + } catch (NullPointerException e) { + //When job failed there may not have counters, so just ignore it + } + return null; + } + + private static long getMaxHeapSize(String s) { + Matcher m = MAX_HEAP_PATTERN.matcher(s); + long val = 0; + if (m.find()) { + val = Long.parseLong(m.group(1)); + if ("k".equalsIgnoreCase(m.group(2))) { + val /= 1024; + } else if ("g".equalsIgnoreCase(m.group(2))) { + val *= 1024; + } + } + return val; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java new file mode 100644 index 0000000000..8dbf41c30a --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; + +public class MapReduceSplitSettingAnalyzer implements Processor { + + private MapReduceJobSuggestionContext context; + + public MapReduceSplitSettingAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + StringBuilder sb = new StringBuilder(); + if (context.getJobconf().getLong(FileInputFormat.SPLIT_MINSIZE, 0) > 1) { + sb.append("Best practice: don't set " + FileInputFormat.SPLIT_MINSIZE); + sb.append(", because it may lower data locality, hence maps will run slower.\n"); + return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + } + return null; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java new file mode 100644 index 0000000000..4a0e89d581 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.commons.io.FileUtils; +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; + +public class MapReduceTaskNumAnalyzer implements Processor { + + private MapReduceJobSuggestionContext context; + + public MapReduceTaskNumAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + StringBuilder sb = new StringBuilder(); + try { + long numReduces = context.getNumReduces(); + long numMaps = context.getNumMaps(); + if (numReduces > 0) { + long avgReduceTime = context.getAvgReduceTimeInSec(); + long avgShuffleTime = context.getAvgShuffleTimeInSec(); + long avgShuffleBytes = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_SHUFFLE_BYTES) + / numReduces; + long avgReduceOutput = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.HDFS_BYTES_WRITTEN) + / numReduces; + long avgReduceTotalTime = avgShuffleTime + avgReduceTime; + + long suggestReduces = 0; + StringBuilder tmpsb = new StringBuilder(); + + if (avgShuffleBytes < context._256MB && avgReduceTotalTime < 300 + && avgReduceOutput < context._256MB && numReduces > 1) { + tmpsb.append("average reduce input bytes is: "); + tmpsb.append(FileUtils.byteCountToDisplaySize(avgShuffleBytes) + ", "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } else if (avgShuffleBytes > 10 * FileUtils.ONE_GB && avgReduceTotalTime > 1800) { + tmpsb.append("average reduce input bytes is: "); + tmpsb.append(FileUtils.byteCountToDisplaySize(avgShuffleBytes) + ", "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } + + if (avgReduceTotalTime < 60 && numReduces > 1) { + tmpsb.append("average reduce time is only " + + (context.getAvgShuffleTimeInSec() + context.getAvgReduceTimeInSec()) + " seconds, "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } else if (avgReduceTotalTime > 3600 && avgReduceTime > 1800) { + tmpsb.append("average reduce time is " + + (avgShuffleTime + avgReduceTime) + " seconds, "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } + + if (avgReduceOutput < 10 * FileUtils.ONE_MB && avgReduceTime < 300 + && avgShuffleBytes < 2 * FileUtils.ONE_GB && numReduces > 1) { + tmpsb.append(" average reduce output is only "); + tmpsb.append(FileUtils.byteCountToDisplaySize(avgReduceOutput) + ", "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } else if (avgReduceOutput > 10 * FileUtils.ONE_GB && avgReduceTime > 1800) { + tmpsb.append(" average reduce output is "); + tmpsb.append(FileUtils.byteCountToDisplaySize(avgReduceOutput) + ", "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } + + if (suggestReduces > 0) { + sb.append("Best practice: "); + sb.append(tmpsb.toString()); + sb.append("please consider "); + if (suggestReduces > numReduces) { + sb.append("increasing the "); + } else { + sb.append("decreasing the "); + } + sb.append("reducer number. You could try -Dmapreduce.job.reduces=" + suggestReduces + "\n"); + } + } + + // map numbers + long avgMapInput = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.HDFS_BYTES_READ) + / numMaps; + long avgMapTime = context.getAvgMapTimeInSec(); + if (avgMapInput < 5 * FileUtils.ONE_MB && avgMapTime < 30 && numMaps > 1) { + sb.append("Best practice: average map input bytes only have "); + sb.append(FileUtils.byteCountToDisplaySize(avgMapInput)); + sb.append(". Please reduce the number of mappers by merging input files.\n"); + } else if (avgMapInput > FileUtils.ONE_GB) { + sb.append("Best practice: average map input bytes have "); + sb.append(FileUtils.byteCountToDisplaySize(avgMapInput)); + sb.append(". Please increase the number of mappers by using splittable compression, a container file format or a smaller block size.\n"); + } + + if (avgMapTime < 10 && numMaps > 1) { + sb.append("Best practice: average map time only have " + avgMapTime); + sb.append(" seconds. Please reduce the number of mappers by merging input files or by using a larger block size.\n"); + } else if (avgMapTime > 600 && avgMapInput < FileUtils.ONE_GB) { + sb.append("Best practice: average map time is " + avgMapInput); + sb.append(" seconds. Please increase the number of mappers by using splittable compression, a container file format or a smaller block size.\n"); + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + } + } catch (NullPointerException e) { + // When job failed there may not have counters, so just ignore it + } + return null; + } + + private long getReduceNum(long avgInputBytes, long avgOutputBytes, long avgTime) { + long newReduceNum = 1; + long tmpReduceNum; + + long numReduces = context.getNumReduces(); + tmpReduceNum = avgInputBytes * numReduces / (3 * FileUtils.ONE_GB); + if (tmpReduceNum > newReduceNum) { + newReduceNum = tmpReduceNum; + } + + tmpReduceNum = avgOutputBytes * numReduces / (2 * FileUtils.ONE_GB); + if (tmpReduceNum > newReduceNum) { + newReduceNum = tmpReduceNum; + } + + tmpReduceNum = avgTime * numReduces / (10 * 60); + if (tmpReduceNum > newReduceNum) { + newReduceNum = tmpReduceNum; + } + + return newReduceNum; + } + +} \ No newline at end of file diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java index a7cca181c2..24c5fb90e3 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java @@ -17,7 +17,7 @@ package org.apache.eagle.jpm.mr.history.parser; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; import org.apache.eagle.jpm.mr.historyentity.JobBaseAPIEntity; import org.apache.eagle.jpm.mr.historyentity.JobExecutionAPIEntity; import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; @@ -30,9 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Map; - /** * JobEventCounterListener provides an interface to add job/task counter analyzers */ @@ -58,15 +55,15 @@ public void jobEntityCreated(JobBaseAPIEntity entity) throws Exception { info.setJobId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_ID.toString())); info.setJobDefId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOD_DEF_ID.toString())); info.setSiteId(jobExecutionAPIEntity.getTags().get(MRJobTagName.SITE.toString())); - info.jobName = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_NAME.toString()); - info.jobQueueName = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_QUEUE.toString()); - info.jobType = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_TYPE.toString()); - info.finishedMaps = jobExecutionAPIEntity.getNumFinishedMaps(); - info.finishedReduces = jobExecutionAPIEntity.getNumFinishedReduces(); - info.failedReduces = jobExecutionAPIEntity.getNumFailedReduces(); - info.failedMaps = jobExecutionAPIEntity.getNumFailedMaps(); - info.totalMaps = jobExecutionAPIEntity.getNumTotalMaps(); - info.totalReduces = jobExecutionAPIEntity.getNumTotalReduces(); + info.setJobName(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_NAME.toString())) ; + info.setJobQueueName(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_QUEUE.toString())); + info.setJobType(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_TYPE.toString())); + info.setFinishedMaps(jobExecutionAPIEntity.getNumFinishedMaps()); + info.setFinishedReduces(jobExecutionAPIEntity.getNumFinishedReduces()); + info.setFailedReduces(jobExecutionAPIEntity.getNumFailedReduces()); + info.setFailedMaps(jobExecutionAPIEntity.getNumFailedMaps()); + info.setTotalMaps(jobExecutionAPIEntity.getNumTotalMaps()); + info.setTotalReduces(jobExecutionAPIEntity.getNumTotalReduces()); } } @@ -75,96 +72,13 @@ public void jobConfigCreated(Configuration configuration) { } public void jobCountersCreated(JobCounters totalCounters, JobCounters mapCounters, JobCounters reduceCounters) { - info.totalCounters = totalCounters; - info.reduceCounters = reduceCounters; - info.mapCounters = mapCounters; + info.setTotalCounters(totalCounters); + info.setReduceCounters(reduceCounters); + info.setMapCounters(mapCounters); } @Override public void flush() throws Exception { } - - public static class MapReduceAnalyzerEntity extends AnalyzerEntity { - String jobName; - String jobQueueName; - String jobType; - int totalMaps; - int totalReduces; - int failedMaps; - int failedReduces; - int finishedMaps; - int finishedReduces; - JobCounters totalCounters; - JobCounters mapCounters; - JobCounters reduceCounters; - Map tasksMap; - Map completedTaskAttemptsMap; - - public MapReduceAnalyzerEntity() { - this.setEndTime(-1); - this.setStartTime(-1); - finishedMaps = finishedReduces = 0; - jobName = jobQueueName = ""; - tasksMap = new HashMap<>(); - completedTaskAttemptsMap = new HashMap<>(); - } - - public String getJobName() { - return jobName; - } - - public String getJobQueueName() { - return jobQueueName; - } - - public String getJobType() { - return jobType; - } - - public int getTotalMaps() { - return totalMaps; - } - - public int getTotalReduces() { - return totalReduces; - } - - public int getFailedMaps() { - return failedMaps; - } - - public int getFailedReduces() { - return failedReduces; - } - - public int getFinishedMaps() { - return finishedMaps; - } - - public int getFinishedReduces() { - return finishedReduces; - } - - public JobCounters getTotalCounters() { - return totalCounters; - } - - public JobCounters getMapCounters() { - return mapCounters; - } - - public JobCounters getReduceCounters() { - return reduceCounters; - } - - public Map getTasksMap() { - return tasksMap; - } - - public Map getCompletedTaskAttemptsMap() { - return completedTaskAttemptsMap; - } - } - } From 46be1b385bb2595edc4e6f2154989261db7cd8d1 Mon Sep 17 00:00:00 2001 From: Qingwen Zhao Date: Thu, 12 Jan 2017 11:00:12 +0800 Subject: [PATCH 06/22] rename method analysis to analyze --- .../java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java | 6 +++--- .../eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java | 2 +- .../org/apache/eagle/jpm/mr/running/parser/MRJobParser.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java index 6cda1cdc26..7cc5c2e44a 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java @@ -18,11 +18,11 @@ package org.apache.eagle.jpm.analyzer; /** - * Each JobAnalyzer contains one or more Evaluators to analysis each job. + * Each JobAnalyzer contains one or more Evaluators to analyze each job. * Each Evaluator is a group of Processors - * Each Processor implements an algorithm or a model to analysis one dimension of a job + * Each Processor implements an algorithm or a model to analyze one dimension of a job * */ public interface JobAnalyzer { - void analysis(AnalyzerEntity analyzerEntity) throws Exception; + void analyze(AnalyzerEntity analyzerEntity) throws Exception; } \ No newline at end of file diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index e0e579a951..e6d0d09099 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -51,7 +51,7 @@ public MRJobPerformanceAnalyzer(Config config) { } @Override - public void analysis(AnalyzerEntity analyzerJobEntity) throws Exception { + public void analyze(AnalyzerEntity analyzerJobEntity) throws Exception { Result result = new Result(); for (Evaluator evaluator : evaluators) { diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java index 120303ee2c..6a9422744c 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java @@ -173,7 +173,7 @@ private void fetchMRRunningInfo() throws Exception { break; } } - mrJobPerformanceAnalyzer.analysis(convertToAnalysisEntity(mrJobEntityMap.get(jobId))); + mrJobPerformanceAnalyzer.analyze(convertToAnalysisEntity(mrJobEntityMap.get(jobId))); } } From f2f516be66333a3c77d22d731144997081cb3cf8 Mon Sep 17 00:00:00 2001 From: Qingwen Zhao Date: Thu, 12 Jan 2017 14:41:27 +0800 Subject: [PATCH 07/22] fix a typo --- eagle-jpm/eagle-jpm-mr-history/pom.xml | 5 +++++ .../eagle/jpm/mr/history/parser/JHFEventReaderBase.java | 2 +- .../jpm/mr/history/parser/JobEntityCreationPublisher.java | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/eagle-jpm/eagle-jpm-mr-history/pom.xml b/eagle-jpm/eagle-jpm-mr-history/pom.xml index 349c489c91..2e3a1a82cc 100644 --- a/eagle-jpm/eagle-jpm-mr-history/pom.xml +++ b/eagle-jpm/eagle-jpm-mr-history/pom.xml @@ -89,6 +89,11 @@ 1.6 test + + org.apache.eagle + eagle-jpm-analyzer + ${project.version} + diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index d89937e937..5d8d1ba8bc 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -332,7 +332,7 @@ private void entityCreated(JobBaseAPIEntity entity) throws Exception { } } - super.notifiyListeners(entity); + super.notifyListeners(entity); } protected abstract JobCounters parseCounters(Object value) throws IOException; diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationPublisher.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationPublisher.java index a80462d912..f2730fdc1c 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationPublisher.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobEntityCreationPublisher.java @@ -29,7 +29,7 @@ public void addListener(HistoryJobEntityCreationListener l) { listeners.add(l); } - public void notifiyListeners(JobBaseAPIEntity entity) throws Exception { + public void notifyListeners(JobBaseAPIEntity entity) throws Exception { for (HistoryJobEntityCreationListener l : listeners) { l.jobEntityCreated(entity); } From a8a3db5f5cdb3991ec821663b1a9c6f82330c8f9 Mon Sep 17 00:00:00 2001 From: Qingwen Zhao Date: Thu, 12 Jan 2017 15:47:27 +0800 Subject: [PATCH 08/22] add job suggestion --- .../eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java | 5 ++++- .../apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java | 5 +++++ .../jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java | 5 +++++ .../apache/eagle/jpm/mr/history/parser/JHFParserFactory.java | 2 ++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index e6d0d09099..b772245970 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -55,7 +55,10 @@ public void analyze(AnalyzerEntity analyzerJobEntity) throws Exception { Result result = new Result(); for (Evaluator evaluator : evaluators) { - result.addEvaluatorResult(evaluator.getClass(), evaluator.evaluate(analyzerJobEntity)); + Result.EvaluatorResult evaluatorResult = evaluator.evaluate(analyzerJobEntity); + if (evaluatorResult != null) { + result.addEvaluatorResult(evaluator.getClass(), evaluatorResult); + } } for (Publisher publisher : publishers) { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java index f10b68d3e7..f8c2dcb26a 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java @@ -26,6 +26,7 @@ import org.apache.eagle.jpm.analyzer.Processor; import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.analyzer.util.Utils; +import org.apache.eagle.jpm.util.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,10 @@ public SLAJobEvaluator(Config config) { @Override public Result.EvaluatorResult evaluate(AnalyzerEntity analyzerJobEntity) { + if (!analyzerJobEntity.getCurrentState().equalsIgnoreCase(Constants.JobState.RUNNING.toString())) { + return null; + } + Result.EvaluatorResult result = new Result.EvaluatorResult(); List jobMetaEntities = Utils.getJobMeta(config, analyzerJobEntity.getJobDefId()); diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java index 79f5318d7e..4f088dc9cb 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java @@ -21,6 +21,7 @@ import org.apache.eagle.jpm.analyzer.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.Evaluator; import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.util.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +38,10 @@ public JobSuggestionEvaluator(Config config) { @Override public Result.EvaluatorResult evaluate(AnalyzerEntity mrJobEntity) { + if (mrJobEntity.getCurrentState().equalsIgnoreCase(Constants.JobState.RUNNING.toString())) { + return null; + } + Result.EvaluatorResult result = new Result.EvaluatorResult(); //TODO return result; diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFParserFactory.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFParserFactory.java index ca49d9cdc3..1d176405db 100755 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFParserFactory.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFParserFactory.java @@ -38,11 +38,13 @@ public static JHFParserBase getParser(Map baseTags, MRHistoryJobConfig.EagleServiceConfig eagleServiceConfig = appConfig.getEagleServiceConfig(); JHFMRVer2EventReader reader2 = new JHFMRVer2EventReader(baseTags, configuration, filter, appConfig); + // add HistoryJobEntityCreationListener reader2.addListener(new JobEntityCreationEagleServiceListener(appConfig)); reader2.addListener(new TaskFailureListener(eagleServiceConfig)); reader2.addListener(new TaskAttemptCounterListener(eagleServiceConfig)); reader2.addListener(new JobConfigurationCreationServiceListener(eagleServiceConfig)); + // add HistoryJobEntityLifecycleListener reader2.register(new JobEntityLifecycleAggregator()); JHFParserBase parser = new JHFMRVer2Parser(reader2); return parser; From 6da62e20b8b1fcc30784e1ccbefba0cb9f0c50c7 Mon Sep 17 00:00:00 2001 From: Qingwen Zhao Date: Fri, 13 Jan 2017 21:08:12 +0800 Subject: [PATCH 09/22] add JobSuggestionListner --- .../eagle/jpm/analyzer/AnalyzerEntity.java | 6 +- .../eagle/jpm/analyzer/JobAnalyzer.java | 4 +- .../analyzer/mr/MRJobPerformanceAnalyzer.java | 5 +- .../analyzer/publisher/EmailPublisher.java | 2 - .../TaskAttemptExecutionAPIEntity.java | 46 ++++- .../DefaultJHFInputStreamCallback.java | 3 +- .../mr/history/parser/JHFEventReaderBase.java | 30 +++- .../jpm/mr/history/parser/JHFFormat.java | 1 + .../history/parser/JHFMRVer2EventReader.java | 11 +- .../history/parser/JobSuggestionListener.java | 170 ++++++++++++++++++ 10 files changed, 254 insertions(+), 24 deletions(-) create mode 100644 eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java index f9b7af0d2b..98e0f97ed1 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java @@ -5,9 +5,9 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java index 7cc5c2e44a..b0955ebfd0 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java @@ -23,6 +23,6 @@ * Each Processor implements an algorithm or a model to analyze one dimension of a job * */ -public interface JobAnalyzer { - void analyze(AnalyzerEntity analyzerEntity) throws Exception; +public interface JobAnalyzer { + void analyze(T analyzerEntity) throws Exception; } \ No newline at end of file diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index b772245970..54f12d621c 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -20,6 +20,7 @@ import com.typesafe.config.Config; import org.apache.eagle.jpm.analyzer.*; import org.apache.eagle.jpm.analyzer.Evaluator; +import org.apache.eagle.jpm.analyzer.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.mr.sla.SLAJobEvaluator; import org.apache.eagle.jpm.analyzer.mr.suggestion.JobSuggestionEvaluator; import org.apache.eagle.jpm.analyzer.publisher.EagleStorePublisher; @@ -33,7 +34,7 @@ import java.util.ArrayList; import java.util.List; -public class MRJobPerformanceAnalyzer implements JobAnalyzer, Serializable { +public class MRJobPerformanceAnalyzer implements JobAnalyzer, Serializable { private static final Logger LOG = LoggerFactory.getLogger(MRJobPerformanceAnalyzer.class); private List evaluators = new ArrayList<>(); @@ -51,7 +52,7 @@ public MRJobPerformanceAnalyzer(Config config) { } @Override - public void analyze(AnalyzerEntity analyzerJobEntity) throws Exception { + public void analyze(T analyzerJobEntity) throws Exception { Result result = new Result(); for (Evaluator evaluator : evaluators) { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java index 4e490942b5..9b6543384a 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java @@ -21,7 +21,6 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.eagle.app.service.ApplicationEmailService; import org.apache.eagle.common.DateTimeUtil; -import org.apache.eagle.common.mail.AlertEmailConstants; import org.apache.eagle.common.mail.AlertEmailContext; import org.apache.eagle.jpm.analyzer.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.dedup.AlertDeduplicator; @@ -31,7 +30,6 @@ import org.slf4j.LoggerFactory; import java.io.Serializable; -import java.net.URLEncoder; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java index 8db7f5cd13..46fcf5e9b9 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java @@ -46,6 +46,40 @@ public class TaskAttemptExecutionAPIEntity extends JobBaseAPIEntity { private String error; @Column("f") private JobCounters jobCounters; + // new added + @Column("g") + private long shuffleFinishTime; + @Column("h") + private long sortFinishTime; + @Column("i") + private long mapFinishTime; + + public long getShuffleFinishTime() { + return shuffleFinishTime; + } + + public void setShuffleFinishTime(long shuffleFinishTime) { + this.shuffleFinishTime = shuffleFinishTime; + valueChanged("shuffleFinishTime"); + } + + public long getSortFinishTime() { + return sortFinishTime; + } + + public void setSortFinishTime(long sortFinishTime) { + this.sortFinishTime = sortFinishTime; + valueChanged("sortFinishTime"); + } + + public long getMapFinishTime() { + return mapFinishTime; + } + + public void setMapFinishTime(long mapFinishTime) { + this.mapFinishTime = mapFinishTime; + valueChanged("mapFinishTime"); + } public String getTaskStatus() { return taskStatus; @@ -53,7 +87,7 @@ public String getTaskStatus() { public void setTaskStatus(String taskStatus) { this.taskStatus = taskStatus; - pcs.firePropertyChange("taskStatus", null, null); + valueChanged("taskStatus"); } public long getStartTime() { @@ -62,7 +96,7 @@ public long getStartTime() { public void setStartTime(long startTime) { this.startTime = startTime; - pcs.firePropertyChange("startTime", null, null); + valueChanged("startTime"); } public long getEndTime() { @@ -71,7 +105,7 @@ public long getEndTime() { public void setEndTime(long endTime) { this.endTime = endTime; - pcs.firePropertyChange("endTime", null, null); + valueChanged("endTime"); } public long getDuration() { @@ -80,7 +114,7 @@ public long getDuration() { public void setDuration(long duration) { this.duration = duration; - pcs.firePropertyChange("duration", null, null); + valueChanged("duration"); } public String getError() { @@ -89,7 +123,7 @@ public String getError() { public void setError(String error) { this.error = error; - pcs.firePropertyChange("error", null, null); + valueChanged("error"); } public JobCounters getJobCounters() { @@ -98,6 +132,6 @@ public JobCounters getJobCounters() { public void setJobCounters(JobCounters jobCounters) { this.jobCounters = jobCounters; - pcs.firePropertyChange("jobCounters", null, null); + valueChanged("jobCounters"); } } diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/DefaultJHFInputStreamCallback.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/DefaultJHFInputStreamCallback.java index 262054e9f6..28ebf4efe0 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/DefaultJHFInputStreamCallback.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/DefaultJHFInputStreamCallback.java @@ -21,6 +21,7 @@ import org.apache.eagle.jpm.mr.history.MRHistoryJobConfig; import org.apache.eagle.jpm.mr.history.parser.JHFParserBase; import org.apache.eagle.jpm.mr.history.parser.JHFParserFactory; +import org.apache.eagle.jpm.util.MRJobTagName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +47,7 @@ public void onInputStream(InputStream jobFileInputStream, org.apache.hadoop.conf @SuppressWarnings("serial") Map baseTags = new HashMap() { { - put("site", appConfig.getJobHistoryEndpointConfig().site); + put(MRJobTagName.SITE.toString(), appConfig.getJobHistoryEndpointConfig().site); } }; diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index 5d8d1ba8bc..c15bb31c1a 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -81,6 +81,7 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl private long sumReduceTaskDuration; private JobCounterMetricsGenerator jobCounterMetricsGenerator; + private JobSuggestionListener jobSuggestionListener; private MRHistoryJobConfig appConfig; @@ -127,6 +128,10 @@ public JHFEventReaderBase(Map baseTags, Configuration configurat this.appConfig = appConfig; this.jobCounterMetricsGenerator = new JobCounterMetricsGenerator(appConfig.getEagleServiceConfig()); + this.jobSuggestionListener = new JobSuggestionListener(); + if (this.configuration != null) { + this.jobSuggestionListener.jobConfigCreated(configuration); + } } public void register(HistoryJobEntityLifecycleListener lifecycleListener) { @@ -179,7 +184,7 @@ private void setJobType(String jobType) { this.jobType = jobType; } - protected void handleJob(EventType eventType, Map values, Object totalCounters) throws Exception { + protected void handleJob(EventType eventType, Map values, Object totalCounters, Object mapCounters, Object reduceCounters) throws Exception { String id = values.get(Keys.JOBID); if (jobId == null) { @@ -300,8 +305,15 @@ protected void handleJob(EventType eventType, Map values, Object t this.jobCounterMetricsGenerator.setBaseTags(jobExecutionEntity.getTags()); formatDiagnostics(values.get(Keys.DIAGNOSTICS)); - entityCreated(jobExecutionEntity); + jobSuggestionListener.jobEntityCreated(jobExecutionEntity); + + if (configuration != null && totalCounters != null) { + JobCounters parsedTotalCounters = parseCounters(totalCounters); + JobCounters parsedMapCounters = parseCounters(mapCounters); + JobCounters parsedReduceCounters = parseCounters(reduceCounters); + jobSuggestionListener.jobCountersCreated(parsedTotalCounters, parsedMapCounters, parsedReduceCounters); + } } } @@ -415,6 +427,7 @@ protected void handleTask(RecordTypes recType, EventType eventType, final Map> groups = new HashMap<>(); JhCounters counters = (JhCounters) value; diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java new file mode 100644 index 0000000000..a7cca181c2 --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.mr.history.parser; + +import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.mr.historyentity.JobBaseAPIEntity; +import org.apache.eagle.jpm.mr.historyentity.JobExecutionAPIEntity; +import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; +import org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity; +import org.apache.eagle.jpm.util.MRJobTagName; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; +import org.apache.eagle.jpm.mr.history.parser.JHFEventReaderBase.Keys; +import org.apache.hadoop.conf.Configuration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * JobEventCounterListener provides an interface to add job/task counter analyzers + */ +public class JobSuggestionListener implements HistoryJobEntityCreationListener { + private static final Logger LOG = LoggerFactory.getLogger(JobSuggestionListener.class); + + private MapReduceAnalyzerEntity info = new MapReduceAnalyzerEntity(); + private Configuration jobConf; + + @Override + public void jobEntityCreated(JobBaseAPIEntity entity) throws Exception { + if (entity instanceof TaskExecutionAPIEntity) { + info.getTasksMap().put(entity.getTags().get(Keys.TASKID), (TaskExecutionAPIEntity) entity); + } else if (entity instanceof TaskAttemptExecutionAPIEntity) { + info.getCompletedTaskAttemptsMap().put(entity.getTags().get(Keys.TASK_ATTEMPT_ID), (TaskAttemptExecutionAPIEntity) entity); + } else if (entity instanceof JobExecutionAPIEntity) { + JobExecutionAPIEntity jobExecutionAPIEntity = (JobExecutionAPIEntity) entity; + info.setCurrentState(jobExecutionAPIEntity.getCurrentState()); + info.setStartTime(jobExecutionAPIEntity.getStartTime()); + info.setEndTime(jobExecutionAPIEntity.getEndTime()); + info.setDurationTime(jobExecutionAPIEntity.getDurationTime()); + info.setUserId(jobExecutionAPIEntity.getTags().get(MRJobTagName.USER.toString())); + info.setJobId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_ID.toString())); + info.setJobDefId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOD_DEF_ID.toString())); + info.setSiteId(jobExecutionAPIEntity.getTags().get(MRJobTagName.SITE.toString())); + info.jobName = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_NAME.toString()); + info.jobQueueName = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_QUEUE.toString()); + info.jobType = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_TYPE.toString()); + info.finishedMaps = jobExecutionAPIEntity.getNumFinishedMaps(); + info.finishedReduces = jobExecutionAPIEntity.getNumFinishedReduces(); + info.failedReduces = jobExecutionAPIEntity.getNumFailedReduces(); + info.failedMaps = jobExecutionAPIEntity.getNumFailedMaps(); + info.totalMaps = jobExecutionAPIEntity.getNumTotalMaps(); + info.totalReduces = jobExecutionAPIEntity.getNumTotalReduces(); + } + } + + public void jobConfigCreated(Configuration configuration) { + this.jobConf = configuration; + } + + public void jobCountersCreated(JobCounters totalCounters, JobCounters mapCounters, JobCounters reduceCounters) { + info.totalCounters = totalCounters; + info.reduceCounters = reduceCounters; + info.mapCounters = mapCounters; + } + + @Override + public void flush() throws Exception { + + } + + public static class MapReduceAnalyzerEntity extends AnalyzerEntity { + String jobName; + String jobQueueName; + String jobType; + int totalMaps; + int totalReduces; + int failedMaps; + int failedReduces; + int finishedMaps; + int finishedReduces; + JobCounters totalCounters; + JobCounters mapCounters; + JobCounters reduceCounters; + Map tasksMap; + Map completedTaskAttemptsMap; + + public MapReduceAnalyzerEntity() { + this.setEndTime(-1); + this.setStartTime(-1); + finishedMaps = finishedReduces = 0; + jobName = jobQueueName = ""; + tasksMap = new HashMap<>(); + completedTaskAttemptsMap = new HashMap<>(); + } + + public String getJobName() { + return jobName; + } + + public String getJobQueueName() { + return jobQueueName; + } + + public String getJobType() { + return jobType; + } + + public int getTotalMaps() { + return totalMaps; + } + + public int getTotalReduces() { + return totalReduces; + } + + public int getFailedMaps() { + return failedMaps; + } + + public int getFailedReduces() { + return failedReduces; + } + + public int getFinishedMaps() { + return finishedMaps; + } + + public int getFinishedReduces() { + return finishedReduces; + } + + public JobCounters getTotalCounters() { + return totalCounters; + } + + public JobCounters getMapCounters() { + return mapCounters; + } + + public JobCounters getReduceCounters() { + return reduceCounters; + } + + public Map getTasksMap() { + return tasksMap; + } + + public Map getCompletedTaskAttemptsMap() { + return completedTaskAttemptsMap; + } + } + +} From 2e756c88579155ea1b8597fcfe2a9b4756427f75 Mon Sep 17 00:00:00 2001 From: Qingwen Zhao Date: Tue, 17 Jan 2017 14:34:02 +0800 Subject: [PATCH 10/22] add job suggestion processors --- eagle-jpm/eagle-jpm-analyzer/pom.xml | 5 + .../eagle/jpm/analyzer/AnalyzerEntity.java | 2 +- .../apache/eagle/jpm/analyzer/Evaluator.java | 4 +- .../apache/eagle/jpm/analyzer/Processor.java | 4 +- .../meta/model/MapReduceAnalyzerEntity.java | 166 +++++++++++++ .../mr/suggestion/JobSuggestionEvaluator.java | 35 ++- .../MapReduceCompressionSettingAnalyzer.java | 82 +++++++ .../suggestion/MapReduceDataSkewAnalyzer.java | 62 +++++ .../suggestion/MapReduceGCTimeAnalyzer.java | 66 ++++++ .../MapReduceJobSuggestionContext.java | 221 ++++++++++++++++++ .../MapReduceQueueResourceAnalyzer.java | 83 +++++++ .../mr/suggestion/MapReduceSpillAnalyzer.java | 115 +++++++++ .../MapReduceSplitSettingAnalyzer.java | 43 ++++ .../suggestion/MapReduceTaskNumAnalyzer.java | 163 +++++++++++++ .../history/parser/JobSuggestionListener.java | 112 ++------- 15 files changed, 1055 insertions(+), 108 deletions(-) create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java create mode 100644 eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java diff --git a/eagle-jpm/eagle-jpm-analyzer/pom.xml b/eagle-jpm/eagle-jpm-analyzer/pom.xml index 50fb803a70..07f5766ced 100644 --- a/eagle-jpm/eagle-jpm-analyzer/pom.xml +++ b/eagle-jpm/eagle-jpm-analyzer/pom.xml @@ -45,6 +45,11 @@ eagle-jpm-util ${project.version} + + org.apache.eagle + eagle-jpm-entity + ${project.version} + org.apache.eagle eagle-app-base diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java index 98e0f97ed1..a600f79ee3 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java @@ -22,7 +22,7 @@ /** * will refactor later if other types of job needs this. - * AnalyzerEntity for each job needed to be analysised + * AnalyzerEntity for each job needed to be analyzed */ public class AnalyzerEntity { private String jobDefId; diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java index 6617916ff5..17fc119a23 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java @@ -19,6 +19,6 @@ import org.apache.eagle.jpm.analyzer.publisher.Result; -public interface Evaluator { - Result.EvaluatorResult evaluate(AnalyzerEntity analyzerEntity); +public interface Evaluator { + Result.EvaluatorResult evaluate(T analyzerEntity); } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java index d5a8a74ea1..0e8b417757 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java @@ -19,6 +19,6 @@ import org.apache.eagle.jpm.analyzer.publisher.Result; -public interface Processor { - Result.ProcessorResult process(AnalyzerEntity jobAnalysisEntity); +public interface Processor { + Result.ProcessorResult process(T jobAnalysisEntity); } \ No newline at end of file diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java new file mode 100644 index 0000000000..2dc26fc27d --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.meta.model; + +import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; +import org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; + +import java.util.HashMap; +import java.util.Map; + +public class MapReduceAnalyzerEntity extends AnalyzerEntity { + private String jobName; + private String jobQueueName; + private String jobType; + private int totalMaps; + private int totalReduces; + private int failedMaps; + private int failedReduces; + private int finishedMaps; + private int finishedReduces; + private JobCounters totalCounters; + private JobCounters mapCounters; + private JobCounters reduceCounters; + private Map tasksMap; + private Map completedTaskAttemptsMap; + + public MapReduceAnalyzerEntity() { + this.setEndTime(-1); + this.setStartTime(-1); + finishedMaps = finishedReduces = 0; + jobName = jobQueueName = ""; + tasksMap = new HashMap<>(); + completedTaskAttemptsMap = new HashMap<>(); + } + + public String getJobName() { + return jobName; + } + + public String getJobQueueName() { + return jobQueueName; + } + + public String getJobType() { + return jobType; + } + + public int getTotalMaps() { + return totalMaps; + } + + public int getTotalReduces() { + return totalReduces; + } + + public int getFailedMaps() { + return failedMaps; + } + + public int getFailedReduces() { + return failedReduces; + } + + public int getFinishedMaps() { + return finishedMaps; + } + + public int getFinishedReduces() { + return finishedReduces; + } + + public JobCounters getTotalCounters() { + return totalCounters; + } + + public JobCounters getMapCounters() { + return mapCounters; + } + + public JobCounters getReduceCounters() { + return reduceCounters; + } + + public Map getTasksMap() { + return tasksMap; + } + + public Map getCompletedTaskAttemptsMap() { + return completedTaskAttemptsMap; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public void setJobQueueName(String jobQueueName) { + this.jobQueueName = jobQueueName; + } + + public void setJobType(String jobType) { + this.jobType = jobType; + } + + public void setTotalMaps(int totalMaps) { + this.totalMaps = totalMaps; + } + + public void setTotalReduces(int totalReduces) { + this.totalReduces = totalReduces; + } + + public void setFailedMaps(int failedMaps) { + this.failedMaps = failedMaps; + } + + public void setFailedReduces(int failedReduces) { + this.failedReduces = failedReduces; + } + + public void setFinishedMaps(int finishedMaps) { + this.finishedMaps = finishedMaps; + } + + public void setFinishedReduces(int finishedReduces) { + this.finishedReduces = finishedReduces; + } + + public void setTotalCounters(JobCounters totalCounters) { + this.totalCounters = totalCounters; + } + + public void setMapCounters(JobCounters mapCounters) { + this.mapCounters = mapCounters; + } + + public void setReduceCounters(JobCounters reduceCounters) { + this.reduceCounters = reduceCounters; + } + + public void setTasksMap(Map tasksMap) { + this.tasksMap = tasksMap; + } + + public void setCompletedTaskAttemptsMap(Map completedTaskAttemptsMap) { + this.completedTaskAttemptsMap = completedTaskAttemptsMap; + } + + +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java index 4f088dc9cb..8afeadbad7 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java @@ -20,14 +20,18 @@ import com.typesafe.config.Config; import org.apache.eagle.jpm.analyzer.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.Evaluator; +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.util.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; -public class JobSuggestionEvaluator implements Evaluator, Serializable { +public class JobSuggestionEvaluator implements Evaluator, Serializable { private static final Logger LOG = LoggerFactory.getLogger(JobSuggestionEvaluator.class); private Config config; @@ -36,14 +40,37 @@ public JobSuggestionEvaluator(Config config) { this.config = config; } + private List loadProcessors(MapReduceJobSuggestionContext context) { + List processors = new ArrayList<>(); + processors.add(new MapReduceCompressionSettingAnalyzer(context)); + processors.add(new MapReduceSplitSettingAnalyzer(context)); + processors.add(new MapReduceDataSkewAnalyzer(context)); + processors.add(new MapReduceGCTimeAnalyzer(context)); + processors.add(new MapReduceSpillAnalyzer(context)); + processors.add(new MapReduceTaskNumAnalyzer(context)); + processors.add(new MapReduceQueueResourceAnalyzer(context)); + + return processors; + } + @Override - public Result.EvaluatorResult evaluate(AnalyzerEntity mrJobEntity) { - if (mrJobEntity.getCurrentState().equalsIgnoreCase(Constants.JobState.RUNNING.toString())) { + public Result.EvaluatorResult evaluate(MapReduceAnalyzerEntity analyzerEntity) { + if (analyzerEntity.getCurrentState().equalsIgnoreCase(Constants.JobState.RUNNING.toString())) { + return null; + } + + MapReduceJobSuggestionContext jobContext = MapReduceJobSuggestionContext.getInstance().buildContext(); + if (jobContext.getNumMaps() == 0) { return null; } Result.EvaluatorResult result = new Result.EvaluatorResult(); - //TODO + for (Processor processor : loadProcessors(jobContext)) { + Result.ProcessorResult processorResult = processor.process(analyzerEntity); + if (processorResult != null) { + result.addProcessorResult(processor.getClass(), processorResult); + } + } return result; } } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java new file mode 100644 index 0000000000..d8870b7bd9 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; + +import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_OUTPUT_COMPRESS; +import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC; +import static org.apache.hadoop.mapreduce.MRJobConfig.NUM_REDUCES; +import static org.apache.hadoop.mapreduce.MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR; + +public class MapReduceCompressionSettingAnalyzer implements Processor { + + private MapReduceJobSuggestionContext context; + + public MapReduceCompressionSettingAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + StringBuilder sb = new StringBuilder(); + JobConf jobconf = new JobConf(context.getJobconf()); + if (jobconf.getLong(NUM_REDUCES, 0) > 0) { + if (jobconf.getCompressMapOutput() == false) { + sb.append("Please set " + MAP_OUTPUT_COMPRESS + + " to true to reduce network IO.\n"); + } else { + String codecClassName = jobconf + .get(MAP_OUTPUT_COMPRESS_CODEC); + if (!(codecClassName.endsWith("LzoCodec") || codecClassName + .endsWith("SnappyCodec"))) { + sb.append("Best practice: use LzoCodec or SnappyCodec for " + + MAP_OUTPUT_COMPRESS_CODEC); + sb.append("\n"); + } + } + } + + if (!jobconf.getBoolean(FileOutputFormat.COMPRESS, false)) { + sb.append("Please set " + FileOutputFormat.COMPRESS + + " to true to reduce disk usage and network IO.\n"); + } else { + String codecName = jobconf.get(FileOutputFormat.COMPRESS_CODEC, ""); + String outputFileFormat = jobconf.get(OUTPUT_FORMAT_CLASS_ATTR, ""); + + if ((codecName.endsWith("GzipCodec") || + codecName.endsWith("SnappyCodec") || + codecName.endsWith("DefaultCodec")) + && outputFileFormat.endsWith("TextOutputFormat")) { + sb.append("Best practice: don't use Gzip/Snappy/DefaultCodec with TextOutputFormat"); + sb.append(" as this will cause the output files to be unsplittable. "); + sb.append("Please use LZO instead or "); + sb.append("use a container file format such as SequenceFileOutputFormat.\n"); + } + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + } + return null; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java new file mode 100644 index 0000000000..ebe2f4f170 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; + +public class MapReduceDataSkewAnalyzer implements Processor { + private MapReduceJobSuggestionContext context; + public MapReduceDataSkewAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + TaskAttemptExecutionAPIEntity worstReduce = context.getWorstReduce(); + if (context.getNumReduces() == 0 || worstReduce == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + try { + long worstTime = (worstReduce.getEndTime() - worstReduce + .getShuffleFinishTime()) / 1000; + if (worstTime - context.getAvgReduceTimeInSec() > 30 * 60 ) { + long avgInputs = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS) + / context.getNumReduces(); + long worstInputs = worstReduce.getJobCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS); + + if (worstInputs / 5 > avgInputs) { + sb.append("Data skew detected in reducers. The average reduce time is " + + context.getAvgReduceTimeInSec()); + sb.append(" seconds, the worst reduce time is " + worstTime); + sb.append(" seconds. Please investigate this problem to improve your job performance.\n"); + } + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + } + } catch (NullPointerException e) { + // When job failed there may not have counters, so just ignore it + } + return null; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java new file mode 100644 index 0000000000..6edfda4bd3 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; + +import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_JAVA_OPTS; +import static org.apache.hadoop.mapreduce.MRJobConfig.REDUCE_JAVA_OPTS; + +public class MapReduceGCTimeAnalyzer implements Processor { + private MapReduceJobSuggestionContext context; + + public MapReduceGCTimeAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + StringBuilder sb = new StringBuilder(); + try { + long mapGCTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS); + long mapCPUTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS); + + if (mapGCTime > mapCPUTime * 0.1) { + sb.append("Map GC_TIME_MILLIS took too long. Please increase mapper memory via -D" + + MAP_JAVA_OPTS); + sb.append(", or optimize your mapper class.\n"); + } + + if (context.getNumReduces() > 0) { + long reduceGCTime = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS); + long reduceCPUTime = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS); + if (reduceGCTime > reduceCPUTime * 0.1) { + sb.append("Reduce GC_TIME_MILLIS took too long. Please increase memory for reduce via -D" + + REDUCE_JAVA_OPTS); + sb.append(", or optimize your reducer class.\n"); + } + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + } + } catch (NullPointerException e) { + // When job failed there may not have counters, so just ignore it + } + return null; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java new file mode 100644 index 0000000000..1c87f60eb5 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.common.DateTimeUtil; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; +import org.apache.eagle.jpm.util.MRJobTagName; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.TaskType; + +import java.util.regex.Pattern; + +import static org.apache.eagle.jpm.util.jobcounter.JobCounters.CounterName.MAP_OUTPUT_BYTES; +import static org.apache.eagle.jpm.util.jobcounter.JobCounters.CounterName.MAP_OUTPUT_RECORDS; +import static org.apache.hadoop.mapreduce.MRJobConfig.NUM_MAPS; +import static org.apache.hadoop.mapreduce.MRJobConfig.NUM_REDUCES; + +public class MapReduceJobSuggestionContext { + + private JobConf jobconf; + private MapReduceAnalyzerEntity job; + + private long numMaps; + private long numReduces; + + private long avgMapTimeInSec; + private long avgReduceTimeInSec; + private long avgShuffleTimeInSec; + private TaskAttemptExecutionAPIEntity worstMap; + private TaskAttemptExecutionAPIEntity worstReduce; + private TaskAttemptExecutionAPIEntity worstShuffle; + private TaskAttemptExecutionAPIEntity lastMap; + private TaskAttemptExecutionAPIEntity lastReduce; + private TaskAttemptExecutionAPIEntity lastShuffle; + private TaskAttemptExecutionAPIEntity firstMap; + private TaskAttemptExecutionAPIEntity firstReduce; + private TaskAttemptExecutionAPIEntity firstShuffle; + + private long minMapSpillMemBytes; + + public static final Pattern MAX_HEAP_PATTERN = Pattern.compile("-Xmx([0-9]+)([kKmMgG]?)"); + public final long _1MB = 1024 * 1024l; + public final long _256MB = 256 * _1MB; + public final long _512MB = 512 * _1MB; + public final long _1GB = 1024 * _1MB; + + private static Boolean initialized = false; + + private MapReduceJobSuggestionContext() {} + private static MapReduceJobSuggestionContext instance; + public static MapReduceJobSuggestionContext getInstance() { + if (instance == null) { + synchronized (MapReduceJobSuggestionContext.class) { + if (instance == null) { + instance = new MapReduceJobSuggestionContext(); + } + } + } + return instance; + } + + public MapReduceJobSuggestionContext buildContext() { + if (!initialized) { + avgMapTimeInSec = avgReduceTimeInSec = avgShuffleTimeInSec = 0; + numMaps = jobconf.getLong(NUM_MAPS, 0); + numReduces = jobconf.getLong(NUM_REDUCES, 0); + + for (TaskAttemptExecutionAPIEntity attempt : job.getCompletedTaskAttemptsMap().values()) { + if (TaskType.MAP.toString().equalsIgnoreCase(getTaskType(attempt))) { + long mapTime = attempt.getEndTime() - attempt.getStartTime(); + avgMapTimeInSec += mapTime; + if (firstMap == null || firstMap.getStartTime() > attempt.getStartTime()) { + firstMap = attempt; + } + if (lastMap == null || lastMap.getEndTime() < attempt.getEndTime()) { + lastMap = attempt; + } + if (worstMap == null || (worstMap.getEndTime() - worstMap.getStartTime()) < mapTime) { + worstMap = attempt; + } + long tmpMem = getMinimumIOSortMemory(attempt); + if (tmpMem > minMapSpillMemBytes) { + minMapSpillMemBytes = tmpMem; + } + } else if (TaskType.REDUCE.toString().equalsIgnoreCase(getTaskType(attempt))) { + long shuffleTime = attempt.getShuffleFinishTime() - attempt.getStartTime(); + avgShuffleTimeInSec += shuffleTime; + if (firstShuffle == null || firstShuffle.getStartTime() > attempt.getStartTime()) { + firstShuffle = attempt; + } + if (lastShuffle == null || lastShuffle.getShuffleFinishTime() < attempt.getShuffleFinishTime()) { + lastShuffle = attempt; + } + if (worstShuffle == null || (worstShuffle.getShuffleFinishTime() - worstShuffle.getStartTime()) < shuffleTime) { + worstShuffle = attempt; + } + + long reduceTime = attempt.getEndTime() - attempt.getShuffleFinishTime(); + avgReduceTimeInSec += reduceTime; + if (firstReduce == null || firstReduce.getStartTime() > attempt.getStartTime()) { + firstReduce = attempt; + } + if (lastReduce == null || lastReduce.getEndTime() < attempt.getEndTime()) { + lastReduce = attempt; + } + if (worstReduce == null || (worstReduce.getEndTime() - worstReduce.getShuffleFinishTime()) < reduceTime) { + worstReduce = attempt; + } + } + } + if (numMaps > 0) { + avgMapTimeInSec = avgMapTimeInSec / numMaps / DateTimeUtil.ONESECOND; + } + if (numReduces > 0) { + avgReduceTimeInSec = avgReduceTimeInSec / numReduces / DateTimeUtil.ONESECOND; + avgShuffleTimeInSec = avgShuffleTimeInSec / numReduces / DateTimeUtil.ONESECOND; + } + initialized = true; + } + return this; + } + + private String getTaskType(TaskAttemptExecutionAPIEntity taskAttemptInfo) { + return taskAttemptInfo.getTags().get(MRJobTagName.TASK_TYPE); + } + + /** + * 16 is the default index size + * @param attempt + * @return + */ + private long getMinimumIOSortMemory(TaskAttemptExecutionAPIEntity attempt) { + long records = attempt.getJobCounters().getCounterValue(MAP_OUTPUT_RECORDS); + long outputBytes = attempt.getJobCounters().getCounterValue(MAP_OUTPUT_BYTES); + return outputBytes + records * 16; + } + + public JobConf getJobconf() { + return jobconf; + } + + public MapReduceAnalyzerEntity getJob() { + return job; + } + + public long getNumMaps() { + return numMaps; + } + + public long getNumReduces() { + return numReduces; + } + + public long getAvgMapTimeInSec() { + return avgMapTimeInSec; + } + + public long getAvgReduceTimeInSec() { + return avgReduceTimeInSec; + } + + public long getAvgShuffleTimeInSec() { + return avgShuffleTimeInSec; + } + + public TaskAttemptExecutionAPIEntity getWorstMap() { + return worstMap; + } + + public TaskAttemptExecutionAPIEntity getWorstReduce() { + return worstReduce; + } + + public TaskAttemptExecutionAPIEntity getWorstShuffle() { + return worstShuffle; + } + + public TaskAttemptExecutionAPIEntity getLastMap() { + return lastMap; + } + + public TaskAttemptExecutionAPIEntity getLastReduce() { + return lastReduce; + } + + public TaskAttemptExecutionAPIEntity getLastShuffle() { + return lastShuffle; + } + + public TaskAttemptExecutionAPIEntity getFirstMap() { + return firstMap; + } + + public TaskAttemptExecutionAPIEntity getFirstReduce() { + return firstReduce; + } + + public TaskAttemptExecutionAPIEntity getFirstShuffle() { + return firstShuffle; + } + + public long getMinMapSpillMemBytes() { + return minMapSpillMemBytes; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java new file mode 100644 index 0000000000..e2d15a3e51 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.common.DateTimeUtil; +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Criterion: (TimeElapsed / (numTasks / 500 * avgTaskTime)) > 20 + */ +public class MapReduceQueueResourceAnalyzer implements Processor { + private static final Logger LOG = LoggerFactory.getLogger(MapReduceQueueResourceAnalyzer.class); + + private MapReduceJobSuggestionContext context; + + public MapReduceQueueResourceAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + try { + String userName = context.getJob().getUserId(); + TaskAttemptExecutionAPIEntity lastMap = context.getLastMap(); + TaskAttemptExecutionAPIEntity firstMap = context.getFirstMap(); + TaskAttemptExecutionAPIEntity lastReduce = context.getLastReduce(); + TaskAttemptExecutionAPIEntity firstShuffle = context.getFirstShuffle(); + + if (checkBatchUser(userName) && lastMap != null && firstMap != null) { + StringBuilder sb = new StringBuilder(); + + long tasksPerTime = 500; // better get it from RM + long mapPhaseTimeInSec = (lastMap.getEndTime() - firstMap.getStartTime()) / DateTimeUtil.ONESECOND; + if (mapPhaseTimeInSec > context.getAvgMapTimeInSec() + * ((context.getNumMaps() + tasksPerTime - 1) / tasksPerTime) * 20) { + sb.append("There appears to have been resource contention during the map phase of your job. Please ask for more resources if your job is SLA-bound,"); + sb.append(" or submit your job when the cluster is less busy.\n"); + } + + if (context.getNumReduces() > 0 && lastReduce != null && firstShuffle != null) { + long reducePhaseTimeInSec = (lastReduce.getEndTime() - firstShuffle.getStartTime()) / DateTimeUtil.ONESECOND; + if (reducePhaseTimeInSec > context.getAvgReduceTimeInSec() + * ((context.getNumReduces() + tasksPerTime - 1) / tasksPerTime) * 20) { + sb.append("Seems there was resource contention when your job in reduce phase, please ask for more resource if your job is SLA enabled,"); + sb.append(" or submit your job when the cluster is less busy.\n"); + } + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + } + } + } catch (Exception e) { + LOG.warn(e.getMessage(), e); + } + return null; + } + + protected boolean checkBatchUser(String userName) { + return userName.startsWith("b_"); + } +} \ No newline at end of file diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java new file mode 100644 index 0000000000..68ed88114c --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.commons.io.FileUtils; +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; + +import java.util.regex.Matcher; + +import static org.apache.eagle.jpm.analyzer.mr.suggestion.MapReduceJobSuggestionContext.MAX_HEAP_PATTERN; +import static org.apache.hadoop.mapreduce.MRJobConfig.IO_SORT_MB; +import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_JAVA_OPTS; +import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_SORT_SPILL_PERCENT; + +/** + * Check whether spilled more than once, if true, find out the minimum value of the memory to hold all the data, + * based on that value, find out how much memory need for heap size. + */ +public class MapReduceSpillAnalyzer implements Processor { + + private MapReduceJobSuggestionContext context; + + public MapReduceSpillAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + StringBuilder sb = new StringBuilder(); + long outputRecords = 0L; // Map output records + long spillRecords = 0L; // Spilled Records + try { + outputRecords = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.MAP_OUTPUT_RECORDS); + spillRecords = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS); + + if (outputRecords < spillRecords) { + sb.append("Total Map output records: " + outputRecords); + sb.append(" Total Spilled Records: " + spillRecords); + sb.append(". Please set"); + + long minMapSpillMemBytes = context.getMinMapSpillMemBytes(); + double spillPercent = context.getJobconf().getDouble(MAP_SORT_SPILL_PERCENT, 0.8); + if (minMapSpillMemBytes > context._512MB * spillPercent) { + if (Math.abs(1.0 - spillPercent) > 0.001) { + sb.append(" -D" + MAP_SORT_SPILL_PERCENT + "=1"); + } + } else { + minMapSpillMemBytes /= spillPercent; + } + + minMapSpillMemBytes = (minMapSpillMemBytes / FileUtils.ONE_MB + 10) / 10 * 10; + if (minMapSpillMemBytes >= 2047 ) { + sb.append(" Please reduce the block size of the input files and make sure they are splittable."); + } else { + sb.append(" -D" + IO_SORT_MB + "=" + minMapSpillMemBytes); + long heapSize = getMaxHeapSize(context.getJobconf().get(MAP_JAVA_OPTS)); + if (heapSize < 3 * minMapSpillMemBytes) { + sb.append(" -D" + MAP_JAVA_OPTS + "=-Xmx" + + (long) (minMapSpillMemBytes * 3 + 1024) / 1024 * 1024 + "M"); + } + } + sb.append(" to avoid spilled records.\n"); + } + + + long reduceInputRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS); + spillRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS); + if (reduceInputRecords < spillRecords) { + sb.append("Please add more memory (mapreduce.reduce.java.opts) to avoid spilled records."); + sb.append(" Total Reduce input records: " + reduceInputRecords); + sb.append(" Total Spilled Records: " + spillRecords); + sb.append("\n"); + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + } + } catch (NullPointerException e) { + //When job failed there may not have counters, so just ignore it + } + return null; + } + + private static long getMaxHeapSize(String s) { + Matcher m = MAX_HEAP_PATTERN.matcher(s); + long val = 0; + if (m.find()) { + val = Long.parseLong(m.group(1)); + if ("k".equalsIgnoreCase(m.group(2))) { + val /= 1024; + } else if ("g".equalsIgnoreCase(m.group(2))) { + val *= 1024; + } + } + return val; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java new file mode 100644 index 0000000000..8dbf41c30a --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; + +public class MapReduceSplitSettingAnalyzer implements Processor { + + private MapReduceJobSuggestionContext context; + + public MapReduceSplitSettingAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + StringBuilder sb = new StringBuilder(); + if (context.getJobconf().getLong(FileInputFormat.SPLIT_MINSIZE, 0) > 1) { + sb.append("Best practice: don't set " + FileInputFormat.SPLIT_MINSIZE); + sb.append(", because it may lower data locality, hence maps will run slower.\n"); + return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + } + return null; + } +} diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java new file mode 100644 index 0000000000..4a0e89d581 --- /dev/null +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.analyzer.mr.suggestion; + +import org.apache.commons.io.FileUtils; +import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.Result; +import org.apache.eagle.jpm.util.jobcounter.JobCounters; + +public class MapReduceTaskNumAnalyzer implements Processor { + + private MapReduceJobSuggestionContext context; + + public MapReduceTaskNumAnalyzer(MapReduceJobSuggestionContext context) { + this.context = context; + } + + @Override + public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { + StringBuilder sb = new StringBuilder(); + try { + long numReduces = context.getNumReduces(); + long numMaps = context.getNumMaps(); + if (numReduces > 0) { + long avgReduceTime = context.getAvgReduceTimeInSec(); + long avgShuffleTime = context.getAvgShuffleTimeInSec(); + long avgShuffleBytes = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_SHUFFLE_BYTES) + / numReduces; + long avgReduceOutput = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.HDFS_BYTES_WRITTEN) + / numReduces; + long avgReduceTotalTime = avgShuffleTime + avgReduceTime; + + long suggestReduces = 0; + StringBuilder tmpsb = new StringBuilder(); + + if (avgShuffleBytes < context._256MB && avgReduceTotalTime < 300 + && avgReduceOutput < context._256MB && numReduces > 1) { + tmpsb.append("average reduce input bytes is: "); + tmpsb.append(FileUtils.byteCountToDisplaySize(avgShuffleBytes) + ", "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } else if (avgShuffleBytes > 10 * FileUtils.ONE_GB && avgReduceTotalTime > 1800) { + tmpsb.append("average reduce input bytes is: "); + tmpsb.append(FileUtils.byteCountToDisplaySize(avgShuffleBytes) + ", "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } + + if (avgReduceTotalTime < 60 && numReduces > 1) { + tmpsb.append("average reduce time is only " + + (context.getAvgShuffleTimeInSec() + context.getAvgReduceTimeInSec()) + " seconds, "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } else if (avgReduceTotalTime > 3600 && avgReduceTime > 1800) { + tmpsb.append("average reduce time is " + + (avgShuffleTime + avgReduceTime) + " seconds, "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } + + if (avgReduceOutput < 10 * FileUtils.ONE_MB && avgReduceTime < 300 + && avgShuffleBytes < 2 * FileUtils.ONE_GB && numReduces > 1) { + tmpsb.append(" average reduce output is only "); + tmpsb.append(FileUtils.byteCountToDisplaySize(avgReduceOutput) + ", "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } else if (avgReduceOutput > 10 * FileUtils.ONE_GB && avgReduceTime > 1800) { + tmpsb.append(" average reduce output is "); + tmpsb.append(FileUtils.byteCountToDisplaySize(avgReduceOutput) + ", "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } + + if (suggestReduces > 0) { + sb.append("Best practice: "); + sb.append(tmpsb.toString()); + sb.append("please consider "); + if (suggestReduces > numReduces) { + sb.append("increasing the "); + } else { + sb.append("decreasing the "); + } + sb.append("reducer number. You could try -Dmapreduce.job.reduces=" + suggestReduces + "\n"); + } + } + + // map numbers + long avgMapInput = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.HDFS_BYTES_READ) + / numMaps; + long avgMapTime = context.getAvgMapTimeInSec(); + if (avgMapInput < 5 * FileUtils.ONE_MB && avgMapTime < 30 && numMaps > 1) { + sb.append("Best practice: average map input bytes only have "); + sb.append(FileUtils.byteCountToDisplaySize(avgMapInput)); + sb.append(". Please reduce the number of mappers by merging input files.\n"); + } else if (avgMapInput > FileUtils.ONE_GB) { + sb.append("Best practice: average map input bytes have "); + sb.append(FileUtils.byteCountToDisplaySize(avgMapInput)); + sb.append(". Please increase the number of mappers by using splittable compression, a container file format or a smaller block size.\n"); + } + + if (avgMapTime < 10 && numMaps > 1) { + sb.append("Best practice: average map time only have " + avgMapTime); + sb.append(" seconds. Please reduce the number of mappers by merging input files or by using a larger block size.\n"); + } else if (avgMapTime > 600 && avgMapInput < FileUtils.ONE_GB) { + sb.append("Best practice: average map time is " + avgMapInput); + sb.append(" seconds. Please increase the number of mappers by using splittable compression, a container file format or a smaller block size.\n"); + } + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + } + } catch (NullPointerException e) { + // When job failed there may not have counters, so just ignore it + } + return null; + } + + private long getReduceNum(long avgInputBytes, long avgOutputBytes, long avgTime) { + long newReduceNum = 1; + long tmpReduceNum; + + long numReduces = context.getNumReduces(); + tmpReduceNum = avgInputBytes * numReduces / (3 * FileUtils.ONE_GB); + if (tmpReduceNum > newReduceNum) { + newReduceNum = tmpReduceNum; + } + + tmpReduceNum = avgOutputBytes * numReduces / (2 * FileUtils.ONE_GB); + if (tmpReduceNum > newReduceNum) { + newReduceNum = tmpReduceNum; + } + + tmpReduceNum = avgTime * numReduces / (10 * 60); + if (tmpReduceNum > newReduceNum) { + newReduceNum = tmpReduceNum; + } + + return newReduceNum; + } + +} \ No newline at end of file diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java index a7cca181c2..24c5fb90e3 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java @@ -17,7 +17,7 @@ package org.apache.eagle.jpm.mr.history.parser; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; import org.apache.eagle.jpm.mr.historyentity.JobBaseAPIEntity; import org.apache.eagle.jpm.mr.historyentity.JobExecutionAPIEntity; import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; @@ -30,9 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Map; - /** * JobEventCounterListener provides an interface to add job/task counter analyzers */ @@ -58,15 +55,15 @@ public void jobEntityCreated(JobBaseAPIEntity entity) throws Exception { info.setJobId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_ID.toString())); info.setJobDefId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOD_DEF_ID.toString())); info.setSiteId(jobExecutionAPIEntity.getTags().get(MRJobTagName.SITE.toString())); - info.jobName = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_NAME.toString()); - info.jobQueueName = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_QUEUE.toString()); - info.jobType = jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_TYPE.toString()); - info.finishedMaps = jobExecutionAPIEntity.getNumFinishedMaps(); - info.finishedReduces = jobExecutionAPIEntity.getNumFinishedReduces(); - info.failedReduces = jobExecutionAPIEntity.getNumFailedReduces(); - info.failedMaps = jobExecutionAPIEntity.getNumFailedMaps(); - info.totalMaps = jobExecutionAPIEntity.getNumTotalMaps(); - info.totalReduces = jobExecutionAPIEntity.getNumTotalReduces(); + info.setJobName(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_NAME.toString())) ; + info.setJobQueueName(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_QUEUE.toString())); + info.setJobType(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_TYPE.toString())); + info.setFinishedMaps(jobExecutionAPIEntity.getNumFinishedMaps()); + info.setFinishedReduces(jobExecutionAPIEntity.getNumFinishedReduces()); + info.setFailedReduces(jobExecutionAPIEntity.getNumFailedReduces()); + info.setFailedMaps(jobExecutionAPIEntity.getNumFailedMaps()); + info.setTotalMaps(jobExecutionAPIEntity.getNumTotalMaps()); + info.setTotalReduces(jobExecutionAPIEntity.getNumTotalReduces()); } } @@ -75,96 +72,13 @@ public void jobConfigCreated(Configuration configuration) { } public void jobCountersCreated(JobCounters totalCounters, JobCounters mapCounters, JobCounters reduceCounters) { - info.totalCounters = totalCounters; - info.reduceCounters = reduceCounters; - info.mapCounters = mapCounters; + info.setTotalCounters(totalCounters); + info.setReduceCounters(reduceCounters); + info.setMapCounters(mapCounters); } @Override public void flush() throws Exception { } - - public static class MapReduceAnalyzerEntity extends AnalyzerEntity { - String jobName; - String jobQueueName; - String jobType; - int totalMaps; - int totalReduces; - int failedMaps; - int failedReduces; - int finishedMaps; - int finishedReduces; - JobCounters totalCounters; - JobCounters mapCounters; - JobCounters reduceCounters; - Map tasksMap; - Map completedTaskAttemptsMap; - - public MapReduceAnalyzerEntity() { - this.setEndTime(-1); - this.setStartTime(-1); - finishedMaps = finishedReduces = 0; - jobName = jobQueueName = ""; - tasksMap = new HashMap<>(); - completedTaskAttemptsMap = new HashMap<>(); - } - - public String getJobName() { - return jobName; - } - - public String getJobQueueName() { - return jobQueueName; - } - - public String getJobType() { - return jobType; - } - - public int getTotalMaps() { - return totalMaps; - } - - public int getTotalReduces() { - return totalReduces; - } - - public int getFailedMaps() { - return failedMaps; - } - - public int getFailedReduces() { - return failedReduces; - } - - public int getFinishedMaps() { - return finishedMaps; - } - - public int getFinishedReduces() { - return finishedReduces; - } - - public JobCounters getTotalCounters() { - return totalCounters; - } - - public JobCounters getMapCounters() { - return mapCounters; - } - - public JobCounters getReduceCounters() { - return reduceCounters; - } - - public Map getTasksMap() { - return tasksMap; - } - - public Map getCompletedTaskAttemptsMap() { - return completedTaskAttemptsMap; - } - } - } From 634bd709bf91b9182384f3a3cc9cad0ae0d68274 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Tue, 17 Jan 2017 16:14:49 +0800 Subject: [PATCH 11/22] update --- .../meta/model/MapReduceAnalyzerEntity.java | 9 ++ .../mr/suggestion/JobSuggestionEvaluator.java | 2 +- .../MapReduceJobSuggestionContext.java | 126 ++++++++---------- .../mr/history/parser/JHFEventReaderBase.java | 3 +- .../history/parser/JobSuggestionListener.java | 15 ++- .../parser/TaskAttemptCounterListener.java | 1 - 6 files changed, 82 insertions(+), 74 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java index 2dc26fc27d..5500d28ff7 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java @@ -21,6 +21,7 @@ import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; import org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity; import org.apache.eagle.jpm.util.jobcounter.JobCounters; +import org.apache.hadoop.conf.Configuration; import java.util.HashMap; import java.util.Map; @@ -40,6 +41,7 @@ public class MapReduceAnalyzerEntity extends AnalyzerEntity { private JobCounters reduceCounters; private Map tasksMap; private Map completedTaskAttemptsMap; + private Configuration jobConf; public MapReduceAnalyzerEntity() { this.setEndTime(-1); @@ -162,5 +164,12 @@ public void setCompletedTaskAttemptsMap(Map attempt.getStartTime()) { + firstMap = attempt; } - } - } - return instance; - } - - public MapReduceJobSuggestionContext buildContext() { - if (!initialized) { - avgMapTimeInSec = avgReduceTimeInSec = avgShuffleTimeInSec = 0; - numMaps = jobconf.getLong(NUM_MAPS, 0); - numReduces = jobconf.getLong(NUM_REDUCES, 0); - - for (TaskAttemptExecutionAPIEntity attempt : job.getCompletedTaskAttemptsMap().values()) { - if (TaskType.MAP.toString().equalsIgnoreCase(getTaskType(attempt))) { - long mapTime = attempt.getEndTime() - attempt.getStartTime(); - avgMapTimeInSec += mapTime; - if (firstMap == null || firstMap.getStartTime() > attempt.getStartTime()) { - firstMap = attempt; - } - if (lastMap == null || lastMap.getEndTime() < attempt.getEndTime()) { - lastMap = attempt; - } - if (worstMap == null || (worstMap.getEndTime() - worstMap.getStartTime()) < mapTime) { - worstMap = attempt; - } - long tmpMem = getMinimumIOSortMemory(attempt); - if (tmpMem > minMapSpillMemBytes) { - minMapSpillMemBytes = tmpMem; - } - } else if (TaskType.REDUCE.toString().equalsIgnoreCase(getTaskType(attempt))) { - long shuffleTime = attempt.getShuffleFinishTime() - attempt.getStartTime(); - avgShuffleTimeInSec += shuffleTime; - if (firstShuffle == null || firstShuffle.getStartTime() > attempt.getStartTime()) { - firstShuffle = attempt; - } - if (lastShuffle == null || lastShuffle.getShuffleFinishTime() < attempt.getShuffleFinishTime()) { - lastShuffle = attempt; - } - if (worstShuffle == null || (worstShuffle.getShuffleFinishTime() - worstShuffle.getStartTime()) < shuffleTime) { - worstShuffle = attempt; - } - - long reduceTime = attempt.getEndTime() - attempt.getShuffleFinishTime(); - avgReduceTimeInSec += reduceTime; - if (firstReduce == null || firstReduce.getStartTime() > attempt.getStartTime()) { - firstReduce = attempt; - } - if (lastReduce == null || lastReduce.getEndTime() < attempt.getEndTime()) { - lastReduce = attempt; - } - if (worstReduce == null || (worstReduce.getEndTime() - worstReduce.getShuffleFinishTime()) < reduceTime) { - worstReduce = attempt; - } + if (lastMap == null || lastMap.getEndTime() < attempt.getEndTime()) { + lastMap = attempt; + } + if (worstMap == null || (worstMap.getEndTime() - worstMap.getStartTime()) < mapTime) { + worstMap = attempt; + } + long tmpMem = getMinimumIOSortMemory(attempt); + if (tmpMem > minMapSpillMemBytes) { + minMapSpillMemBytes = tmpMem; + } + } else if (TaskType.REDUCE.toString().equalsIgnoreCase(getTaskType(attempt))) { + long shuffleTime = attempt.getShuffleFinishTime() - attempt.getStartTime(); + avgShuffleTimeInSec += shuffleTime; + if (firstShuffle == null || firstShuffle.getStartTime() > attempt.getStartTime()) { + firstShuffle = attempt; + } + if (lastShuffle == null || lastShuffle.getShuffleFinishTime() < attempt.getShuffleFinishTime()) { + lastShuffle = attempt; + } + if (worstShuffle == null || (worstShuffle.getShuffleFinishTime() - worstShuffle.getStartTime()) < shuffleTime) { + worstShuffle = attempt; + } + + long reduceTime = attempt.getEndTime() - attempt.getShuffleFinishTime(); + avgReduceTimeInSec += reduceTime; + if (firstReduce == null || firstReduce.getStartTime() > attempt.getStartTime()) { + firstReduce = attempt; + } + if (lastReduce == null || lastReduce.getEndTime() < attempt.getEndTime()) { + lastReduce = attempt; + } + if (worstReduce == null || (worstReduce.getEndTime() - worstReduce.getShuffleFinishTime()) < reduceTime) { + worstReduce = attempt; } } - if (numMaps > 0) { - avgMapTimeInSec = avgMapTimeInSec / numMaps / DateTimeUtil.ONESECOND; - } - if (numReduces > 0) { - avgReduceTimeInSec = avgReduceTimeInSec / numReduces / DateTimeUtil.ONESECOND; - avgShuffleTimeInSec = avgShuffleTimeInSec / numReduces / DateTimeUtil.ONESECOND; - } - initialized = true; + } + if (numMaps > 0) { + avgMapTimeInSec = avgMapTimeInSec / numMaps / DateTimeUtil.ONESECOND; + } + if (numReduces > 0) { + avgReduceTimeInSec = avgReduceTimeInSec / numReduces / DateTimeUtil.ONESECOND; + avgShuffleTimeInSec = avgShuffleTimeInSec / numReduces / DateTimeUtil.ONESECOND; } return this; } diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index c15bb31c1a..174c31ba85 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -128,7 +128,7 @@ public JHFEventReaderBase(Map baseTags, Configuration configurat this.appConfig = appConfig; this.jobCounterMetricsGenerator = new JobCounterMetricsGenerator(appConfig.getEagleServiceConfig()); - this.jobSuggestionListener = new JobSuggestionListener(); + this.jobSuggestionListener = new JobSuggestionListener(appConfig.getConfig()); if (this.configuration != null) { this.jobSuggestionListener.jobConfigCreated(configuration); } @@ -158,6 +158,7 @@ public void flush() throws Exception { for (HistoryJobEntityLifecycleListener listener : this.jobEntityLifecycleListeners) { listener.flush(); } + jobSuggestionListener.flush(); } private String buildJobTrackingUrl(String jobId) { diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java index 24c5fb90e3..e3bd57e141 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java @@ -17,7 +17,9 @@ package org.apache.eagle.jpm.mr.history.parser; +import com.typesafe.config.Config; import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; +import org.apache.eagle.jpm.analyzer.mr.MRJobPerformanceAnalyzer; import org.apache.eagle.jpm.mr.historyentity.JobBaseAPIEntity; import org.apache.eagle.jpm.mr.historyentity.JobExecutionAPIEntity; import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; @@ -36,8 +38,13 @@ public class JobSuggestionListener implements HistoryJobEntityCreationListener { private static final Logger LOG = LoggerFactory.getLogger(JobSuggestionListener.class); - private MapReduceAnalyzerEntity info = new MapReduceAnalyzerEntity(); - private Configuration jobConf; + private MapReduceAnalyzerEntity info; + private MRJobPerformanceAnalyzer analyzer; + + public JobSuggestionListener(Config config) { + this.info = new MapReduceAnalyzerEntity(); + this.analyzer = new MRJobPerformanceAnalyzer<>(config); + } @Override public void jobEntityCreated(JobBaseAPIEntity entity) throws Exception { @@ -68,7 +75,7 @@ public void jobEntityCreated(JobBaseAPIEntity entity) throws Exception { } public void jobConfigCreated(Configuration configuration) { - this.jobConf = configuration; + info.setJobConf(configuration); } public void jobCountersCreated(JobCounters totalCounters, JobCounters mapCounters, JobCounters reduceCounters) { @@ -79,6 +86,6 @@ public void jobCountersCreated(JobCounters totalCounters, JobCounters mapCounter @Override public void flush() throws Exception { - + analyzer.analyze(info); } } diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/TaskAttemptCounterListener.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/TaskAttemptCounterListener.java index 856f051180..24c734d628 100755 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/TaskAttemptCounterListener.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/TaskAttemptCounterListener.java @@ -18,7 +18,6 @@ package org.apache.eagle.jpm.mr.history.parser; -import org.apache.eagle.jpm.mr.history.MRHistoryJobConfig; import org.apache.eagle.jpm.mr.historyentity.JobBaseAPIEntity; import org.apache.eagle.jpm.mr.historyentity.TaskAttemptCounterAPIEntity; import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; From d0a61b1b4ee08cad8623996ecfac06bb40a28e46 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Wed, 18 Jan 2017 10:13:36 +0800 Subject: [PATCH 12/22] update JHFEventReaderBase.java --- .../eagle/jpm/mr/history/parser/JHFEventReaderBase.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index 174c31ba85..fab2a6bc76 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -129,9 +129,7 @@ public JHFEventReaderBase(Map baseTags, Configuration configurat this.appConfig = appConfig; this.jobCounterMetricsGenerator = new JobCounterMetricsGenerator(appConfig.getEagleServiceConfig()); this.jobSuggestionListener = new JobSuggestionListener(appConfig.getConfig()); - if (this.configuration != null) { - this.jobSuggestionListener.jobConfigCreated(configuration); - } + this.addListener(jobSuggestionListener); } public void register(HistoryJobEntityLifecycleListener lifecycleListener) { @@ -158,7 +156,6 @@ public void flush() throws Exception { for (HistoryJobEntityLifecycleListener listener : this.jobEntityLifecycleListeners) { listener.flush(); } - jobSuggestionListener.flush(); } private String buildJobTrackingUrl(String jobId) { @@ -314,6 +311,7 @@ protected void handleJob(EventType eventType, Map values, Object t JobCounters parsedMapCounters = parseCounters(mapCounters); JobCounters parsedReduceCounters = parseCounters(reduceCounters); jobSuggestionListener.jobCountersCreated(parsedTotalCounters, parsedMapCounters, parsedReduceCounters); + jobSuggestionListener.jobConfigCreated(configuration); } } } From 7651da89104d14bdd4ad873521411f13b7d0dd6f Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Wed, 18 Jan 2017 11:06:55 +0800 Subject: [PATCH 13/22] Fix code checkstyle --- .../MapReduceCompressionSettingAnalyzer.java | 4 +--- .../mr/suggestion/MapReduceDataSkewAnalyzer.java | 2 ++ .../mr/suggestion/MapReduceJobSuggestionContext.java | 12 +++--------- .../suggestion/MapReduceQueueResourceAnalyzer.java | 2 +- .../mr/suggestion/MapReduceSpillAnalyzer.java | 2 +- .../mr/suggestion/MapReduceTaskNumAnalyzer.java | 4 ++-- .../jpm/mr/history/parser/JobSuggestionListener.java | 2 +- 7 files changed, 11 insertions(+), 17 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java index d8870b7bd9..3dc9ddde52 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java @@ -63,9 +63,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) String codecName = jobconf.get(FileOutputFormat.COMPRESS_CODEC, ""); String outputFileFormat = jobconf.get(OUTPUT_FORMAT_CLASS_ATTR, ""); - if ((codecName.endsWith("GzipCodec") || - codecName.endsWith("SnappyCodec") || - codecName.endsWith("DefaultCodec")) + if ((codecName.endsWith("GzipCodec") || codecName.endsWith("SnappyCodec") || codecName.endsWith("DefaultCodec")) && outputFileFormat.endsWith("TextOutputFormat")) { sb.append("Best practice: don't use Gzip/Snappy/DefaultCodec with TextOutputFormat"); sb.append(" as this will cause the output files to be unsplittable. "); diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java index ebe2f4f170..ac6cc9769f 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java @@ -25,9 +25,11 @@ public class MapReduceDataSkewAnalyzer implements Processor { private MapReduceJobSuggestionContext context; + public MapReduceDataSkewAnalyzer(MapReduceJobSuggestionContext context) { this.context = context; } + @Override public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { TaskAttemptExecutionAPIEntity worstReduce = context.getWorstReduce(); diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java index fefb2ed629..0735e827e5 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java @@ -22,7 +22,6 @@ import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; import org.apache.eagle.jpm.util.MRJobTagName; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.TaskType; @@ -57,12 +56,6 @@ public class MapReduceJobSuggestionContext { private long minMapSpillMemBytes; public static final Pattern MAX_HEAP_PATTERN = Pattern.compile("-Xmx([0-9]+)([kKmMgG]?)"); - public final long _1MB = 1024 * 1024l; - public final long _256MB = 256 * _1MB; - public final long _512MB = 512 * _1MB; - public final long _1GB = 1024 * _1MB; - - private static Boolean initialized = false; public MapReduceJobSuggestionContext(MapReduceAnalyzerEntity job) { this.job = job; @@ -133,9 +126,10 @@ private String getTaskType(TaskAttemptExecutionAPIEntity taskAttemptInfo) { } /** - * 16 is the default index size + * The default index size is 16. + * * @param attempt - * @return + * @return minimal sort memory */ private long getMinimumIOSortMemory(TaskAttemptExecutionAPIEntity attempt) { long records = attempt.getJobCounters().getCounterValue(MAP_OUTPUT_RECORDS); diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java index e2d15a3e51..2ff7c1179e 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java @@ -26,7 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** +/* * Criterion: (TimeElapsed / (numTasks / 500 * avgTaskTime)) > 20 */ public class MapReduceQueueResourceAnalyzer implements Processor { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java index 68ed88114c..ca44bc1cf5 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java @@ -58,7 +58,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) long minMapSpillMemBytes = context.getMinMapSpillMemBytes(); double spillPercent = context.getJobconf().getDouble(MAP_SORT_SPILL_PERCENT, 0.8); - if (minMapSpillMemBytes > context._512MB * spillPercent) { + if (minMapSpillMemBytes > 512 * FileUtils.ONE_MB * spillPercent) { if (Math.abs(1.0 - spillPercent) > 0.001) { sb.append(" -D" + MAP_SORT_SPILL_PERCENT + "=1"); } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java index 4a0e89d581..7d26ac3292 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java @@ -49,8 +49,8 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) long suggestReduces = 0; StringBuilder tmpsb = new StringBuilder(); - if (avgShuffleBytes < context._256MB && avgReduceTotalTime < 300 - && avgReduceOutput < context._256MB && numReduces > 1) { + if (avgShuffleBytes < 256 * FileUtils.ONE_MB && avgReduceTotalTime < 300 + && avgReduceOutput < 256 * FileUtils.ONE_MB && numReduces > 1) { tmpsb.append("average reduce input bytes is: "); tmpsb.append(FileUtils.byteCountToDisplaySize(avgShuffleBytes) + ", "); if (suggestReduces == 0) { diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java index e3bd57e141..b38d4622a3 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java @@ -32,7 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** +/* * JobEventCounterListener provides an interface to add job/task counter analyzers */ public class JobSuggestionListener implements HistoryJobEntityCreationListener { From a902b5af435dc40569a76ba0b3cc870b18607e56 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Wed, 18 Jan 2017 15:22:28 +0800 Subject: [PATCH 14/22] rename class name --- .../analyzer/mr/MRJobPerformanceAnalyzer.java | 2 +- .../mr/suggestion/JobSuggestionEvaluator.java | 33 +++++++++++-------- ...MapReduceCompressionSettingProcessor.java} | 4 +-- ...r.java => MapReduceDataSkewProcessor.java} | 4 +-- ...zer.java => MapReduceGCTimeProcessor.java} | 6 ++-- .../MapReduceJobSuggestionContext.java | 8 +++-- ...a => MapReduceQueueResourceProcessor.java} | 6 ++-- ...yzer.java => MapReduceSpillProcessor.java} | 6 ++-- ...va => MapReduceSplitSettingProcessor.java} | 6 ++-- ...er.java => MapReduceTaskNumProcessor.java} | 6 ++-- .../mr/history/parser/JHFEventReaderBase.java | 3 -- .../src/main/resources/application.conf | 2 +- .../jpm/util/jobcounter/JobCounters.java | 6 +++- 13 files changed, 50 insertions(+), 42 deletions(-) rename eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/{MapReduceCompressionSettingAnalyzer.java => MapReduceCompressionSettingProcessor.java} (94%) rename eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/{MapReduceDataSkewAnalyzer.java => MapReduceDataSkewProcessor.java} (94%) rename eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/{MapReduceGCTimeAnalyzer.java => MapReduceGCTimeProcessor.java} (93%) rename eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/{MapReduceQueueResourceAnalyzer.java => MapReduceQueueResourceProcessor.java} (94%) rename eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/{MapReduceSpillAnalyzer.java => MapReduceSpillProcessor.java} (96%) rename eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/{MapReduceSplitSettingAnalyzer.java => MapReduceSplitSettingProcessor.java} (89%) rename eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/{MapReduceTaskNumAnalyzer.java => MapReduceTaskNumProcessor.java} (97%) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index 54f12d621c..129044025b 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -47,7 +47,7 @@ public MRJobPerformanceAnalyzer(Config config) { evaluators.add(new SLAJobEvaluator(config)); evaluators.add(new JobSuggestionEvaluator(config)); - publishers.add(new EagleStorePublisher(config)); + //publishers.add(new EagleStorePublisher(config)); publishers.add(new EmailPublisher(config)); } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java index d9411b7d56..9b53561e44 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java @@ -18,7 +18,6 @@ package org.apache.eagle.jpm.analyzer.mr.suggestion; import com.typesafe.config.Config; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.Evaluator; import org.apache.eagle.jpm.analyzer.Processor; import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; @@ -42,13 +41,13 @@ public JobSuggestionEvaluator(Config config) { private List loadProcessors(MapReduceJobSuggestionContext context) { List processors = new ArrayList<>(); - processors.add(new MapReduceCompressionSettingAnalyzer(context)); - processors.add(new MapReduceSplitSettingAnalyzer(context)); - processors.add(new MapReduceDataSkewAnalyzer(context)); - processors.add(new MapReduceGCTimeAnalyzer(context)); - processors.add(new MapReduceSpillAnalyzer(context)); - processors.add(new MapReduceTaskNumAnalyzer(context)); - processors.add(new MapReduceQueueResourceAnalyzer(context)); + processors.add(new MapReduceCompressionSettingProcessor(context)); + processors.add(new MapReduceSplitSettingProcessor(context)); + processors.add(new MapReduceDataSkewProcessor(context)); + processors.add(new MapReduceGCTimeProcessor(context)); + processors.add(new MapReduceSpillProcessor(context)); + processors.add(new MapReduceTaskNumProcessor(context)); + //processors.add(new MapReduceQueueResourceProcessor(context)); return processors; } @@ -64,13 +63,19 @@ public Result.EvaluatorResult evaluate(MapReduceAnalyzerEntity analyzerEntity) { return null; } - Result.EvaluatorResult result = new Result.EvaluatorResult(); - for (Processor processor : loadProcessors(jobContext)) { - Result.ProcessorResult processorResult = processor.process(analyzerEntity); - if (processorResult != null) { - result.addProcessorResult(processor.getClass(), processorResult); + try { + Result.EvaluatorResult result = new Result.EvaluatorResult(); + for (Processor processor : loadProcessors(jobContext)) { + Result.ProcessorResult processorResult = processor.process(analyzerEntity); + if (processorResult != null) { + result.addProcessorResult(processor.getClass(), processorResult); + } } + return result; + } catch (Exception ex) { + LOG.error(ex.getMessage(), ex); + return null; } - return result; + } } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java similarity index 94% rename from eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java rename to eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java index 3dc9ddde52..64d09612c2 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java @@ -28,11 +28,11 @@ import static org.apache.hadoop.mapreduce.MRJobConfig.NUM_REDUCES; import static org.apache.hadoop.mapreduce.MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR; -public class MapReduceCompressionSettingAnalyzer implements Processor { +public class MapReduceCompressionSettingProcessor implements Processor { private MapReduceJobSuggestionContext context; - public MapReduceCompressionSettingAnalyzer(MapReduceJobSuggestionContext context) { + public MapReduceCompressionSettingProcessor(MapReduceJobSuggestionContext context) { this.context = context; } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java similarity index 94% rename from eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java rename to eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java index ac6cc9769f..48ac47d6c9 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java @@ -23,10 +23,10 @@ import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; import org.apache.eagle.jpm.util.jobcounter.JobCounters; -public class MapReduceDataSkewAnalyzer implements Processor { +public class MapReduceDataSkewProcessor implements Processor { private MapReduceJobSuggestionContext context; - public MapReduceDataSkewAnalyzer(MapReduceJobSuggestionContext context) { + public MapReduceDataSkewProcessor(MapReduceJobSuggestionContext context) { this.context = context; } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java similarity index 93% rename from eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java rename to eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java index 6edfda4bd3..e1b491345f 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java @@ -25,10 +25,10 @@ import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_JAVA_OPTS; import static org.apache.hadoop.mapreduce.MRJobConfig.REDUCE_JAVA_OPTS; -public class MapReduceGCTimeAnalyzer implements Processor { +public class MapReduceGCTimeProcessor implements Processor { private MapReduceJobSuggestionContext context; - public MapReduceGCTimeAnalyzer(MapReduceJobSuggestionContext context) { + public MapReduceGCTimeProcessor(MapReduceJobSuggestionContext context) { this.context = context; } @@ -56,7 +56,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java index 0735e827e5..1f4e548360 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceJobSuggestionContext.java @@ -20,6 +20,7 @@ import org.apache.eagle.common.DateTimeUtil; import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; +import org.apache.eagle.jpm.util.Constants; import org.apache.eagle.jpm.util.MRJobTagName; import org.apache.hadoop.mapred.JobConf; @@ -69,7 +70,8 @@ private MapReduceJobSuggestionContext buildContext() { numReduces = jobconf.getLong(NUM_REDUCES, 0); for (TaskAttemptExecutionAPIEntity attempt : job.getCompletedTaskAttemptsMap().values()) { - if (TaskType.MAP.toString().equalsIgnoreCase(getTaskType(attempt))) { + String taskType = getTaskType(attempt); + if (Constants.TaskType.MAP.toString().equalsIgnoreCase(taskType)) { long mapTime = attempt.getEndTime() - attempt.getStartTime(); avgMapTimeInSec += mapTime; if (firstMap == null || firstMap.getStartTime() > attempt.getStartTime()) { @@ -85,7 +87,7 @@ private MapReduceJobSuggestionContext buildContext() { if (tmpMem > minMapSpillMemBytes) { minMapSpillMemBytes = tmpMem; } - } else if (TaskType.REDUCE.toString().equalsIgnoreCase(getTaskType(attempt))) { + } else if (TaskType.REDUCE.toString().equalsIgnoreCase(taskType)) { long shuffleTime = attempt.getShuffleFinishTime() - attempt.getStartTime(); avgShuffleTimeInSec += shuffleTime; if (firstShuffle == null || firstShuffle.getStartTime() > attempt.getStartTime()) { @@ -122,7 +124,7 @@ private MapReduceJobSuggestionContext buildContext() { } private String getTaskType(TaskAttemptExecutionAPIEntity taskAttemptInfo) { - return taskAttemptInfo.getTags().get(MRJobTagName.TASK_TYPE); + return taskAttemptInfo.getTags().get(MRJobTagName.TASK_TYPE.toString()); } /** diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java similarity index 94% rename from eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java rename to eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java index 2ff7c1179e..7189b1eb02 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java @@ -29,12 +29,12 @@ /* * Criterion: (TimeElapsed / (numTasks / 500 * avgTaskTime)) > 20 */ -public class MapReduceQueueResourceAnalyzer implements Processor { - private static final Logger LOG = LoggerFactory.getLogger(MapReduceQueueResourceAnalyzer.class); +public class MapReduceQueueResourceProcessor implements Processor { + private static final Logger LOG = LoggerFactory.getLogger(MapReduceQueueResourceProcessor.class); private MapReduceJobSuggestionContext context; - public MapReduceQueueResourceAnalyzer(MapReduceJobSuggestionContext context) { + public MapReduceQueueResourceProcessor(MapReduceJobSuggestionContext context) { this.context = context; } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java similarity index 96% rename from eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java rename to eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java index ca44bc1cf5..050bdc3a7c 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java @@ -34,11 +34,11 @@ * Check whether spilled more than once, if true, find out the minimum value of the memory to hold all the data, * based on that value, find out how much memory need for heap size. */ -public class MapReduceSpillAnalyzer implements Processor { +public class MapReduceSpillProcessor implements Processor { private MapReduceJobSuggestionContext context; - public MapReduceSpillAnalyzer(MapReduceJobSuggestionContext context) { + public MapReduceSpillProcessor(MapReduceJobSuggestionContext context) { this.context = context; } @@ -91,7 +91,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); } } catch (NullPointerException e) { //When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java similarity index 89% rename from eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java rename to eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java index 8dbf41c30a..813446f0df 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java @@ -22,11 +22,11 @@ import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -public class MapReduceSplitSettingAnalyzer implements Processor { +public class MapReduceSplitSettingProcessor implements Processor { private MapReduceJobSuggestionContext context; - public MapReduceSplitSettingAnalyzer(MapReduceJobSuggestionContext context) { + public MapReduceSplitSettingProcessor(MapReduceJobSuggestionContext context) { this.context = context; } @@ -36,7 +36,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) if (context.getJobconf().getLong(FileInputFormat.SPLIT_MINSIZE, 0) > 1) { sb.append("Best practice: don't set " + FileInputFormat.SPLIT_MINSIZE); sb.append(", because it may lower data locality, hence maps will run slower.\n"); - return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); } return null; } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java similarity index 97% rename from eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java rename to eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java index 7d26ac3292..d56a2063b2 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java @@ -23,11 +23,11 @@ import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.util.jobcounter.JobCounters; -public class MapReduceTaskNumAnalyzer implements Processor { +public class MapReduceTaskNumProcessor implements Processor { private MapReduceJobSuggestionContext context; - public MapReduceTaskNumAnalyzer(MapReduceJobSuggestionContext context) { + public MapReduceTaskNumProcessor(MapReduceJobSuggestionContext context) { this.context = context; } @@ -129,7 +129,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.CRITICAL, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index fab2a6bc76..d58eadc93b 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -304,7 +304,6 @@ protected void handleJob(EventType eventType, Map values, Object t formatDiagnostics(values.get(Keys.DIAGNOSTICS)); entityCreated(jobExecutionEntity); - jobSuggestionListener.jobEntityCreated(jobExecutionEntity); if (configuration != null && totalCounters != null) { JobCounters parsedTotalCounters = parseCounters(totalCounters); @@ -426,7 +425,6 @@ protected void handleTask(RecordTypes recType, EventType eventType, final Map Date: Wed, 18 Jan 2017 17:20:14 +0800 Subject: [PATCH 15/22] refactor code --- .../MapReduceCompressionSettingProcessor.java | 17 +- .../MapReduceDataSkewProcessor.java | 13 +- .../suggestion/MapReduceGCTimeProcessor.java | 6 +- .../suggestion/MapReduceSpillProcessor.java | 12 +- .../suggestion/MapReduceTaskNumProcessor.java | 191 ++++++++++-------- 5 files changed, 127 insertions(+), 112 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java index 64d09612c2..4c48726447 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java @@ -41,24 +41,19 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) StringBuilder sb = new StringBuilder(); JobConf jobconf = new JobConf(context.getJobconf()); if (jobconf.getLong(NUM_REDUCES, 0) > 0) { - if (jobconf.getCompressMapOutput() == false) { - sb.append("Please set " + MAP_OUTPUT_COMPRESS - + " to true to reduce network IO.\n"); + if (!jobconf.getCompressMapOutput()) { + sb.append("Please set " + MAP_OUTPUT_COMPRESS + " to true to reduce network IO.\n"); } else { - String codecClassName = jobconf - .get(MAP_OUTPUT_COMPRESS_CODEC); - if (!(codecClassName.endsWith("LzoCodec") || codecClassName - .endsWith("SnappyCodec"))) { - sb.append("Best practice: use LzoCodec or SnappyCodec for " - + MAP_OUTPUT_COMPRESS_CODEC); + String codecClassName = jobconf.get(MAP_OUTPUT_COMPRESS_CODEC); + if (!(codecClassName.endsWith("LzoCodec") || codecClassName.endsWith("SnappyCodec"))) { + sb.append("Best practice: use LzoCodec or SnappyCodec for " + MAP_OUTPUT_COMPRESS_CODEC); sb.append("\n"); } } } if (!jobconf.getBoolean(FileOutputFormat.COMPRESS, false)) { - sb.append("Please set " + FileOutputFormat.COMPRESS - + " to true to reduce disk usage and network IO.\n"); + sb.append("Please set " + FileOutputFormat.COMPRESS + " to true to reduce disk usage and network IO.\n"); } else { String codecName = jobconf.get(FileOutputFormat.COMPRESS_CODEC, ""); String outputFileFormat = jobconf.get(OUTPUT_FORMAT_CLASS_ATTR, ""); diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java index 48ac47d6c9..5249aec1c1 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java @@ -17,6 +17,7 @@ package org.apache.eagle.jpm.analyzer.mr.suggestion; +import org.apache.eagle.common.DateTimeUtil; import org.apache.eagle.jpm.analyzer.Processor; import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; @@ -38,17 +39,15 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } StringBuilder sb = new StringBuilder(); try { - long worstTime = (worstReduce.getEndTime() - worstReduce - .getShuffleFinishTime()) / 1000; - if (worstTime - context.getAvgReduceTimeInSec() > 30 * 60 ) { + long worstTimeInSec = (worstReduce.getEndTime() - worstReduce.getShuffleFinishTime()) / DateTimeUtil.ONESECOND; + if (worstTimeInSec - context.getAvgReduceTimeInSec() > 30 * 60 ) { long avgInputs = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS) / context.getNumReduces(); long worstInputs = worstReduce.getJobCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS); - if (worstInputs / 5 > avgInputs) { - sb.append("Data skew detected in reducers. The average reduce time is " - + context.getAvgReduceTimeInSec()); - sb.append(" seconds, the worst reduce time is " + worstTime); + if (worstInputs > avgInputs * 5) { + sb.append("Data skew detected in reducers. The average reduce time is " + context.getAvgReduceTimeInSec()); + sb.append(" seconds, the worst reduce time is " + worstTimeInSec); sb.append(" seconds. Please investigate this problem to improve your job performance.\n"); } } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java index e1b491345f..2308468a4e 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java @@ -40,8 +40,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) long mapCPUTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS); if (mapGCTime > mapCPUTime * 0.1) { - sb.append("Map GC_TIME_MILLIS took too long. Please increase mapper memory via -D" - + MAP_JAVA_OPTS); + sb.append("Map GC_TIME_MILLIS took too long. Please increase mapper memory via -D" + MAP_JAVA_OPTS); sb.append(", or optimize your mapper class.\n"); } @@ -49,8 +48,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) long reduceGCTime = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS); long reduceCPUTime = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS); if (reduceGCTime > reduceCPUTime * 0.1) { - sb.append("Reduce GC_TIME_MILLIS took too long. Please increase memory for reduce via -D" - + REDUCE_JAVA_OPTS); + sb.append("Reduce GC_TIME_MILLIS took too long. Please increase memory for reduce via -D" + REDUCE_JAVA_OPTS); sb.append(", or optimize your reducer class.\n"); } } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java index 050bdc3a7c..e180c2d075 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java @@ -66,15 +66,15 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) minMapSpillMemBytes /= spillPercent; } - minMapSpillMemBytes = (minMapSpillMemBytes / FileUtils.ONE_MB + 10) / 10 * 10; - if (minMapSpillMemBytes >= 2047 ) { + long minMapSpillMemMB = (minMapSpillMemBytes / FileUtils.ONE_MB + 10) / 10 * 10; + if (minMapSpillMemMB >= 2047 ) { sb.append(" Please reduce the block size of the input files and make sure they are splittable."); } else { - sb.append(" -D" + IO_SORT_MB + "=" + minMapSpillMemBytes); + sb.append(" -D" + IO_SORT_MB + "=" + minMapSpillMemMB); long heapSize = getMaxHeapSize(context.getJobconf().get(MAP_JAVA_OPTS)); - if (heapSize < 3 * minMapSpillMemBytes) { - sb.append(" -D" + MAP_JAVA_OPTS + "=-Xmx" - + (long) (minMapSpillMemBytes * 3 + 1024) / 1024 * 1024 + "M"); + if (heapSize < 3 * minMapSpillMemMB) { + long expectedHeapSizeMB = (minMapSpillMemMB * 3 + 1024) / 1024 * 1024; + sb.append(" -D" + MAP_JAVA_OPTS + "=-Xmx" + expectedHeapSizeMB + "M"); } } sb.append(" to avoid spilled records.\n"); diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java index d56a2063b2..e14826605d 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java @@ -24,7 +24,7 @@ import org.apache.eagle.jpm.util.jobcounter.JobCounters; public class MapReduceTaskNumProcessor implements Processor { - + private static final String[] SIZE_UNITS = {"B", "K", "M", "G", "T", "P"}; private MapReduceJobSuggestionContext context; public MapReduceTaskNumProcessor(MapReduceJobSuggestionContext context) { @@ -35,106 +35,110 @@ public MapReduceTaskNumProcessor(MapReduceJobSuggestionContext context) { public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { StringBuilder sb = new StringBuilder(); try { - long numReduces = context.getNumReduces(); - long numMaps = context.getNumMaps(); - if (numReduces > 0) { - long avgReduceTime = context.getAvgReduceTimeInSec(); - long avgShuffleTime = context.getAvgShuffleTimeInSec(); - long avgShuffleBytes = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_SHUFFLE_BYTES) + sb.append(analyzeReduceTaskNum()); + sb.append(analyzeMapTaskNum()); + + if (sb.length() > 0) { + return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + } + } catch (NullPointerException e) { + // When job failed there may not have counters, so just ignore it + } + return null; + } + + + private String analyzeReduceTaskNum() { + StringBuilder sb = new StringBuilder(); + + long numReduces = context.getNumReduces(); + if (numReduces > 0) { + long avgReduceTime = context.getAvgReduceTimeInSec(); + long avgShuffleTime = context.getAvgShuffleTimeInSec(); + long avgShuffleBytes = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_SHUFFLE_BYTES) / numReduces; - long avgReduceOutput = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.HDFS_BYTES_WRITTEN) + long avgReduceOutput = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.HDFS_BYTES_WRITTEN) / numReduces; - long avgReduceTotalTime = avgShuffleTime + avgReduceTime; + long avgReduceTotalTime = avgShuffleTime + avgReduceTime; - long suggestReduces = 0; - StringBuilder tmpsb = new StringBuilder(); + long suggestReduces = 0; + StringBuilder tmpsb = new StringBuilder(); - if (avgShuffleBytes < 256 * FileUtils.ONE_MB && avgReduceTotalTime < 300 + String avgShuffleDisplaySize = bytesToHumanReadable(avgShuffleBytes); + if (avgShuffleBytes < 256 * FileUtils.ONE_MB && avgReduceTotalTime < 300 && avgReduceOutput < 256 * FileUtils.ONE_MB && numReduces > 1) { - tmpsb.append("average reduce input bytes is: "); - tmpsb.append(FileUtils.byteCountToDisplaySize(avgShuffleBytes) + ", "); - if (suggestReduces == 0) { - suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); - } - } else if (avgShuffleBytes > 10 * FileUtils.ONE_GB && avgReduceTotalTime > 1800) { - tmpsb.append("average reduce input bytes is: "); - tmpsb.append(FileUtils.byteCountToDisplaySize(avgShuffleBytes) + ", "); - if (suggestReduces == 0) { - suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); - } - } + tmpsb.append("average reduce input bytes is: ").append(avgShuffleDisplaySize).append(", "); + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } else if (avgShuffleBytes > 10 * FileUtils.ONE_GB && avgReduceTotalTime > 1800) { + tmpsb.append("average reduce input bytes is: ").append(avgShuffleDisplaySize).append(", "); + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } - if (avgReduceTotalTime < 60 && numReduces > 1) { - tmpsb.append("average reduce time is only " - + (context.getAvgShuffleTimeInSec() + context.getAvgReduceTimeInSec()) + " seconds, "); - if (suggestReduces == 0) { - suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); - } - } else if (avgReduceTotalTime > 3600 && avgReduceTime > 1800) { - tmpsb.append("average reduce time is " - + (avgShuffleTime + avgReduceTime) + " seconds, "); - if (suggestReduces == 0) { - suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); - } + if (avgReduceTotalTime < 60 && numReduces > 1) { + tmpsb.append("average reduce time is only ").append(avgReduceTotalTime).append(" seconds, "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); + } + } else if (avgReduceTotalTime > 3600 && avgReduceTime > 1800) { + tmpsb.append("average reduce time is ").append(avgReduceTotalTime).append(" seconds, "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); } + } - if (avgReduceOutput < 10 * FileUtils.ONE_MB && avgReduceTime < 300 + String avgReduceOutputDisplaySize = bytesToHumanReadable(avgReduceOutput); + if (avgReduceOutput < 10 * FileUtils.ONE_MB && avgReduceTime < 300 && avgShuffleBytes < 2 * FileUtils.ONE_GB && numReduces > 1) { - tmpsb.append(" average reduce output is only "); - tmpsb.append(FileUtils.byteCountToDisplaySize(avgReduceOutput) + ", "); - if (suggestReduces == 0) { - suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); - } - } else if (avgReduceOutput > 10 * FileUtils.ONE_GB && avgReduceTime > 1800) { - tmpsb.append(" average reduce output is "); - tmpsb.append(FileUtils.byteCountToDisplaySize(avgReduceOutput) + ", "); - if (suggestReduces == 0) { - suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); - } + tmpsb.append(" average reduce output is only ").append(avgReduceOutputDisplaySize).append(", "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); } - - if (suggestReduces > 0) { - sb.append("Best practice: "); - sb.append(tmpsb.toString()); - sb.append("please consider "); - if (suggestReduces > numReduces) { - sb.append("increasing the "); - } else { - sb.append("decreasing the "); - } - sb.append("reducer number. You could try -Dmapreduce.job.reduces=" + suggestReduces + "\n"); + } else if (avgReduceOutput > 10 * FileUtils.ONE_GB && avgReduceTime > 1800) { + tmpsb.append(" average reduce output is ").append(avgReduceOutputDisplaySize).append(", "); + if (suggestReduces == 0) { + suggestReduces = getReduceNum(avgShuffleBytes, avgReduceOutput, avgReduceTime); } } - // map numbers - long avgMapInput = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.HDFS_BYTES_READ) - / numMaps; - long avgMapTime = context.getAvgMapTimeInSec(); - if (avgMapInput < 5 * FileUtils.ONE_MB && avgMapTime < 30 && numMaps > 1) { - sb.append("Best practice: average map input bytes only have "); - sb.append(FileUtils.byteCountToDisplaySize(avgMapInput)); - sb.append(". Please reduce the number of mappers by merging input files.\n"); - } else if (avgMapInput > FileUtils.ONE_GB) { - sb.append("Best practice: average map input bytes have "); - sb.append(FileUtils.byteCountToDisplaySize(avgMapInput)); - sb.append(". Please increase the number of mappers by using splittable compression, a container file format or a smaller block size.\n"); + if (suggestReduces > 0) { + sb.append("Best practice: ").append(tmpsb.toString()).append("please consider "); + if (suggestReduces > numReduces) { + sb.append("increasing the "); + } else { + sb.append("decreasing the "); + } + sb.append("reducer number. You could try -Dmapreduce.job.reduces=").append(suggestReduces).append("\n"); } + } + return sb.toString(); + } - if (avgMapTime < 10 && numMaps > 1) { - sb.append("Best practice: average map time only have " + avgMapTime); - sb.append(" seconds. Please reduce the number of mappers by merging input files or by using a larger block size.\n"); - } else if (avgMapTime > 600 && avgMapInput < FileUtils.ONE_GB) { - sb.append("Best practice: average map time is " + avgMapInput); - sb.append(" seconds. Please increase the number of mappers by using splittable compression, a container file format or a smaller block size.\n"); - } + private String analyzeMapTaskNum() { + StringBuilder sb = new StringBuilder(); - if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); - } - } catch (NullPointerException e) { - // When job failed there may not have counters, so just ignore it + long numMaps = context.getNumMaps(); + long avgMapTime = context.getAvgMapTimeInSec(); + long avgMapInput = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.HDFS_BYTES_READ) + / numMaps; + String avgMapInputDisplaySize = bytesToHumanReadable(avgMapInput); + + if (avgMapInput < 5 * FileUtils.ONE_MB && avgMapTime < 30 && numMaps > 1) { + sb.append("Best practice: average map input bytes only have ").append(avgMapInputDisplaySize); + sb.append(". Please reduce the number of mappers by merging input files.\n"); + } else if (avgMapInput > FileUtils.ONE_GB) { + sb.append("Best practice: average map input bytes have ").append(avgMapInputDisplaySize); + sb.append(". Please increase the number of mappers by using splittable compression, a container file format or a smaller block size.\n"); } - return null; + + if (avgMapTime < 10 && numMaps > 1) { + sb.append("Best practice: average map time only have ").append(avgMapTime); + sb.append(" seconds. Please reduce the number of mappers by merging input files or by using a larger block size.\n"); + } else if (avgMapTime > 600 && avgMapInput < FileUtils.ONE_GB) { + sb.append("Best practice: average map time is ").append(avgMapInput); + sb.append(" seconds. Please increase the number of mappers by using splittable compression, a container file format or a smaller block size.\n"); + } + + return sb.toString(); } private long getReduceNum(long avgInputBytes, long avgOutputBytes, long avgTime) { @@ -160,4 +164,23 @@ private long getReduceNum(long avgInputBytes, long avgOutputBytes, long avgTime) return newReduceNum; } + + private static String bytesToHumanReadable(long bytes) { + double val = bytes; + int idx = 0; + while (val >= 1024) { + val /= 1024.0; + idx += 1; + } + StringBuilder sb = new StringBuilder(); + sb.append((int)Math.floor(val)); + sb.append(SIZE_UNITS[idx]); + int tmp = (int)(1000 * val) % 1000; + if (idx >= 1 && tmp > 0) { + sb.append(tmp); + sb.append(SIZE_UNITS[idx - 1]); + } + return sb.toString(); + } + } \ No newline at end of file From 475411808d472b749f4b50d7bcbae3cf6321772b Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Wed, 18 Jan 2017 17:54:28 +0800 Subject: [PATCH 16/22] update EmailPublisher.java --- .../apache/eagle/jpm/analyzer/publisher/EmailPublisher.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java index 9b6543384a..4a63365943 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java @@ -67,7 +67,8 @@ public void publish(AnalyzerEntity analyzerJobEntity, Result result) { basic.put("end", analyzerJobEntity.getEndTime() == 0 ? "0" : DateTimeUtil.millisecondsToHumanDateWithSeconds(analyzerJobEntity.getEndTime())); - basic.put("progress", analyzerJobEntity.getProgress() + "%"); + double progress = analyzerJobEntity.getCurrentState().equalsIgnoreCase(org.apache.eagle.jpm.util.Constants.JobState.RUNNING.toString()) ? analyzerJobEntity.getProgress() : 100; + basic.put("progress", progress + "%"); basic.put("detail", getJobLink(analyzerJobEntity)); From 81ed7fd024f5a05ff64e61cde44a64521d40c87e Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Wed, 18 Jan 2017 19:01:47 +0800 Subject: [PATCH 17/22] change level from warning to none --- .../jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java | 2 +- .../jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java | 2 +- .../analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java | 2 +- .../jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java | 2 +- .../analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java | 2 +- .../jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java index 5249aec1c1..470c82a190 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java @@ -53,7 +53,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java index 2308468a4e..7cd22e4b77 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java @@ -54,7 +54,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java index 7189b1eb02..cbdad08fda 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java @@ -68,7 +68,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); } } } catch (Exception e) { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java index e180c2d075..2e4cb7158b 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java @@ -91,7 +91,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); } } catch (NullPointerException e) { //When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java index 813446f0df..148a5f1c00 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java @@ -36,7 +36,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) if (context.getJobconf().getLong(FileInputFormat.SPLIT_MINSIZE, 0) > 1) { sb.append("Best practice: don't set " + FileInputFormat.SPLIT_MINSIZE); sb.append(", because it may lower data locality, hence maps will run slower.\n"); - return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); } return null; } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java index e14826605d..e675a22183 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java @@ -39,7 +39,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) sb.append(analyzeMapTaskNum()); if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it From e17657ce7848d9ab7ceb0a68a780f78731f9d775 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Wed, 18 Jan 2017 21:19:40 +0800 Subject: [PATCH 18/22] add a new column in the alert --- .../MapReduceCompressionSettingProcessor.java | 13 ++++-- .../MapReduceDataSkewProcessor.java | 6 +-- .../suggestion/MapReduceGCTimeProcessor.java | 16 ++++++-- .../MapReduceQueueResourceProcessor.java | 4 +- .../suggestion/MapReduceSpillProcessor.java | 28 +++++++++---- .../MapReduceSplitSettingProcessor.java | 6 ++- .../suggestion/MapReduceTaskNumProcessor.java | 20 ++++++--- .../analyzer/publisher/EmailPublisher.java | 13 ++---- .../eagle/jpm/analyzer/publisher/Result.java | 41 +++++++++++++++++-- .../main/resources/AnalyzerReportTemplate.vm | 10 +++-- 10 files changed, 114 insertions(+), 43 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java index 4c48726447..bdac03bfc8 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java @@ -23,6 +23,9 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import java.util.ArrayList; +import java.util.List; + import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_OUTPUT_COMPRESS; import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC; import static org.apache.hadoop.mapreduce.MRJobConfig.NUM_REDUCES; @@ -39,20 +42,24 @@ public MapReduceCompressionSettingProcessor(MapReduceJobSuggestionContext contex @Override public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { StringBuilder sb = new StringBuilder(); + List optSettings = new ArrayList<>(); + JobConf jobconf = new JobConf(context.getJobconf()); if (jobconf.getLong(NUM_REDUCES, 0) > 0) { if (!jobconf.getCompressMapOutput()) { + optSettings.add(String.format("%s=true", MAP_OUTPUT_COMPRESS)); sb.append("Please set " + MAP_OUTPUT_COMPRESS + " to true to reduce network IO.\n"); } else { String codecClassName = jobconf.get(MAP_OUTPUT_COMPRESS_CODEC); if (!(codecClassName.endsWith("LzoCodec") || codecClassName.endsWith("SnappyCodec"))) { - sb.append("Best practice: use LzoCodec or SnappyCodec for " + MAP_OUTPUT_COMPRESS_CODEC); - sb.append("\n"); + optSettings.add(String.format("%s=LzoCodec or SnappyCodec", MAP_OUTPUT_COMPRESS_CODEC)); + sb.append("Best practice: use LzoCodec or SnappyCodec for " + MAP_OUTPUT_COMPRESS_CODEC).append("\n"); } } } if (!jobconf.getBoolean(FileOutputFormat.COMPRESS, false)) { + optSettings.add(String.format("%s=true", FileOutputFormat.COMPRESS)); sb.append("Please set " + FileOutputFormat.COMPRESS + " to true to reduce disk usage and network IO.\n"); } else { String codecName = jobconf.get(FileOutputFormat.COMPRESS_CODEC, ""); @@ -68,7 +75,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.WARNING, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString(), optSettings); } return null; } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java index 470c82a190..52629fe627 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java @@ -46,14 +46,14 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) long worstInputs = worstReduce.getJobCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS); if (worstInputs > avgInputs * 5) { - sb.append("Data skew detected in reducers. The average reduce time is " + context.getAvgReduceTimeInSec()); - sb.append(" seconds, the worst reduce time is " + worstTimeInSec); + sb.append("Data skew detected in reducers. The average reduce time is ").append(context.getAvgReduceTimeInSec()); + sb.append(" seconds, the worst reduce time is ").append(worstTimeInSec); sb.append(" seconds. Please investigate this problem to improve your job performance.\n"); } } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString()); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java index 7cd22e4b77..bb3830146c 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java @@ -22,6 +22,9 @@ import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.util.jobcounter.JobCounters; +import java.util.ArrayList; +import java.util.List; + import static org.apache.hadoop.mapreduce.MRJobConfig.MAP_JAVA_OPTS; import static org.apache.hadoop.mapreduce.MRJobConfig.REDUCE_JAVA_OPTS; @@ -35,12 +38,17 @@ public MapReduceGCTimeProcessor(MapReduceJobSuggestionContext context) { @Override public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { StringBuilder sb = new StringBuilder(); + List optSettings = new ArrayList<>(); + String setting; + try { long mapGCTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS); long mapCPUTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS); if (mapGCTime > mapCPUTime * 0.1) { - sb.append("Map GC_TIME_MILLIS took too long. Please increase mapper memory via -D" + MAP_JAVA_OPTS); + setting = String.format("-D%s", MAP_JAVA_OPTS); + optSettings.add(setting); + sb.append("Map GC_TIME_MILLIS took too long. Please increase mapper memory via ").append(setting); sb.append(", or optimize your mapper class.\n"); } @@ -48,13 +56,15 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) long reduceGCTime = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS); long reduceCPUTime = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS); if (reduceGCTime > reduceCPUTime * 0.1) { - sb.append("Reduce GC_TIME_MILLIS took too long. Please increase memory for reduce via -D" + REDUCE_JAVA_OPTS); + setting = String.format("-D%s", REDUCE_JAVA_OPTS); + optSettings.add(setting); + sb.append("Reduce GC_TIME_MILLIS took too long. Please increase memory for reduce via ").append(setting); sb.append(", or optimize your reducer class.\n"); } } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString(), optSettings); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java index cbdad08fda..d444facf1c 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java @@ -26,6 +26,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; + /* * Criterion: (TimeElapsed / (numTasks / 500 * avgTaskTime)) > 20 */ @@ -68,7 +70,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString()); } } } catch (Exception e) { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java index 2e4cb7158b..911a489bad 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java @@ -23,6 +23,8 @@ import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.util.jobcounter.JobCounters; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import static org.apache.eagle.jpm.analyzer.mr.suggestion.MapReduceJobSuggestionContext.MAX_HEAP_PATTERN; @@ -45,6 +47,9 @@ public MapReduceSpillProcessor(MapReduceJobSuggestionContext context) { @Override public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { StringBuilder sb = new StringBuilder(); + List optSettings = new ArrayList<>(); + String setting; + long outputRecords = 0L; // Map output records long spillRecords = 0L; // Spilled Records try { @@ -52,15 +57,16 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) spillRecords = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS); if (outputRecords < spillRecords) { - sb.append("Total Map output records: " + outputRecords); - sb.append(" Total Spilled Records: " + spillRecords); - sb.append(". Please set"); + sb.append("Total Map output records: ").append(outputRecords); + sb.append(" Total Spilled Records: ").append(spillRecords).append(". Please set"); long minMapSpillMemBytes = context.getMinMapSpillMemBytes(); double spillPercent = context.getJobconf().getDouble(MAP_SORT_SPILL_PERCENT, 0.8); if (minMapSpillMemBytes > 512 * FileUtils.ONE_MB * spillPercent) { if (Math.abs(1.0 - spillPercent) > 0.001) { - sb.append(" -D" + MAP_SORT_SPILL_PERCENT + "=1"); + setting = String.format("-D%s=1", MAP_SORT_SPILL_PERCENT); + sb.append(" ").append(setting); + optSettings.add(setting); } } else { minMapSpillMemBytes /= spillPercent; @@ -70,11 +76,15 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) if (minMapSpillMemMB >= 2047 ) { sb.append(" Please reduce the block size of the input files and make sure they are splittable."); } else { - sb.append(" -D" + IO_SORT_MB + "=" + minMapSpillMemMB); + setting = String.format("-D%s=%s", IO_SORT_MB, minMapSpillMemMB); + sb.append(" ").append(setting); + optSettings.add(setting); long heapSize = getMaxHeapSize(context.getJobconf().get(MAP_JAVA_OPTS)); if (heapSize < 3 * minMapSpillMemMB) { long expectedHeapSizeMB = (minMapSpillMemMB * 3 + 1024) / 1024 * 1024; - sb.append(" -D" + MAP_JAVA_OPTS + "=-Xmx" + expectedHeapSizeMB + "M"); + setting = String.format(" -D%s=-Xmx%sM", MAP_JAVA_OPTS, expectedHeapSizeMB); + sb.append(" ").append(setting); + optSettings.add(setting); } } sb.append(" to avoid spilled records.\n"); @@ -85,13 +95,13 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) spillRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS); if (reduceInputRecords < spillRecords) { sb.append("Please add more memory (mapreduce.reduce.java.opts) to avoid spilled records."); - sb.append(" Total Reduce input records: " + reduceInputRecords); - sb.append(" Total Spilled Records: " + spillRecords); + sb.append(" Total Reduce input records: ").append(reduceInputRecords); + sb.append(" Total Spilled Records: ").append(spillRecords); sb.append("\n"); } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString(), optSettings); } } catch (NullPointerException e) { //When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java index 148a5f1c00..304642965f 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java @@ -22,6 +22,9 @@ import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import java.util.ArrayList; +import java.util.List; + public class MapReduceSplitSettingProcessor implements Processor { private MapReduceJobSuggestionContext context; @@ -33,10 +36,11 @@ public MapReduceSplitSettingProcessor(MapReduceJobSuggestionContext context) { @Override public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { StringBuilder sb = new StringBuilder(); + if (context.getJobconf().getLong(FileInputFormat.SPLIT_MINSIZE, 0) > 1) { sb.append("Best practice: don't set " + FileInputFormat.SPLIT_MINSIZE); sb.append(", because it may lower data locality, hence maps will run slower.\n"); - return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString()); } return null; } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java index e675a22183..61c4ead3fc 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java @@ -23,6 +23,11 @@ import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.util.jobcounter.JobCounters; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.hadoop.mapreduce.MRJobConfig.NUM_REDUCES; + public class MapReduceTaskNumProcessor implements Processor { private static final String[] SIZE_UNITS = {"B", "K", "M", "G", "T", "P"}; private MapReduceJobSuggestionContext context; @@ -34,12 +39,13 @@ public MapReduceTaskNumProcessor(MapReduceJobSuggestionContext context) { @Override public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) { StringBuilder sb = new StringBuilder(); + List optSettings = new ArrayList<>(); try { - sb.append(analyzeReduceTaskNum()); - sb.append(analyzeMapTaskNum()); + sb.append(analyzeReduceTaskNum(optSettings)); + sb.append(analyzeMapTaskNum(optSettings)); if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.NONE, sb.toString()); + return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString(), optSettings); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it @@ -48,7 +54,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } - private String analyzeReduceTaskNum() { + private String analyzeReduceTaskNum(List optSettings) { StringBuilder sb = new StringBuilder(); long numReduces = context.getNumReduces(); @@ -107,13 +113,15 @@ private String analyzeReduceTaskNum() { } else { sb.append("decreasing the "); } - sb.append("reducer number. You could try -Dmapreduce.job.reduces=").append(suggestReduces).append("\n"); + String setting = String.format("-D%s=%s", NUM_REDUCES, suggestReduces); + sb.append("reducer number. You could try ").append(setting).append("\n"); + optSettings.add(setting); } } return sb.toString(); } - private String analyzeMapTaskNum() { + private String analyzeMapTaskNum(List optSettings) { StringBuilder sb = new StringBuilder(); long numMaps = context.getNumMaps(); diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java index 4a63365943..b66286bc04 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java @@ -71,16 +71,11 @@ public void publish(AnalyzerEntity analyzerJobEntity, Result result) { basic.put("progress", progress + "%"); basic.put("detail", getJobLink(analyzerJobEntity)); - - Map> extend = new HashMap<>(); - Map>> alertMessages = result.getAlertMessages(); - for (String evaluator : alertMessages.keySet()) { - List> messages = alertMessages.get(evaluator); - extend.put(evaluator, new HashMap<>()); - for (Pair message : messages) { + Map> extend = result.getAlertMessages(); + for (String evaluator : extend.keySet()) { + for (Result.ProcessorResult message : extend.get(evaluator)) { LOG.info("Job [{}] Got Message [{}], Level [{}] By Evaluator [{}]", - analyzerJobEntity.getJobDefId(), message.getRight(), message.getLeft(), evaluator); - extend.get(evaluator).put(message.getRight(), message.getLeft().toString()); + analyzerJobEntity.getJobDefId(), message.getMessage(), message.getResultLevel(), evaluator); } } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java index a12f589ea0..0c77f525f6 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java @@ -17,7 +17,7 @@ package org.apache.eagle.jpm.analyzer.publisher; -import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.StringUtils; import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity; import java.util.ArrayList; @@ -29,7 +29,7 @@ public class Result { //for EagleStorePublisher private TaggedLogAPIEntity alertEntity = null;//TODO //for EmailPublisher - private Map>> alertMessages = new HashMap<>(); + private Map> alertMessages = new HashMap<>(); public void addEvaluatorResult(Class type, EvaluatorResult result) { Map, ProcessorResult> processorResults = result.getProcessorResults(); @@ -43,7 +43,8 @@ public void addEvaluatorResult(Class type, EvaluatorResult result) { if (!alertMessages.containsKey(typeName)) { alertMessages.put(typeName, new ArrayList<>()); } - alertMessages.get(typeName).add(Pair.of(processorResult.getResultLevel(), processorResult.getMessage())); + normalizeResult(processorResult); + alertMessages.get(typeName).add(processorResult); } } @@ -51,16 +52,23 @@ public TaggedLogAPIEntity getAlertEntity() { return alertEntity; } - public Map>> getAlertMessages() { + public Map> getAlertMessages() { return alertMessages; } + private void normalizeResult(ProcessorResult processorResult) { + if (processorResult.getSettings() != null && !processorResult.getSettings().isEmpty()) { + processorResult.setSuggestion(StringUtils.join(processorResult.getSettings(), "\n")); + } + } + /** * Processor result. */ public enum ResultLevel { NONE, + INFO, NOTICE, WARNING, CRITICAL @@ -69,10 +77,19 @@ public enum ResultLevel { public static class ProcessorResult { private ResultLevel resultLevel; private String message; + private List settings; + private String suggestion; + + public ProcessorResult(ResultLevel resultLevel, String message, List settings) { + this.resultLevel = resultLevel; + this.message = message; + this.settings = settings; + } public ProcessorResult(ResultLevel resultLevel, String message) { this.resultLevel = resultLevel; this.message = message; + this.settings = new ArrayList<>(); } public ResultLevel getResultLevel() { @@ -90,6 +107,22 @@ public String getMessage() { public void setMessage(String message) { this.message = message; } + + public List getSettings() { + return settings; + } + + public void setSettings(List settings) { + this.settings = settings; + } + + public String getSuggestion() { + return suggestion; + } + + public void setSuggestion(String suggestion) { + this.suggestion = suggestion; + } } /** diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm b/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm index 39cec6850d..32716db12f 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm @@ -115,13 +115,15 @@ - + + - #foreach($message in ${elem["extend"].get($evaluator).keySet()}) + #foreach($result in ${elem["extend"].get($evaluator)}) - - + + + #end
Analysis By $evaluator
levellevel messagesuggestion
${elem["extend"].get($evaluator).get($message)}$message${result.resultLevel}${result.message}${result.suggestion}
From 13054d2adb2671c4a61d0608cef9b4f0493661fd Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Wed, 18 Jan 2017 21:35:48 +0800 Subject: [PATCH 19/22] update AnalyzerReportTemplate.vm --- .../src/main/resources/AnalyzerReportTemplate.vm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm b/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm index 32716db12f..5f9db76c89 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm @@ -122,8 +122,8 @@ #foreach($result in ${elem["extend"].get($evaluator)}) ${result.resultLevel} - ${result.message} - ${result.suggestion} + ${result.message} + ${result.suggestion} #end From 19befb11e3b09de72b45d41df88861fde37834a1 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Thu, 19 Jan 2017 10:17:19 +0800 Subject: [PATCH 20/22] refactor code --- .../apache/eagle/jpm/analyzer/Evaluator.java | 1 + .../eagle/jpm/analyzer/JobAnalyzer.java | 2 ++ .../apache/eagle/jpm/analyzer/Processor.java | 1 + .../{ => meta/model}/AnalyzerEntity.java | 26 +++++++++---------- .../meta/model/MapReduceAnalyzerEntity.java | 1 - .../analyzer/mr/MRJobPerformanceAnalyzer.java | 3 +-- .../jpm/analyzer/mr/sla/SLAJobEvaluator.java | 2 +- .../sla/processors/LongStuckJobProcessor.java | 2 +- .../UnExpectedLongDurationJobProcessor.java | 2 +- .../publisher/EagleStorePublisher.java | 2 +- .../analyzer/publisher/EmailPublisher.java | 3 +-- .../jpm/analyzer/publisher/Publisher.java | 2 +- .../publisher/dedup/AlertDeduplicator.java | 2 +- .../dedup/impl/SimpleDeduplicator.java | 2 +- .../jpm/mr/running/parser/MRJobParser.java | 2 +- 15 files changed, 27 insertions(+), 26 deletions(-) rename eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/{ => meta/model}/AnalyzerEntity.java (73%) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java index 17fc119a23..60ee8d688c 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Evaluator.java @@ -17,6 +17,7 @@ package org.apache.eagle.jpm.analyzer; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; public interface Evaluator { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java index b0955ebfd0..1e9c00ef37 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/JobAnalyzer.java @@ -17,6 +17,8 @@ package org.apache.eagle.jpm.analyzer; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; + /** * Each JobAnalyzer contains one or more Evaluators to analyze each job. * Each Evaluator is a group of Processors diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java index 0e8b417757..419e402612 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/Processor.java @@ -17,6 +17,7 @@ package org.apache.eagle.jpm.analyzer; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; public interface Processor { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/AnalyzerEntity.java similarity index 73% rename from eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java rename to eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/AnalyzerEntity.java index a600f79ee3..189d85dba1 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/AnalyzerEntity.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/AnalyzerEntity.java @@ -1,21 +1,21 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -package org.apache.eagle.jpm.analyzer; +package org.apache.eagle.jpm.analyzer.meta.model; import java.util.HashMap; import java.util.Map; diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java index 5500d28ff7..cd6249de41 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/meta/model/MapReduceAnalyzerEntity.java @@ -17,7 +17,6 @@ package org.apache.eagle.jpm.analyzer.meta.model; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; import org.apache.eagle.jpm.mr.historyentity.TaskAttemptExecutionAPIEntity; import org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity; import org.apache.eagle.jpm.util.jobcounter.JobCounters; diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index 129044025b..f9f7431ece 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -20,10 +20,9 @@ import com.typesafe.config.Config; import org.apache.eagle.jpm.analyzer.*; import org.apache.eagle.jpm.analyzer.Evaluator; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.mr.sla.SLAJobEvaluator; import org.apache.eagle.jpm.analyzer.mr.suggestion.JobSuggestionEvaluator; -import org.apache.eagle.jpm.analyzer.publisher.EagleStorePublisher; import org.apache.eagle.jpm.analyzer.publisher.EmailPublisher; import org.apache.eagle.jpm.analyzer.publisher.Publisher; import org.apache.eagle.jpm.analyzer.publisher.Result; diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java index f8c2dcb26a..a77e55de2a 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/SLAJobEvaluator.java @@ -18,7 +18,7 @@ package org.apache.eagle.jpm.analyzer.mr.sla; import com.typesafe.config.Config; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.Evaluator; import org.apache.eagle.jpm.analyzer.meta.model.JobMetaEntity; import org.apache.eagle.jpm.analyzer.mr.sla.processors.LongStuckJobProcessor; diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/LongStuckJobProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/LongStuckJobProcessor.java index 35f3b27aae..4903f83663 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/LongStuckJobProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/LongStuckJobProcessor.java @@ -18,7 +18,7 @@ package org.apache.eagle.jpm.analyzer.mr.sla.processors; import com.typesafe.config.Config; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.Processor; import org.apache.eagle.jpm.analyzer.publisher.Result; import org.slf4j.Logger; diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java index 9d4ce2b79d..415fdfc74d 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java @@ -18,7 +18,7 @@ package org.apache.eagle.jpm.analyzer.mr.sla.processors; import com.typesafe.config.Config; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.analyzer.Processor; import org.apache.eagle.jpm.analyzer.util.Constants; diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EagleStorePublisher.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EagleStorePublisher.java index 610970411c..42dca606a4 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EagleStorePublisher.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EagleStorePublisher.java @@ -18,7 +18,7 @@ package org.apache.eagle.jpm.analyzer.publisher; import com.typesafe.config.Config; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java index b66286bc04..842e0ac35c 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EmailPublisher.java @@ -18,11 +18,10 @@ package org.apache.eagle.jpm.analyzer.publisher; import com.typesafe.config.Config; -import org.apache.commons.lang3.tuple.Pair; import org.apache.eagle.app.service.ApplicationEmailService; import org.apache.eagle.common.DateTimeUtil; import org.apache.eagle.common.mail.AlertEmailContext; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.dedup.AlertDeduplicator; import org.apache.eagle.jpm.analyzer.publisher.dedup.impl.SimpleDeduplicator; import org.apache.eagle.jpm.analyzer.util.Constants; diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Publisher.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Publisher.java index 2f42bf94d6..1f42ef9d0a 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Publisher.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Publisher.java @@ -17,7 +17,7 @@ package org.apache.eagle.jpm.analyzer.publisher; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; public interface Publisher { void publish(AnalyzerEntity analyzerJobEntity, Result result); diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/dedup/AlertDeduplicator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/dedup/AlertDeduplicator.java index 4b18f7c27c..6a51a76412 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/dedup/AlertDeduplicator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/dedup/AlertDeduplicator.java @@ -17,7 +17,7 @@ package org.apache.eagle.jpm.analyzer.publisher.dedup; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; public interface AlertDeduplicator { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/dedup/impl/SimpleDeduplicator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/dedup/impl/SimpleDeduplicator.java index 09f1af6c12..b139b3c250 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/dedup/impl/SimpleDeduplicator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/dedup/impl/SimpleDeduplicator.java @@ -17,7 +17,7 @@ package org.apache.eagle.jpm.analyzer.publisher.dedup.impl; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.analyzer.publisher.dedup.AlertDeduplicator; import org.apache.eagle.jpm.analyzer.util.Constants; diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java index 6a9422744c..6b33d31a2d 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java @@ -19,7 +19,7 @@ package org.apache.eagle.jpm.mr.running.parser; import com.typesafe.config.Config; -import org.apache.eagle.jpm.analyzer.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.mr.MRJobPerformanceAnalyzer; import org.apache.eagle.jpm.mr.running.MRRunningJobConfig; import org.apache.eagle.jpm.mr.running.recover.MRRunningJobManager; From 218c1e229810fd411a0a2302f985410f70af6ed6 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Thu, 19 Jan 2017 14:34:28 +0800 Subject: [PATCH 21/22] update --- .../analyzer/mr/MRJobPerformanceAnalyzer.java | 3 +- .../sla/processors/LongStuckJobProcessor.java | 2 +- .../UnExpectedLongDurationJobProcessor.java | 6 +- .../mr/suggestion/JobSuggestionEvaluator.java | 24 +++++++ .../MapReduceCompressionSettingProcessor.java | 2 +- .../MapReduceDataSkewProcessor.java | 2 +- .../suggestion/MapReduceGCTimeProcessor.java | 2 +- .../MapReduceQueueResourceProcessor.java | 2 +- .../suggestion/MapReduceSpillProcessor.java | 2 +- .../MapReduceSplitSettingProcessor.java | 2 +- .../suggestion/MapReduceTaskNumProcessor.java | 2 +- .../publisher/EagleStorePublisher.java | 38 +++++++++++ .../eagle/jpm/analyzer/publisher/Result.java | 67 +++++++++++++++---- .../eagle/jpm/analyzer/util/Constants.java | 1 + .../main/resources/AnalyzerReportTemplate.vm | 10 +-- .../mr/historyentity/JPAEntityRepository.java | 1 + .../historyentity/JobSuggestionAPIEntity.java | 63 +++++++++++++++++ .../org/apache/eagle/jpm/util/Constants.java | 1 + 18 files changed, 201 insertions(+), 29 deletions(-) create mode 100644 eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index f9f7431ece..e32a37c0f4 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -23,6 +23,7 @@ import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; import org.apache.eagle.jpm.analyzer.mr.sla.SLAJobEvaluator; import org.apache.eagle.jpm.analyzer.mr.suggestion.JobSuggestionEvaluator; +import org.apache.eagle.jpm.analyzer.publisher.EagleStorePublisher; import org.apache.eagle.jpm.analyzer.publisher.EmailPublisher; import org.apache.eagle.jpm.analyzer.publisher.Publisher; import org.apache.eagle.jpm.analyzer.publisher.Result; @@ -46,7 +47,7 @@ public MRJobPerformanceAnalyzer(Config config) { evaluators.add(new SLAJobEvaluator(config)); evaluators.add(new JobSuggestionEvaluator(config)); - //publishers.add(new EagleStorePublisher(config)); + publishers.add(new EagleStorePublisher(config)); publishers.add(new EmailPublisher(config)); } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/LongStuckJobProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/LongStuckJobProcessor.java index 4903f83663..b3322ed4ea 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/LongStuckJobProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/LongStuckJobProcessor.java @@ -38,6 +38,6 @@ public LongStuckJobProcessor(Config config) { @Override public Result.ProcessorResult process(AnalyzerEntity analyzerJobEntity) { LOG.info("Job {} In LongStuckJobProcessor", analyzerJobEntity.getJobDefId()); - return new Result.ProcessorResult(Result.ResultLevel.NONE, ""); + return new Result.ProcessorResult(Result.RuleType.LONG_STUCK_JOB, Result.ResultLevel.NONE, ""); } } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java index 415fdfc74d..8f655bae67 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java @@ -50,7 +50,7 @@ public Result.ProcessorResult process(AnalyzerEntity analyzerJobEntity) { Map jobMetaData = analyzerJobEntity.getJobMeta(); long avgDurationTime = getAvgDuration(analyzerJobEntity, jobMetaData); if (avgDurationTime == 0L) { - return new Result.ProcessorResult(Result.ResultLevel.NONE, Constants.PROCESS_NONE); + return new Result.ProcessorResult(Result.RuleType.LONG_DURATION_JOB, Result.ResultLevel.NONE, Constants.PROCESS_NONE); } Map alertThreshold = Constants.DEFAULT_ALERT_THRESHOLD; @@ -62,12 +62,12 @@ public Result.ProcessorResult process(AnalyzerEntity analyzerJobEntity) { double expirePercent = (analyzerJobEntity.getDurationTime() - avgDurationTime) * 1.0 / avgDurationTime; for (Map.Entry entry : sorted) { if (expirePercent >= entry.getValue()) { - return new Result.ProcessorResult(entry.getKey(), String.format("Job duration exceeds average duration by %d%%, average duration is %ds", + return new Result.ProcessorResult(Result.RuleType.LONG_DURATION_JOB, entry.getKey(), String.format("Job duration exceeds average duration by %d%%, average duration is %ds", (int)(expirePercent * 100), avgDurationTime / 1000)); } } - return new Result.ProcessorResult(Result.ResultLevel.NONE, Constants.PROCESS_NONE); + return new Result.ProcessorResult(Result.RuleType.LONG_DURATION_JOB, Result.ResultLevel.NONE, Constants.PROCESS_NONE); } private long getAvgDuration(AnalyzerEntity mrJobAnalysisEntity, Map jobMetaData) { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java index 9b53561e44..52356311d1 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java @@ -20,6 +20,7 @@ import com.typesafe.config.Config; import org.apache.eagle.jpm.analyzer.Evaluator; import org.apache.eagle.jpm.analyzer.Processor; +import org.apache.eagle.jpm.mr.historyentity.JobSuggestionAPIEntity; import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity; import org.apache.eagle.jpm.analyzer.publisher.Result; import org.apache.eagle.jpm.util.Constants; @@ -28,7 +29,11 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; + +import static org.apache.eagle.jpm.util.MRJobTagName.*; public class JobSuggestionEvaluator implements Evaluator, Serializable { private static final Logger LOG = LoggerFactory.getLogger(JobSuggestionEvaluator.class); @@ -69,6 +74,7 @@ public Result.EvaluatorResult evaluate(MapReduceAnalyzerEntity analyzerEntity) { Result.ProcessorResult processorResult = processor.process(analyzerEntity); if (processorResult != null) { result.addProcessorResult(processor.getClass(), processorResult); + result.addProcessorEntity(processor.getClass(), createJobSuggestionEntity(processorResult, analyzerEntity)); } } return result; @@ -78,4 +84,22 @@ public Result.EvaluatorResult evaluate(MapReduceAnalyzerEntity analyzerEntity) { } } + + private static JobSuggestionAPIEntity createJobSuggestionEntity(Result.ProcessorResult processorResult, MapReduceAnalyzerEntity entity) { + Map tags = new HashMap<>(); + tags.put(JOB_ID.toString(), entity.getJobId()); + tags.put(JOD_DEF_ID.toString(), entity.getJobDefId()); + tags.put(SITE.toString(), entity.getSiteId()); + tags.put(USER.toString(), entity.getUserId()); + tags.put(RULE_TYPE.toString(), processorResult.getRuleType().toString()); + tags.put(JOB_QUEUE.toString(), entity.getJobQueueName()); + JobSuggestionAPIEntity jobSuggestionAPIEntity = new JobSuggestionAPIEntity(); + jobSuggestionAPIEntity.setTags(tags); + jobSuggestionAPIEntity.setTimestamp(entity.getStartTime()); // startTime as the job timestamp + jobSuggestionAPIEntity.setOptimizerSuggestion(processorResult.getMessage()); + jobSuggestionAPIEntity.setOptimizerSettings(processorResult.getSettings()); + + return jobSuggestionAPIEntity; + } + } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java index bdac03bfc8..62c5c2bf3c 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceCompressionSettingProcessor.java @@ -75,7 +75,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString(), optSettings); + return new Result.ProcessorResult(Result.RuleType.COMPRESS, Result.ResultLevel.INFO, sb.toString(), optSettings); } return null; } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java index 52629fe627..b21a927ba2 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceDataSkewProcessor.java @@ -53,7 +53,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString()); + return new Result.ProcessorResult(Result.RuleType.DATA_SKEW, Result.ResultLevel.INFO, sb.toString()); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java index bb3830146c..103de7a4a5 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java @@ -64,7 +64,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString(), optSettings); + return new Result.ProcessorResult(Result.RuleType.GC_TIME, Result.ResultLevel.INFO, sb.toString(), optSettings); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java index d444facf1c..a1b57bf4ba 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceQueueResourceProcessor.java @@ -70,7 +70,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString()); + return new Result.ProcessorResult(Result.RuleType.RESOURCE_CONTENTION, Result.ResultLevel.INFO, sb.toString()); } } } catch (Exception e) { diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java index 911a489bad..97bba51ad9 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java @@ -101,7 +101,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) } if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString(), optSettings); + return new Result.ProcessorResult(Result.RuleType.SPILL, Result.ResultLevel.INFO, sb.toString(), optSettings); } } catch (NullPointerException e) { //When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java index 304642965f..8eba4687a2 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSplitSettingProcessor.java @@ -40,7 +40,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) if (context.getJobconf().getLong(FileInputFormat.SPLIT_MINSIZE, 0) > 1) { sb.append("Best practice: don't set " + FileInputFormat.SPLIT_MINSIZE); sb.append(", because it may lower data locality, hence maps will run slower.\n"); - return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString()); + return new Result.ProcessorResult(Result.RuleType.SPLIT, Result.ResultLevel.INFO, sb.toString()); } return null; } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java index 61c4ead3fc..00d5cc988f 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceTaskNumProcessor.java @@ -45,7 +45,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) sb.append(analyzeMapTaskNum(optSettings)); if (sb.length() > 0) { - return new Result.ProcessorResult(Result.ResultLevel.INFO, sb.toString(), optSettings); + return new Result.ProcessorResult(Result.RuleType.TASK_NUMBER, Result.ResultLevel.INFO, sb.toString(), optSettings); } } catch (NullPointerException e) { // When job failed there may not have counters, so just ignore it diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EagleStorePublisher.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EagleStorePublisher.java index 42dca606a4..0d7d2d7783 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EagleStorePublisher.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/EagleStorePublisher.java @@ -19,22 +19,60 @@ import com.typesafe.config.Config; import org.apache.eagle.jpm.analyzer.meta.model.AnalyzerEntity; +import org.apache.eagle.jpm.analyzer.publisher.dedup.AlertDeduplicator; +import org.apache.eagle.jpm.analyzer.publisher.dedup.impl.SimpleDeduplicator; +import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity; +import org.apache.eagle.service.client.IEagleServiceClient; +import org.apache.eagle.service.client.impl.EagleServiceClientImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.io.Serializable; +import java.util.List; +import java.util.Map; public class EagleStorePublisher implements Publisher, Serializable { private static final Logger LOG = LoggerFactory.getLogger(EagleStorePublisher.class); private Config config; + private IEagleServiceClient client; + private AlertDeduplicator alertDeduplicator; public EagleStorePublisher(Config config) { this.config = config; + this.alertDeduplicator = new SimpleDeduplicator(); } @Override public void publish(AnalyzerEntity analyzerJobEntity, Result result) { + if (result.getAlertMessages().size() == 0) { + return; + } + + LOG.info("EagleStorePublisher gets job {}", analyzerJobEntity.getJobDefId()); + if (alertDeduplicator.dedup(analyzerJobEntity, result)) { + LOG.info("skip job {} alert because it is duplicated", analyzerJobEntity.getJobDefId()); + return; + } + + try { + this.client = new EagleServiceClientImpl(config); + for (Map.Entry> entry : result.getAlertEntities().entrySet()) { + client.create(entry.getValue()); + LOG.info("successfully persist {} entities for evaluator {}", entry.getValue().size(), entry.getKey()); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } finally { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + } } } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java index 0c77f525f6..e65d2f8474 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java @@ -27,38 +27,46 @@ public class Result { //for EagleStorePublisher - private TaggedLogAPIEntity alertEntity = null;//TODO + private Map> alertEntities = new HashMap<>(); //for EmailPublisher private Map> alertMessages = new HashMap<>(); public void addEvaluatorResult(Class type, EvaluatorResult result) { Map, ProcessorResult> processorResults = result.getProcessorResults(); + Map, TaggedLogAPIEntity> processorEntities = result.getProcessorEntities(); + for (Class processorType : processorResults.keySet()) { ProcessorResult processorResult = processorResults.get(processorType); + if (processorResult.resultLevel.equals(ResultLevel.NONE)) { continue; } + TaggedLogAPIEntity entity = processorEntities.get(processorType); + String typeName = type.getName(); if (!alertMessages.containsKey(typeName)) { alertMessages.put(typeName, new ArrayList<>()); + alertEntities.put(typeName, new ArrayList<>()); } normalizeResult(processorResult); alertMessages.get(typeName).add(processorResult); - } - } + alertEntities.get(typeName).add(entity); - public TaggedLogAPIEntity getAlertEntity() { - return alertEntity; + } } public Map> getAlertMessages() { return alertMessages; } + public Map> getAlertEntities() { + return alertEntities; + } + private void normalizeResult(ProcessorResult processorResult) { if (processorResult.getSettings() != null && !processorResult.getSettings().isEmpty()) { - processorResult.setSuggestion(StringUtils.join(processorResult.getSettings(), "\n")); + processorResult.setSettingList(StringUtils.join(processorResult.getSettings(), "\n")); } } @@ -74,24 +82,48 @@ public enum ResultLevel { CRITICAL } + public enum RuleType { + COMPRESS, + SPLIT, + SPILL, + TASK_NUMBER, + GC_TIME, + RESOURCE_CONTENTION, + DATA_SKEW, + + LONG_STUCK_JOB, + LONG_DURATION_JOB + } + public static class ProcessorResult { + private RuleType ruleType; private ResultLevel resultLevel; private String message; private List settings; - private String suggestion; + private String settingList; - public ProcessorResult(ResultLevel resultLevel, String message, List settings) { + public ProcessorResult(RuleType ruleType, ResultLevel resultLevel, String message, List settings) { + this.ruleType = ruleType; this.resultLevel = resultLevel; this.message = message; this.settings = settings; } - public ProcessorResult(ResultLevel resultLevel, String message) { + public ProcessorResult(RuleType ruleType, ResultLevel resultLevel, String message) { + this.ruleType = ruleType; this.resultLevel = resultLevel; this.message = message; this.settings = new ArrayList<>(); } + public RuleType getRuleType() { + return ruleType; + } + + public void setRuleType(RuleType ruleType) { + this.ruleType = ruleType; + } + public ResultLevel getResultLevel() { return resultLevel; } @@ -116,12 +148,12 @@ public void setSettings(List settings) { this.settings = settings; } - public String getSuggestion() { - return suggestion; + public String getSettingList() { + return settingList; } - public void setSuggestion(String suggestion) { - this.suggestion = suggestion; + public void setSettingList(String settingList) { + this.settingList = settingList; } } @@ -130,13 +162,22 @@ public void setSuggestion(String suggestion) { */ public static class EvaluatorResult { private Map, ProcessorResult> processorResults = new HashMap<>(); + private Map, TaggedLogAPIEntity> processorEntities = new HashMap<>(); public void addProcessorResult(Class type, ProcessorResult result) { this.processorResults.put(type, result); } + public void addProcessorEntity(Class type, TaggedLogAPIEntity entity) { + this.processorEntities.put(type, entity); + } + public Map, ProcessorResult> getProcessorResults() { return this.processorResults; } + + public Map, TaggedLogAPIEntity> getProcessorEntities() { + return processorEntities; + } } } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Constants.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Constants.java index 774e6d28ec..4c6661a43a 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Constants.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Constants.java @@ -62,4 +62,5 @@ public class Constants { public static final String ANALYZER_REPORT_DATA_BASIC_KEY = "basic"; public static final String ANALYZER_REPORT_DATA_EXTEND_KEY = "extend"; + } diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm b/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm index 5f9db76c89..996adba044 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/resources/AnalyzerReportTemplate.vm @@ -115,15 +115,17 @@ - + - + + #foreach($result in ${elem["extend"].get($evaluator)}) - + - + + #end
Analysis By $evaluator
leveltype messagesuggestionoptimizer settinglevel
${result.resultLevel}${result.ruleType} ${result.message}${result.suggestion}${result.settingList}${result.resultLevel}
diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java index cbbdad3445..8c65adfb14 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java @@ -38,5 +38,6 @@ public JPAEntityRepository() { entitySet.add(JobProcessTimeStampEntity.class); entitySet.add(JobCountEntity.class); entitySet.add(TaskAttemptErrorCategoryEntity.class); + entitySet.add(JobSuggestionAPIEntity.class); } } diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java new file mode 100644 index 0000000000..3863a5d323 --- /dev/null +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.mr.historyentity; + +import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity; +import org.apache.eagle.log.entity.meta.*; + +import java.util.List; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import static org.apache.eagle.jpm.util.Constants.JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME; + +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@Table("eaglejpa") +@ColumnFamily("f") +@Prefix("jsuggestion") +@Service(JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME) +@TimeSeries(true) +@Partition({"site"}) +@Indexes({ + @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = true), + @Index(name = "Index_2_jobDefId", columns = { "jobDefId" }, unique = false) + }) +public class JobSuggestionAPIEntity extends TaggedLogAPIEntity { + @Column("a") + private String optimizerSuggestion; + @Column("b") + private List optimizerSettings; + + public String getOptimizerSuggestion() { + return optimizerSuggestion; + } + + public void setOptimizerSuggestion(String optimizerSuggestion) { + this.optimizerSuggestion = optimizerSuggestion; + valueChanged("optimizerSuggestion"); + } + + public List getOptimizerSettings() { + return optimizerSettings; + } + + public void setOptimizerSettings(List optimizerSettings) { + this.optimizerSettings = optimizerSettings; + valueChanged("optimizerSettings"); + } + +} diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java index 0ba6521bc3..4ee58a1148 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java @@ -117,6 +117,7 @@ public static enum SuggestionType { public static final String JPA_RUNNING_TASK_EXECUTION_SERVICE_NAME = "RunningTaskExecutionService"; public static final String JPA_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "RunningTaskAttemptExecutionService"; public static final String JPA_JOB_PROCESS_TIME_STAMP_NAME = "JobProcessTimeStampService"; + public static final String JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME = "JobOptimizerSuggestionService"; public static final String JOB_TASK_TYPE_TAG = "taskType"; From a3adf45d70d4b52b4734a13a391c1b320077166b Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Thu, 19 Jan 2017 16:38:06 +0800 Subject: [PATCH 22/22] fix code style --- .../jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java | 1 + .../analyzer/mr/suggestion/MapReduceSpillProcessor.java | 6 +++--- .../org/apache/eagle/jpm/analyzer/publisher/Result.java | 4 +--- .../eagle/jpm/mr/history/parser/JobSuggestionListener.java | 7 +++++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java index 52356311d1..ea60ff9667 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java @@ -93,6 +93,7 @@ private static JobSuggestionAPIEntity createJobSuggestionEntity(Result.Processor tags.put(USER.toString(), entity.getUserId()); tags.put(RULE_TYPE.toString(), processorResult.getRuleType().toString()); tags.put(JOB_QUEUE.toString(), entity.getJobQueueName()); + tags.put(JOB_TYPE.toString(), entity.getJobType()); JobSuggestionAPIEntity jobSuggestionAPIEntity = new JobSuggestionAPIEntity(); jobSuggestionAPIEntity.setTags(tags); jobSuggestionAPIEntity.setTimestamp(entity.getStartTime()); // startTime as the job timestamp diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java index 97bba51ad9..835b382ac9 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java @@ -57,8 +57,8 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) spillRecords = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS); if (outputRecords < spillRecords) { - sb.append("Total Map output records: ").append(outputRecords); - sb.append(" Total Spilled Records: ").append(spillRecords).append(". Please set"); + sb.append("Total map output records: ").append(outputRecords); + sb.append(" Total map spilled records: ").append(spillRecords).append(". Please set"); long minMapSpillMemBytes = context.getMinMapSpillMemBytes(); double spillPercent = context.getJobconf().getDouble(MAP_SORT_SPILL_PERCENT, 0.8); @@ -74,7 +74,7 @@ public Result.ProcessorResult process(MapReduceAnalyzerEntity jobAnalysisEntity) long minMapSpillMemMB = (minMapSpillMemBytes / FileUtils.ONE_MB + 10) / 10 * 10; if (minMapSpillMemMB >= 2047 ) { - sb.append(" Please reduce the block size of the input files and make sure they are splittable."); + sb.append("\nPlease reduce the block size of the input files and make sure they are splittable."); } else { setting = String.format("-D%s=%s", IO_SORT_MB, minMapSpillMemMB); sb.append(" ").append(setting); diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java index e65d2f8474..a9f51324fb 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java @@ -42,8 +42,6 @@ public void addEvaluatorResult(Class type, EvaluatorResult result) { continue; } - TaggedLogAPIEntity entity = processorEntities.get(processorType); - String typeName = type.getName(); if (!alertMessages.containsKey(typeName)) { alertMessages.put(typeName, new ArrayList<>()); @@ -51,7 +49,7 @@ public void addEvaluatorResult(Class type, EvaluatorResult result) { } normalizeResult(processorResult); alertMessages.get(typeName).add(processorResult); - alertEntities.get(typeName).add(entity); + alertEntities.get(typeName).add(processorEntities.get(processorType)); } } diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java index b38d4622a3..e5b0d2ed6d 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JobSuggestionListener.java @@ -32,6 +32,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.eagle.jpm.util.MRJobTagName.TASK_ATTEMPT_ID; +import static org.apache.eagle.jpm.util.MRJobTagName.TASK_ID; + /* * JobEventCounterListener provides an interface to add job/task counter analyzers */ @@ -49,9 +52,9 @@ public JobSuggestionListener(Config config) { @Override public void jobEntityCreated(JobBaseAPIEntity entity) throws Exception { if (entity instanceof TaskExecutionAPIEntity) { - info.getTasksMap().put(entity.getTags().get(Keys.TASKID), (TaskExecutionAPIEntity) entity); + info.getTasksMap().put(entity.getTags().get(TASK_ID.toString()), (TaskExecutionAPIEntity) entity); } else if (entity instanceof TaskAttemptExecutionAPIEntity) { - info.getCompletedTaskAttemptsMap().put(entity.getTags().get(Keys.TASK_ATTEMPT_ID), (TaskAttemptExecutionAPIEntity) entity); + info.getCompletedTaskAttemptsMap().put(entity.getTags().get(TASK_ATTEMPT_ID.toString()), (TaskAttemptExecutionAPIEntity) entity); } else if (entity instanceof JobExecutionAPIEntity) { JobExecutionAPIEntity jobExecutionAPIEntity = (JobExecutionAPIEntity) entity; info.setCurrentState(jobExecutionAPIEntity.getCurrentState());