From 20d7a1555f269c1e1728cd843e1369ae453deaa0 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Wed, 12 Apr 2017 14:46:53 +0800 Subject: [PATCH 1/5] refine app config --- .../org.apache.eagle.gc.GCLogApplicationProvider.xml | 4 ++-- ...e.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml | 4 ++-- ...e.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml | 2 +- ...g.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml | 4 ++-- ...he.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml b/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml index 49d54412f4..afa6d77edd 100644 --- a/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml +++ b/eagle-gc/src/main/resources/META-INF/providers/org.apache.eagle.gc.GCLogApplicationProvider.xml @@ -54,7 +54,7 @@ dataSourceConfig.topic dataSourceConfig.topic - gc_log + gc_log_${siteId} data source topic @@ -112,7 +112,7 @@ dataSinkConfig.topic dataSinkConfig.topic - hbase_audit_log_parsed + gc_log_event_${siteId} topic for kafka data sink diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml index e5c001a63f..5c479a3cf0 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml @@ -76,14 +76,14 @@ dataSinkConfig.MAP_REDUCE_JOB_STREAM.topic Destination(Kafka Topic) Of Job Stream Data - sandbox-map_reduce_job + map_reduce_job_${siteId} application emits job stream data to this kafka topic true dataSinkConfig.MAP_REDUCE_TASK_ATTEMPT_STREAM.topic Destination(Kafka Topic) Of Task Attempt Stream Data - sandbox-map_reduce_task_attempt + map_reduce_task_attempt_${siteId} application emits task attempt stream data to this kafka topic true diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml index 907e35a947..2fe957510d 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml @@ -104,7 +104,7 @@ dataSinkConfig.ACCEPTED_APP_STREAM.topic Destination(Kafka Topic) Of App Stream Data - yarn_accepted_app_{siteId} + yarn_accepted_app_${siteId} topic for kafka data sink true diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml index 18b2ed34e5..1fbf86315a 100644 --- a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml +++ b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml @@ -24,7 +24,7 @@ dataSourceConfig.topic dataSourceConfig.topic - hbase_audit_log + hbase_audit_log_${siteId} data source topic @@ -104,7 +104,7 @@ dataSinkConfig.topic dataSinkConfig.topic - hbase_audit_log_parsed + hbase_audit_event_${siteId} topic for kafka data sink diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml index 5434a9dfe7..289229db39 100644 --- a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml +++ b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml @@ -130,7 +130,7 @@ dataSinkConfig.topic Kafka Topic for Parsed Data Sink - oozie_audit_log_enriched + oozie_audit_event_${siteId} topic for kafka data sink From 3099b71e204d8f3c115b82b4e21495e84e932fb0 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Tue, 18 Apr 2017 14:52:34 +0800 Subject: [PATCH 2/5] fix null pointer exception in JobCounters.java --- .../jpm/analyzer/mr/MRJobPerformanceAnalyzer.java | 12 ++++++++---- .../eagle/jpm/util/jobcounter/JobCounters.java | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java index 34365dc768..cca6b1885e 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java @@ -61,14 +61,18 @@ public void analyze(T analyzerJobEntity) throws Exception { Result result = new Result(); for (Evaluator evaluator : evaluators) { - Result.EvaluatorResult evaluatorResult = evaluator.evaluate(analyzerJobEntity); - if (evaluatorResult != null) { - result.addEvaluatorResult(evaluator.getClass(), evaluatorResult); + try { + Result.EvaluatorResult evaluatorResult = evaluator.evaluate(analyzerJobEntity); + if (evaluatorResult != null) { + result.addEvaluatorResult(evaluator.getClass(), evaluatorResult); + } + } catch (Throwable e) { + LOG.error("evaluator {} fails to analyse job {}", evaluator, analyzerJobEntity.getJobId(), e); } } if (alertDeduplicator.dedup(analyzerJobEntity, result)) { - LOG.info("skip publish job {} alert because it is duplicated", analyzerJobEntity.getJobDefId()); + LOG.info("skip publish job {} alert because it is duplicated", analyzerJobEntity.getJobId()); return; } diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java index bbb80cdf16..bcf6de1a80 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java @@ -48,7 +48,8 @@ public void clear() { } public Long getCounterValue(CounterName counterName) { - if (counters.get(counterName.group.name).containsKey(counterName.name)) { + if (counters.containsKey(counterName.group.name) && + counters.get(counterName.group.name).containsKey(counterName.name)) { return counters.get(counterName.group.name).get(counterName.name); } else { return 0L; From 92d5deb63750557642989b47a9b2d5e66ec8ae32 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Tue, 18 Apr 2017 15:21:19 +0800 Subject: [PATCH 3/5] fix code format --- .../org/apache/eagle/jpm/util/jobcounter/JobCounters.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java index bcf6de1a80..e337c13641 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java @@ -48,8 +48,8 @@ public void clear() { } public Long getCounterValue(CounterName counterName) { - if (counters.containsKey(counterName.group.name) && - counters.get(counterName.group.name).containsKey(counterName.name)) { + if (counters.containsKey(counterName.group.name) + && counters.get(counterName.group.name).containsKey(counterName.name)) { return counters.get(counterName.group.name).get(counterName.name); } else { return 0L; From 536c17310b3d9a8139834dd84ef186b9223c08bf Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Tue, 18 Apr 2017 15:30:55 +0800 Subject: [PATCH 4/5] update --- .../mr/suggestion/JobSuggestionEvaluator.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java index e1a357a9ee..7e70f83094 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java @@ -73,13 +73,14 @@ public Result.EvaluatorResult evaluate(MapReduceAnalyzerEntity analyzerEntity) { return null; } - MapReduceJobSuggestionContext jobContext = new MapReduceJobSuggestionContext(analyzerEntity); - if (jobContext.getNumMaps() == 0) { - return null; - } - try { Result.EvaluatorResult result = new Result.EvaluatorResult(); + + MapReduceJobSuggestionContext jobContext = new MapReduceJobSuggestionContext(analyzerEntity); + if (jobContext.getNumMaps() == 0) { + return null; + } + for (Processor processor : loadProcessors(jobContext)) { Result.ProcessorResult processorResult = processor.process(analyzerEntity); if (processorResult != null) { From 726bf6aaccd2e6f080290bbd367bfe8f8d297154 Mon Sep 17 00:00:00 2001 From: "Zhao, Qingwen" Date: Tue, 18 Apr 2017 16:18:07 +0800 Subject: [PATCH 5/5] pdate --- .../eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java index 7e70f83094..4e2726b13c 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java @@ -63,7 +63,6 @@ public Result.EvaluatorResult evaluate(MapReduceAnalyzerEntity analyzerEntity) { return null; } - if (analyzerEntity.getTotalCounters() == null) { LOG.warn("Total counters of Job {} is null", analyzerEntity.getJobId()); return null;