From 84ef5b70ea3b5de307738c546f433ad8793ba5b4 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Tue, 21 Feb 2017 14:51:14 +0800 Subject: [PATCH 1/3] add Job error category and error message mapping --- .../mr/MRMetricsAggregateContainer.java | 3 +- .../UnExpectedLongDurationJobProcessor.java | 2 +- .../mr/historyentity/JPAEntityRepository.java | 1 + .../JobConfigurationAPIEntity.java | 2 +- .../jpm/mr/historyentity/JobCountEntity.java | 2 +- .../JobErrorCategoryMappingAPIEntity.java | 62 +++++++++++++++++++ .../mr/historyentity/JobEventAPIEntity.java | 2 +- .../historyentity/JobExecutionAPIEntity.java | 4 +- .../JobProcessTimeStampEntity.java | 2 +- .../historyentity/JobSuggestionAPIEntity.java | 4 +- .../TaskAttemptCounterAPIEntity.java | 2 +- .../TaskAttemptErrorCategoryEntity.java | 2 +- .../TaskAttemptExecutionAPIEntity.java | 5 +- .../historyentity/TaskExecutionAPIEntity.java | 2 +- .../TaskFailureCountAPIEntity.java | 2 +- .../runningentity/JobExecutionAPIEntity.java | 2 +- .../TaskAttemptExecutionAPIEntity.java | 2 +- .../runningentity/TaskExecutionAPIEntity.java | 2 +- .../MRHistoryJobApplicationHealthCheck.java | 2 +- .../mr/history/MRHistoryJobDailyReporter.java | 8 +-- .../mr/history/parser/JHFEventReaderBase.java | 29 +++++++++ .../service/jpm/MRJobExecutionResource.java | 16 ++--- .../service/jpm/MRTaskExecutionResource.java | 12 ++-- .../service/jpm/RunningQueueResource.java | 8 +-- .../org/apache/eagle/jpm/util/Constants.java | 29 ++++----- 25 files changed, 149 insertions(+), 58 deletions(-) create mode 100644 eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java index dd1980b81c..be139fb252 100644 --- a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java +++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java @@ -21,7 +21,6 @@ import org.apache.eagle.jpm.aggregation.AggregationConfig; import org.apache.eagle.jpm.aggregation.common.MetricAggregator; import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer; -import org.apache.eagle.jpm.mr.historyentity.JobProcessTimeStampEntity; import org.apache.eagle.jpm.util.Constants; import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity; import org.apache.eagle.service.client.IEagleServiceClient; @@ -62,7 +61,7 @@ public long fetchLatestJobProcessTime() { eagleServiceConfig.password); String query = String.format("%s[@site=\"%s\"]<@site>{max(currentTimeStamp)}", - Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME, + Constants.MR_JOB_PROCESS_TIME_STAMP_NAME, appConfig.getStormConfig().site); GenericServiceAPIResponseEntity response = client diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java index 8f655bae67..f7748f84a4 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java @@ -90,7 +90,7 @@ private long getAvgDuration(AnalyzerEntity mrJobAnalysisEntity, Map{avg(durationTime)}", - org.apache.eagle.jpm.util.Constants.JPA_JOB_EXECUTION_SERVICE_NAME, + org.apache.eagle.jpm.util.Constants.MR_JOB_EXECUTION_SERVICE_NAME, mrJobAnalysisEntity.getSiteId(), URLEncoder.encode(mrJobAnalysisEntity.getJobDefId())); diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java index 8c65adfb14..54383bde1e 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java @@ -30,6 +30,7 @@ public JPAEntityRepository() { entitySet.add(JobConfigurationAPIEntity.class); entitySet.add(JobEventAPIEntity.class); entitySet.add(JobExecutionAPIEntity.class); + entitySet.add(JobErrorCategoryMappingAPIEntity.class); entitySet.add(TaskAttemptExecutionAPIEntity.class); entitySet.add(TaskExecutionAPIEntity.class); diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java index 6298fe8af2..36875e387e 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java @@ -26,7 +26,7 @@ @Table("eaglejpa") @ColumnFamily("f") @Prefix("jconf") -@Service(Constants.JPA_JOB_CONFIG_SERVICE_NAME) +@Service(Constants.MR_JOB_CONFIG_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java index 5b4163a51d..0fde454b48 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java @@ -27,7 +27,7 @@ @Table("eaglejcount") @ColumnFamily("f") @Prefix("jcount") -@Service(Constants.JPA_JOB_COUNT_SERVICE_NAME) +@Service(Constants.MR_JOB_COUNT_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class JobCountEntity extends TaggedLogAPIEntity { diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java new file mode 100644 index 0000000000..06977fa48f --- /dev/null +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.eagle.jpm.mr.historyentity; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.eagle.jpm.util.Constants; +import org.apache.eagle.log.entity.meta.*; + +import java.util.List; + +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@Table("eaglejpa") +@ColumnFamily("f") +@Prefix("jecm") +@Service(Constants.MR_JOB_ERROR_MAPPING_SERVICE_NAME) +@TimeSeries(true) +@Partition({"site"}) +@Indexes({ + @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false), + @Index(name = "Index_1_jobDefId", columns = { "jobDefId" }, unique = false), + @Index(name = "Index_1_jobIdAndErrorCategory", columns = { "jobId", "errorCategory" }, unique = true) +}) +public class JobErrorCategoryMappingAPIEntity extends JobBaseAPIEntity { + @Column("a") + private String error; + @Column("b") + private List taskAttempts; + + public String getError() { + return error; + } + + public void setError(String error) { + this.error = error; + valueChanged("error"); + } + + public List getTaskAttempts() { + return taskAttempts; + } + + public void setTaskAttempts(List taskAttempts) { + this.taskAttempts = taskAttempts; + valueChanged("taskAttempts"); + } +} diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java index aef3750821..56f541b24d 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java @@ -26,7 +26,7 @@ @Table("eaglejpa") @ColumnFamily("f") @Prefix("jevent") -@Service(Constants.JPA_JOB_EVENT_SERVICE_NAME) +@Service(Constants.MR_JOB_EVENT_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class JobEventAPIEntity extends JobBaseAPIEntity { diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java index df7b271950..746843dbcf 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java @@ -23,13 +23,11 @@ import org.apache.eagle.log.entity.meta.*; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import java.util.Map; - @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @Table("eaglejpa") @ColumnFamily("f") @Prefix("jexec") -@Service(Constants.JPA_JOB_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_JOB_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java index ee632f48a9..1490390687 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java @@ -27,7 +27,7 @@ @Table("eaglejpa_process") @ColumnFamily("f") @Prefix("process") -@Service(Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME) +@Service(Constants.MR_JOB_PROCESS_TIME_STAMP_NAME) @TimeSeries(true) @Partition({"site"}) public class JobProcessTimeStampEntity extends TaggedLogAPIEntity { diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java index 3863a5d323..e433672c80 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java @@ -23,13 +23,13 @@ import java.util.List; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import static org.apache.eagle.jpm.util.Constants.JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME; +import static org.apache.eagle.jpm.util.Constants.MR_JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME; @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @Table("eaglejpa") @ColumnFamily("f") @Prefix("jsuggestion") -@Service(JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME) +@Service(MR_JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java index 04fe7d7f60..33a5102552 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java @@ -26,7 +26,7 @@ @Table("eaglejpa_anomaly") @ColumnFamily("f") @Prefix("tacount") -@Service(Constants.JPA_TASK_ATTEMPT_COUNTER_SERVICE_NAME) +@Service(Constants.MR_TASK_ATTEMPT_COUNTER_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class TaskAttemptCounterAPIEntity extends JobBaseAPIEntity { diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java index 37f6451d0c..1228f4e1c3 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java @@ -28,7 +28,7 @@ @Table("eaglejpa_task") @ColumnFamily("f") @Prefix("taexece") -@Service(Constants.JPA_TASK_ATTEMPT_ERROR_SERVICE_NAME) +@Service(Constants.MR_TASK_ATTEMPT_ERROR_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java index 46fcf5e9b9..b4bb916238 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java @@ -27,11 +27,12 @@ @Table("eaglejpa_task") @ColumnFamily("f") @Prefix("taexec") -@Service(Constants.JPA_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ - @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false) + @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false), + @Index(name = "Index_1_taskAttemptId", columns = { "taskAttemptId" }, unique = true) }) public class TaskAttemptExecutionAPIEntity extends JobBaseAPIEntity { @Column("a") diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java index 9600fe2a3b..9f855556f5 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java @@ -27,7 +27,7 @@ @Table("eaglejpa_task") @ColumnFamily("f") @Prefix("texec") -@Service(Constants.JPA_TASK_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_TASK_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java index 3e5e997e87..f82bcaf027 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java @@ -26,7 +26,7 @@ @Table("eaglejpa_anomaly") @ColumnFamily("f") @Prefix("taskfailurecount") -@Service(Constants.JPA_TASK_FAILURE_COUNT_SERVICE_NAME) +@Service(Constants.MR_TASK_FAILURE_COUNT_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class TaskFailureCountAPIEntity extends JobBaseAPIEntity { diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java index 082612b187..3327a18e0b 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java @@ -29,7 +29,7 @@ @Table("eagleMRRunningJobs") @ColumnFamily("f") @Prefix("jobs") -@Service(Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java index 0d1703acfa..4fa7775270 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java @@ -27,7 +27,7 @@ @Table("eagleMRRunningTasks") @ColumnFamily("f") @Prefix("tasks_exec_attempt") -@Service(Constants.JPA_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java index 61c92c1e7e..e9f0fcfeb6 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java @@ -28,7 +28,7 @@ @Table("eagleMRRunningTasks") @ColumnFamily("f") @Prefix("tasks_exec") -@Service(Constants.JPA_RUNNING_TASK_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_RUNNING_TASK_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java index 4016e6d932..90544e7915 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java @@ -61,7 +61,7 @@ public Result check() { } String query = String.format("%s[@site=\"%s\"]<@site>{max(currentTimeStamp)}", - Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME, + Constants.MR_JOB_PROCESS_TIME_STAMP_NAME, mrHistoryJobConfig.getJobHistoryEndpointConfig().site); GenericServiceAPIResponseEntity response = client diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java index 9aef64005a..541d35266a 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java @@ -208,16 +208,16 @@ private Map buildAlertData(String site, long startTime, long end private Map buildJobSummery(String site, long startTime, long endTime) { Map data = new HashMap<>(); - String query = String.format(STATUS_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); + String query = String.format(STATUS_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, endTime); Map jobSummery = queryGroupByMetrics(query, startTime, endTime, Integer.MAX_VALUE); if (jobSummery == null || jobSummery.isEmpty()) { LOG.warn("Result set is empty for query={}", query); return data; } Long totalJobs = jobSummery.values().stream().reduce((a, b) -> a + b).get(); - String finishedJobQuery = String.format(FINISHED_JOB_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); - String failedJobQuery = String.format(FAILED_JOBS_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); - String succeededJobQuery = String.format(SUCCEEDED_JOB_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, jobOvertimeLimit * DateTimeUtil.ONEHOUR, endTime); + String finishedJobQuery = String.format(FINISHED_JOB_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, endTime); + String failedJobQuery = String.format(FAILED_JOBS_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, endTime); + String succeededJobQuery = String.format(SUCCEEDED_JOB_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, jobOvertimeLimit * DateTimeUtil.ONEHOUR, endTime); data.put(SUMMARY_INFO_KEY, processResult(jobSummery, totalJobs)); data.put(FAILED_JOB_USERS_KEY, buildJobSummery(failedJobQuery, startTime, endTime, jobSummery.get(Constants.JobState.FAILED.toString()))); data.put(SUCCEEDED_JOB_USERS_KEY, buildJobSummery(succeededJobQuery, startTime, endTime, jobSummery.get(Constants.JobState.SUCCEEDED.toString()))); diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index d58eadc93b..ff935d2ebf 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -57,6 +57,9 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl // taskAttemptID to task attempt startTime protected Map taskAttemptStartTime; + //errorCategory, + protected Map> errorCategoryTaskMapping; + // taskID to host mapping, for task it's the host where the last attempt runs on protected Map taskRunningHosts; // hostname to rack mapping @@ -117,6 +120,7 @@ public JHFEventReaderBase(Map baseTags, Configuration configurat taskStartTime = new HashMap<>(); taskAttemptStartTime = new HashMap<>(); + errorCategoryTaskMapping = new HashMap<>(); this.configuration = configuration; @@ -305,6 +309,24 @@ protected void handleJob(EventType eventType, Map values, Object t formatDiagnostics(values.get(Keys.DIAGNOSTICS)); entityCreated(jobExecutionEntity); + for (String errorCategory : errorCategoryTaskMapping.keySet()) { + JobErrorCategoryMappingAPIEntity jobErrorCategoryMappingAPIEntity = new JobErrorCategoryMappingAPIEntity(); + jobErrorCategoryMappingAPIEntity.setTags(new HashMap<>(jobExecutionEntity.getTags())); + jobErrorCategoryMappingAPIEntity.getTags().put(MRJobTagName.ERROR_CATEGORY.toString(), errorCategory); + + for (String taskId : errorCategoryTaskMapping.get(errorCategory).keySet()) { + jobErrorCategoryMappingAPIEntity.getTaskAttempts().add(errorCategoryTaskMapping.get(errorCategory).get(taskId)); + } + + if (jobErrorCategoryMappingAPIEntity.getTaskAttempts().size() > 0) { + jobErrorCategoryMappingAPIEntity.setError( + attempt2ErrorMsg.get( + jobErrorCategoryMappingAPIEntity.getTaskAttempts().get(0) + ).getRight() + ); + } + entityCreated(jobErrorCategoryMappingAPIEntity); + } if (configuration != null && totalCounters != null) { JobCounters parsedTotalCounters = parseCounters(totalCounters); JobCounters parsedMapCounters = parseCounters(mapCounters); @@ -491,6 +513,13 @@ protected void handleTask(RecordTypes recType, EventType eventType, final Map()); + } + + errorCategoryTaskMapping.get(errorCategory).put(taskID, taskAttemptID); } taskAttemptStartTime.remove(taskAttemptID); } else { diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java index 28e6bb30ca..5804e0ca12 100644 --- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java +++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java @@ -69,7 +69,7 @@ public GenericServiceAPIResponseEntity listJobs(@QueryParam("query") String quer StopWatch stopWatch = new StopWatch(); stopWatch.start(); - String jobQuery = String.format(query, Constants.JPA_JOB_EXECUTION_SERVICE_NAME); + String jobQuery = String.format(query, Constants.MR_JOB_EXECUTION_SERVICE_NAME); GenericServiceAPIResponseEntity res = resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose); if (res.isSuccess() && res.getObj() != null) { @@ -80,7 +80,7 @@ public GenericServiceAPIResponseEntity listJobs(@QueryParam("query") String quer jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString())); } } - jobQuery = String.format(query, Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME); + jobQuery = String.format(query, Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME); GenericServiceAPIResponseEntity runningRes = resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose); if (runningRes.isSuccess() && runningRes.getObj() != null) { @@ -150,7 +150,7 @@ public GenericServiceAPIResponseEntity searchJobsById(@QueryParam("jobId") Strin StopWatch stopWatch = new StopWatch(); stopWatch.start(); String queryFormat = "%s[%s]{*}"; - String queryString = String.format(queryFormat, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, condition); + String queryString = String.format(queryFormat, Constants.MR_JOB_EXECUTION_SERVICE_NAME, condition); GenericServiceAPIResponseEntity res = ResourceUtils.getQueryResult(queryString, null, null); if (res.isSuccess() && res.getObj() != null) { for (TaggedLogAPIEntity o : res.getObj()) { @@ -158,7 +158,7 @@ public GenericServiceAPIResponseEntity searchJobsById(@QueryParam("jobId") Strin jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString())); } } - queryString = String.format(queryFormat, Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME, condition); + queryString = String.format(queryFormat, Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME, condition); res = ResourceUtils.getQueryResult(queryString, null, null); if (res.isSuccess() && res.getObj() != null) { for (TaggedLogAPIEntity o : res.getObj()) { @@ -214,13 +214,13 @@ public JobCountResponse getRunningJobCount(@QueryParam("site") String site, response.errMessage = e.getMessage(); return response; } - String query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site); + String query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.MR_JOB_EXECUTION_SERVICE_NAME, site); GenericServiceAPIResponseEntity historyRes = ResourceUtils.getQueryResult(query, searchStartTime, searchEndTime); if (!historyRes.isSuccess() || historyRes.getObj() == null) { response.errMessage = String.format("Catch an exception during fetch history jobs: %s with query=%s", historyRes.getException(), query); return response; } - query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME, site); + query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME, site); GenericServiceAPIResponseEntity runningRes = ResourceUtils.getQueryResult(query, searchStartTime, searchEndTime); if (!runningRes.isSuccess() || runningRes.getObj() == null) { response.errMessage = String.format("Catch an exception during fetch running jobs: %s with query=%s", runningRes.getException(), query); @@ -289,7 +289,7 @@ public Object getJobMetrics(String site, String timePoint, String metricName, lo return response; } String query = String.format("%s[@site=\"%s\" AND @startTime<=\"%s\" AND @endTime>=\"%s\"]{@startTime,@endTime}", - Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, timePointsInMills, timePointsInMills); + Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, timePointsInMills, timePointsInMills); GenericServiceAPIResponseEntity historyRes = ResourceUtils.getQueryResult(query, searchStartTime, searchEndTime); if (!historyRes.isSuccess() || historyRes.getObj() == null) { return historyRes; @@ -333,7 +333,7 @@ public JobCountResponse getJobCountGroupByDuration(@QueryParam("site") String si response.errMessage = "IllegalArgument: site, startTime, endTime, or timeDistInSecs is null"; return response; } - String query = String.format("%s[@site=\"%s\"]{@durationTime,@jobType}", Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site); + String query = String.format("%s[@site=\"%s\"]{@durationTime,@jobType}", Constants.MR_JOB_EXECUTION_SERVICE_NAME, site); GenericServiceAPIResponseEntity historyRes = ResourceUtils.getQueryResult(query, startTime, endTime); if (!historyRes.isSuccess() || historyRes.getObj() == null) { response.errMessage = String.format("Catch an exception: %s with query=%s", historyRes.getException(), query); diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java index 1125387852..d81e7d922a 100644 --- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java +++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java @@ -60,7 +60,7 @@ public MRJobTaskCountResponse.TaskCountPerJobResponse getTaskCountsGroupByDurati List finishedTaskCount = new ArrayList<>(); List times = ResourceUtils.parseDistributionList(timeDistInSecs); - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, jobId); + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, jobId); GenericServiceAPIResponseEntity historyRes = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (historyRes.isSuccess() && historyRes.getObj() != null && historyRes.getObj().size() > 0) { @@ -72,7 +72,7 @@ public MRJobTaskCountResponse.TaskCountPerJobResponse getTaskCountsGroupByDurati counter.entities.add(o); } } else { - query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_RUNNING_TASK_EXECUTION_SERVICE_NAME, site, jobId); + query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_RUNNING_TASK_EXECUTION_SERVICE_NAME, site, jobId); GenericServiceAPIResponseEntity runningRes = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (runningRes.isSuccess() && runningRes.getObj() != null) { @@ -105,7 +105,7 @@ private MRTaskExecutionResponse.TaskGroupResponse getTaskGroups(@QueryParam("sit @QueryParam("shortJob_id") String shortDurationJobId, @QueryParam("longJob_id") String longDurationJobId) { MRTaskExecutionResponse.TaskGroupResponse result = new MRTaskExecutionResponse.TaskGroupResponse(); - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, shortDurationJobId); + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, shortDurationJobId); GenericServiceAPIResponseEntity smallResponse = ResourceUtils.getQueryResult(query, null, null); if (!smallResponse.isSuccess() || smallResponse.getObj() == null) { result.errMessage = smallResponse.getException(); @@ -117,7 +117,7 @@ private MRTaskExecutionResponse.TaskGroupResponse getTaskGroups(@QueryParam("sit longestDuration = entity.getDuration(); } } - query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, longDurationJobId); + query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, longDurationJobId); GenericServiceAPIResponseEntity largeResponse = ResourceUtils.getQueryResult(query, null, null); if (!largeResponse.isSuccess() || largeResponse.getObj() == null) { result.errMessage = largeResponse.getException(); @@ -193,7 +193,7 @@ public MRJobTaskCountResponse.HistoryTaskCountResponse getTaskCountInMinute(@Que return result; } - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{@startTime,@endTime,@taskType}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, jobId); + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{@startTime,@endTime,@taskType}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, jobId); GenericServiceAPIResponseEntity response = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (!response.isSuccess() || response.getObj() == null) { result.errMessage = response.getException(); @@ -221,7 +221,7 @@ public MRTaskExecutionResponse.TaskDistributionResponse getTaskDistributionByCou @PathParam("counterName") String counterName, @QueryParam("distRange") String distRange) { MRTaskExecutionResponse.TaskDistributionResponse result = new MRTaskExecutionResponse.TaskDistributionResponse(); - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\" AND @taskType=\"%s\"]{@jobCounters}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\" AND @taskType=\"%s\"]{@jobCounters}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, jobId, Constants.TaskType.MAP.toString()); GenericServiceAPIResponseEntity response = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (!response.isSuccess() || response.getObj() == null) { diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java index 2632423d1c..2b0205204c 100644 --- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java +++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java @@ -31,8 +31,8 @@ import java.util.*; import static org.apache.eagle.hadoop.queue.common.HadoopClusterConstants.QUEUE_MAPPING_SERVICE_NAME; -import static org.apache.eagle.jpm.util.Constants.JPA_JOB_EXECUTION_SERVICE_NAME; -import static org.apache.eagle.jpm.util.Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME; +import static org.apache.eagle.jpm.util.Constants.MR_JOB_EXECUTION_SERVICE_NAME; +import static org.apache.eagle.jpm.util.Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME; import static org.apache.eagle.jpm.util.MRJobTagName.JOB_ID; import static org.apache.eagle.jpm.util.MRJobTagName.JOB_QUEUE; import static org.apache.eagle.jpm.util.MRJobTagName.USER; @@ -87,7 +87,7 @@ public RunningQueueResponse getTopByQueue(@QueryParam("site") String site, private List getRunningJobs(String site, long currentTime, String startTime, String endTime) throws Exception { GenericEntityServiceResource resource = new GenericEntityServiceResource(); String query = String.format("%s[@site=\"%s\" and @startTime<=%s and (@internalState=\"RUNNING\" or @endTime>%s)]{@jobId, @user, @queue, @allocatedMB}", - JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); + MR_RUNNING_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); GenericServiceAPIResponseEntity runningJobResponse = resource.search(query, startTime, endTime, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false); if (!runningJobResponse.isSuccess() || runningJobResponse.getObj() == null) { @@ -102,7 +102,7 @@ private List getJob String startTime, String endTime) throws Exception { GenericEntityServiceResource resource = new GenericEntityServiceResource(); - String query = String.format("%s[@site=\"%s\" and @startTime<=%s and @endTime>%s]{@jobId}", JPA_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); + String query = String.format("%s[@site=\"%s\" and @startTime<=%s and @endTime>%s]{@jobId}", MR_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); GenericServiceAPIResponseEntity response = resource.search(query, startTime, endTime, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false); diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java index 4ee58a1148..b87c41d91b 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java @@ -104,20 +104,21 @@ public static enum SuggestionType { public static final String TASK_FINISHED = "FINISHED"; //MR - public static final String JPA_JOB_CONFIG_SERVICE_NAME = "JobConfigService"; - public static final String JPA_JOB_EVENT_SERVICE_NAME = "JobEventService"; - public static final String JPA_JOB_EXECUTION_SERVICE_NAME = "JobExecutionService"; - public static final String JPA_JOB_COUNT_SERVICE_NAME = "JobCountService"; - public static final String JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME = "RunningJobExecutionService"; - public static final String JPA_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "TaskAttemptExecutionService"; - public static final String JPA_TASK_ATTEMPT_ERROR_SERVICE_NAME = "TaskAttemptErrorCategoryService"; - public static final String JPA_TASK_FAILURE_COUNT_SERVICE_NAME = "TaskFailureCountService"; - public static final String JPA_TASK_ATTEMPT_COUNTER_SERVICE_NAME = "TaskAttemptCounterService"; - public static final String JPA_TASK_EXECUTION_SERVICE_NAME = "TaskExecutionService"; - public static final String JPA_RUNNING_TASK_EXECUTION_SERVICE_NAME = "RunningTaskExecutionService"; - public static final String JPA_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "RunningTaskAttemptExecutionService"; - public static final String JPA_JOB_PROCESS_TIME_STAMP_NAME = "JobProcessTimeStampService"; - public static final String JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME = "JobOptimizerSuggestionService"; + public static final String MR_JOB_CONFIG_SERVICE_NAME = "JobConfigService"; + public static final String MR_JOB_EVENT_SERVICE_NAME = "JobEventService"; + public static final String MR_JOB_EXECUTION_SERVICE_NAME = "JobExecutionService"; + public static final String MR_JOB_ERROR_MAPPING_SERVICE_NAME = "JobErrorMappingService"; + public static final String MR_JOB_COUNT_SERVICE_NAME = "JobCountService"; + public static final String MR_RUNNING_JOB_EXECUTION_SERVICE_NAME = "RunningJobExecutionService"; + public static final String MR_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "TaskAttemptExecutionService"; + public static final String MR_TASK_ATTEMPT_ERROR_SERVICE_NAME = "TaskAttemptErrorCategoryService"; + public static final String MR_TASK_FAILURE_COUNT_SERVICE_NAME = "TaskFailureCountService"; + public static final String MR_TASK_ATTEMPT_COUNTER_SERVICE_NAME = "TaskAttemptCounterService"; + public static final String MR_TASK_EXECUTION_SERVICE_NAME = "TaskExecutionService"; + public static final String MR_RUNNING_TASK_EXECUTION_SERVICE_NAME = "RunningTaskExecutionService"; + public static final String MR_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "RunningTaskAttemptExecutionService"; + public static final String MR_JOB_PROCESS_TIME_STAMP_NAME = "JobProcessTimeStampService"; + public static final String MR_JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME = "JobOptimizerSuggestionService"; public static final String JOB_TASK_TYPE_TAG = "taskType"; From f9afafe7810d49e5338a9f8de76315c801f6fe24 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Tue, 21 Feb 2017 15:08:40 +0800 Subject: [PATCH 2/3] add Job error category and error message mapping --- .../apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index ff935d2ebf..afcd7be656 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -313,6 +313,7 @@ protected void handleJob(EventType eventType, Map values, Object t JobErrorCategoryMappingAPIEntity jobErrorCategoryMappingAPIEntity = new JobErrorCategoryMappingAPIEntity(); jobErrorCategoryMappingAPIEntity.setTags(new HashMap<>(jobExecutionEntity.getTags())); jobErrorCategoryMappingAPIEntity.getTags().put(MRJobTagName.ERROR_CATEGORY.toString(), errorCategory); + jobErrorCategoryMappingAPIEntity.setTimestamp(jobExecutionEntity.getTimestamp()); for (String taskId : errorCategoryTaskMapping.get(errorCategory).keySet()) { jobErrorCategoryMappingAPIEntity.getTaskAttempts().add(errorCategoryTaskMapping.get(errorCategory).get(taskId)); From fbee685dc1394629f71a4bc785864a7d5b127332 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Tue, 21 Feb 2017 15:34:09 +0800 Subject: [PATCH 3/3] add Job error category and error message mapping --- .../jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java index 06977fa48f..91c2c68b68 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java @@ -35,7 +35,7 @@ @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false), @Index(name = "Index_1_jobDefId", columns = { "jobDefId" }, unique = false), @Index(name = "Index_1_jobIdAndErrorCategory", columns = { "jobId", "errorCategory" }, unique = true) -}) + }) public class JobErrorCategoryMappingAPIEntity extends JobBaseAPIEntity { @Column("a") private String error;