From 37050b01b14acd5713ab4aef024eabe9781d116d Mon Sep 17 00:00:00 2001 From: r7raul1984 Date: Wed, 5 Apr 2017 02:42:44 +0000 Subject: [PATCH 1/5] [EAGLE-986] HBaseStorage unable to deserialize Deserialize from java object bytes - recover allocatedMB type from int to long https://issues.apache.org/jira/browse/EAGLE-986 --- .../jpm/util/resourcefetch/model/AppInfo.java | 6 +-- .../jpm/util/TestAppInfoDeserialize.java | 46 ++++++++++++++++++ .../src/test/resources/appinfobyteint | Bin 0 -> 989 bytes .../src/test/resources/appinfobytelong | Bin 0 -> 989 bytes 4 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 eagle-jpm/eagle-jpm-util/src/test/java/org/apache/eagle/jpm/util/TestAppInfoDeserialize.java create mode 100644 eagle-jpm/eagle-jpm-util/src/test/resources/appinfobyteint create mode 100644 eagle-jpm/eagle-jpm-util/src/test/resources/appinfobytelong diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/model/AppInfo.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/model/AppInfo.java index 5b0269df0d..a410ccf74c 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/model/AppInfo.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourcefetch/model/AppInfo.java @@ -43,7 +43,7 @@ public class AppInfo implements Serializable { private long elapsedTime; private String amContainerLogs; private String amHostHttpAddress; - private int allocatedMB; + private long allocatedMB; private int allocatedVCores; private int runningContainers; // for HDP 2.7 @@ -186,11 +186,11 @@ public void setAmHostHttpAddress(String amHostHttpAddress) { this.amHostHttpAddress = amHostHttpAddress; } - public int getAllocatedMB() { + public long getAllocatedMB() { return allocatedMB; } - public void setAllocatedMB(int allocatedMB) { + public void setAllocatedMB(long allocatedMB) { this.allocatedMB = allocatedMB; } diff --git a/eagle-jpm/eagle-jpm-util/src/test/java/org/apache/eagle/jpm/util/TestAppInfoDeserialize.java b/eagle-jpm/eagle-jpm-util/src/test/java/org/apache/eagle/jpm/util/TestAppInfoDeserialize.java new file mode 100644 index 0000000000..f833962f3a --- /dev/null +++ b/eagle-jpm/eagle-jpm-util/src/test/java/org/apache/eagle/jpm/util/TestAppInfoDeserialize.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.util; + +import org.apache.commons.io.IOUtils; +import org.apache.eagle.common.SerializableUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; + +public class TestAppInfoDeserialize { + @Test + public void testAppInfoDeserializeFromByteArray() throws IOException { + InputStream appinfostream = this.getClass().getResourceAsStream("/appinfobytelong"); + //serialize allocatedMB as long byte deserializeto long + byte[] bytes = IOUtils.toByteArray(appinfostream); + Assert.assertNotNull(bytes); + SerializableUtils.deserializeFromByteArray(bytes, "AppInfo Deserialize From ByteArray"); + } + + @Test(expected = IllegalArgumentException.class) + public void testAppInfoDeserializeFromByteArray1() throws IOException { + InputStream appinfostream = this.getClass().getResourceAsStream("/appinfobyteint"); + //serialize allocatedMB as int byte deserialize to long + byte[] bytes = IOUtils.toByteArray(appinfostream); + SerializableUtils.deserializeFromByteArray(bytes, "AppInfo Deserialize From ByteArray"); + } +} diff --git a/eagle-jpm/eagle-jpm-util/src/test/resources/appinfobyteint b/eagle-jpm/eagle-jpm-util/src/test/resources/appinfobyteint new file mode 100644 index 0000000000000000000000000000000000000000..6343fe6a0f9ff7325c64f669628b7f0867a64c53 GIT binary patch literal 989 zcma)5&2AGh5O%5-C@Dg%dgp)yr|b{;+boJiXae0TiBQ{62?<%&uCwdnwHCxV7G%NDFa>>XZ{@XmZJNdWH_> zOfZwJ%^|v{v`pDAy{06 z)-7$qFMvQ%xpr7}e^-#a0^3^@wAdP!K2*ki&?Np~DbAuQ&(@>)#>TYj&94D5(MGOS9m3r8m_$mx6)RSdo;1Z3L`1fw|OqIOvlY@CpelX;|4Rv6U%| zhJDZLbvi+_;dgxBZ~H+j@ceGmYpjT@=EfDEJC!VV&u7zVG<^-|(0-e$?>@dAiQUlJHX2J1FZ>n)F1p}$EA}Iyi2v~6fGpX~i-y<>L6%tm`u(DNRD^nN^ zdY;#9w}VFAZ~MOA@`GmJ`JIMWUlCc&^=m+PDp~GcOed3Q@*2>Q{WewKjY-*RfG%$- WRkwR Date: Thu, 6 Apr 2017 02:24:17 +0000 Subject: [PATCH 2/5] Only query history jobs when click compare button in jpm/detail/jobid page. --- .../src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js | 2 +- .../eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js index 121e80ecc6..3af537300b 100644 --- a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js +++ b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js @@ -44,7 +44,7 @@ // ================================================================== // = Fetch Job List = // ================================================================== - var jobList = $scope.jobList = JPM.findMRJobs($scope.site, $scope.jobDefId); + var jobList = $scope.jobList = JPM.findMRHistoryJobs($scope.site, $scope.jobDefId); jobList._promise.then(function () { if(jobList.length <= 1) { $.dialog({ diff --git a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js index 178b24f517..29785a14e6 100644 --- a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js +++ b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js @@ -81,6 +81,7 @@ JPM.QUERY_METRICS_AGG = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]<${groups}>{${field}}${order}${top}&metricName=${metric}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}'; JPM.QUERY_METRICS_INTERVAL = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]<${groups}>{${field}}${order}${top}&metricName=${metric}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}&intervalmin=${intervalMin}&timeSeries=true'; JPM.QUERY_MR_JOBS = '${baseURL}/rest/mrJobs/search'; + JPM.QUERY_MR_HISTORY_JOBS = '${baseURL}/rest/entities?query=JobExecutionService[${condition}]{*}'; JPM.QUERY_JOB_LIST = '${baseURL}/rest/mrJobs?query=%s[${condition}]{${fields}}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}'; JPM.QUERY_JOB_STATISTIC = '${baseURL}/rest/mrJobs/jobCountsByDuration?site=${site}&timeDistInSecs=${times}&startTime=${startTime}&endTime=${endTime}'; JPM.QUERY_TASK_STATISTIC = '${baseURL}/rest/mrTasks/taskCountsByDuration?jobId=${jobId}&site=${site}&timeDistInSecs=${times}&top=${top}'; @@ -404,6 +405,11 @@ return JPM.get(url); }; + JPM.findMRHistoryJobs = function (site, jobDefId) { + var findMRJobs_url = common.template(getQuery("MR_HISTORY_JOBS"), {condition: JPM.condition({site: site, jobDefId: jobDefId})}); + return wrapList(JPM.get(findMRJobs_url)); + }; + /** * Get job list by sam jobDefId * @param {string} site From eba2086178b4aa62db0b55b0236bd4b2f64d6745 Mon Sep 17 00:00:00 2001 From: r7raul1984 Date: Thu, 6 Apr 2017 05:46:48 +0000 Subject: [PATCH 3/5] use QUERY_LIST instead of new const QUERY_MR_HISTORY_JOBS --- .../src/main/webapp/app/apps/jpm/index.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js index 29785a14e6..3323aa43d8 100644 --- a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js +++ b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js @@ -81,7 +81,6 @@ JPM.QUERY_METRICS_AGG = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]<${groups}>{${field}}${order}${top}&metricName=${metric}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}'; JPM.QUERY_METRICS_INTERVAL = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]<${groups}>{${field}}${order}${top}&metricName=${metric}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}&intervalmin=${intervalMin}&timeSeries=true'; JPM.QUERY_MR_JOBS = '${baseURL}/rest/mrJobs/search'; - JPM.QUERY_MR_HISTORY_JOBS = '${baseURL}/rest/entities?query=JobExecutionService[${condition}]{*}'; JPM.QUERY_JOB_LIST = '${baseURL}/rest/mrJobs?query=%s[${condition}]{${fields}}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}'; JPM.QUERY_JOB_STATISTIC = '${baseURL}/rest/mrJobs/jobCountsByDuration?site=${site}&timeDistInSecs=${times}&startTime=${startTime}&endTime=${endTime}'; JPM.QUERY_TASK_STATISTIC = '${baseURL}/rest/mrTasks/taskCountsByDuration?jobId=${jobId}&site=${site}&timeDistInSecs=${times}&top=${top}'; @@ -406,9 +405,18 @@ }; JPM.findMRHistoryJobs = function (site, jobDefId) { - var findMRJobs_url = common.template(getQuery("MR_HISTORY_JOBS"), {condition: JPM.condition({site: site, jobDefId: jobDefId})}); + var endTime = new Time(); + var config = { + query: "JobExecutionService", + condition: JPM.condition({site: site, jobDefId:jobDefId}), + fields: "*", + startTime: Time.format(endTime.clone().subtract(365,'day')), + endTime: Time.format(endTime), + limit: 10000 + }; + var findMRJobs_url = common.template(getQuery("LIST"), config); return wrapList(JPM.get(findMRJobs_url)); - }; + } /** * Get job list by sam jobDefId From 08c0e2a77942a367134edf1c6c9b7585cabbfae5 Mon Sep 17 00:00:00 2001 From: r7raul1984 Date: Thu, 6 Apr 2017 05:58:09 +0000 Subject: [PATCH 4/5] use JPM.list instead of JPM.findMRHistoryJobs --- .../main/webapp/app/apps/jpm/ctrl/compareCtrl.js | 5 ++++- .../src/main/webapp/app/apps/jpm/index.js | 14 -------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js index 3af537300b..6541abbd61 100644 --- a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js +++ b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js @@ -44,7 +44,10 @@ // ================================================================== // = Fetch Job List = // ================================================================== - var jobList = $scope.jobList = JPM.findMRHistoryJobs($scope.site, $scope.jobDefId); + var endTime = new Time(); + var startTime = endTime.clone().subtract(365,'day'); + var condition = JPM.condition({site: $scope.site, jobDefId:$scope.jobDefId}); + var jobList = $scope.jobList = JPM.list("JobExecutionService", condition, startTime, endTime, [], 10000); jobList._promise.then(function () { if(jobList.length <= 1) { $.dialog({ diff --git a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js index 3323aa43d8..178b24f517 100644 --- a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js +++ b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/index.js @@ -404,20 +404,6 @@ return JPM.get(url); }; - JPM.findMRHistoryJobs = function (site, jobDefId) { - var endTime = new Time(); - var config = { - query: "JobExecutionService", - condition: JPM.condition({site: site, jobDefId:jobDefId}), - fields: "*", - startTime: Time.format(endTime.clone().subtract(365,'day')), - endTime: Time.format(endTime), - limit: 10000 - }; - var findMRJobs_url = common.template(getQuery("LIST"), config); - return wrapList(JPM.get(findMRJobs_url)); - } - /** * Get job list by sam jobDefId * @param {string} site From 72cb2e65831c197ffc41310866c7844e8c3b09cc Mon Sep 17 00:00:00 2001 From: r7raul1984 Date: Thu, 6 Apr 2017 06:06:47 +0000 Subject: [PATCH 5/5] change time range from 1 year to 30 days --- .../src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js index 6541abbd61..85168e30d5 100644 --- a/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js +++ b/eagle-jpm/eagle-jpm-web/src/main/webapp/app/apps/jpm/ctrl/compareCtrl.js @@ -45,7 +45,7 @@ // = Fetch Job List = // ================================================================== var endTime = new Time(); - var startTime = endTime.clone().subtract(365,'day'); + var startTime = endTime.clone().subtract(30, 'day'); var condition = JPM.condition({site: $scope.site, jobDefId:$scope.jobDefId}); var jobList = $scope.jobList = JPM.list("JobExecutionService", condition, startTime, endTime, [], 10000); jobList._promise.then(function () {