From e86b35ba32b8def75c7ef4f7cfb6813b85bc45a9 Mon Sep 17 00:00:00 2001 From: chitin Date: Thu, 19 Jan 2017 13:05:05 +0800 Subject: [PATCH 1/3] [IMPROVEMENT] Using aggregation query in backend instead of calculating count in frontend. - Using siteId as query argument. - Using aggregation query in backend instead of calculating count in frontend. --- .../app/apps/hadoop_metric/ctrls/overview.js | 96 +++++++++---------- .../webapp/app/apps/hadoop_metric/index.js | 20 ++++ .../apps/hadoop_metric/partials/overview.html | 38 ++++++-- .../widgets/availabilityChart.js | 72 +++++++------- 4 files changed, 134 insertions(+), 92 deletions(-) diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/overview.js b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/overview.js index 8e9140fda4..406adf2dd1 100644 --- a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/overview.js +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/overview.js @@ -69,12 +69,12 @@ host: $scope.defaultHostname }; return METRIC.aggMetricsToEntities(METRIC.hbaseMetricsAggregation(jobCond, name, ["site"], "avg(value)", intervalMin, trendStartTime, trendEndTime), flag) - ._promise.then(function (list) { - var metricFlag = $.map(list, function (metrics) { - return metrics[0].flag; + ._promise.then(function (list) { + var metricFlag = $.map(list, function (metrics) { + return metrics[0].flag; + }); + return [metricFlag, list]; }); - return [metricFlag, list]; - }); }); } @@ -82,7 +82,7 @@ var series = []; $.each(metrics, function (i, metricMap) { - if(typeof metricMap !== 'undefined') { + if (typeof metricMap !== 'undefined') { series.push(METRIC.metricsToSeries(legendName[i], metricMap[0], option)); } }); @@ -93,12 +93,28 @@ }; } + function countHBaseRole(site, status, role, groups, filed, limit) { + var jobCond = { + site: site, + status: status, + role: role + }; + return METRIC.aggHBaseInstance(jobCond, groups, filed, limit); + } + + function sumAllRegions(site, role, groups, filed, limit) { + var jobCond = { + site: site, + role: role + }; + return METRIC.aggHBaseInstance(jobCond, groups, filed, limit); + } + // TODO: Optimize the chart count // TODO: ECharts dynamic refresh series bug: https://github.com/ecomfe/echarts/issues/4033 $scope.refresh = function () { - var hbaseservers = METRIC.hbasehostStatus({site: $scope.site}); var metricspromies = []; METRIC.getMetricObj().then(function (res) { var masterMetricList = res.master; @@ -107,7 +123,7 @@ } $q.all(metricspromies).then(function (resp) { var metricObj = {}; - for(var i=0; i < resp.length; i+=1) { + for (var i = 0; i < resp.length; i += 1) { metricObj[resp[i][0]] = resp[i][1]; } return metricObj; @@ -132,51 +148,31 @@ }); }); - METRIC.regionserverStatus($scope.hostname, $scope.site)._promise.then(function (res) { - $scope.regionstatus = res; + countHBaseRole($scope.site, "active", "hmaster", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.hmasteractivenum = data.value[0]; + }); }); - - hbaseservers._promise.then(function (res) { - var regionserverhealtynum = 0; - var regionservertotal = 0; - var hmasterstandbynum = 0; - var hmasteractivenum = 0; - var regionsnum = 0; - var regionArray = []; - $.each(res, function (i, server) { - var role = server.tags.role; - var status = server.status; - if (role === "regionserver") { - regionservertotal++; - if (status === "live") { - regionserverhealtynum++; - } - regionArray.push(METRIC.regionserverStatus(server.tags.hostname, $scope.site)._promise.then(function (res) { - return res[0].numRegions; - })); - } - else if (role === "hmaster") { - if (status === "active") { - hmasteractivenum++; - } else { - hmasterstandbynum++; - } - - } + countHBaseRole($scope.site, "standby", "hmaster", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.hmasterstandbynum = data.value[0] }); + }); - $q.all(regionArray).then(function (res) { - - for(var i=0; i
- + - + - + - - + @@ -67,7 +88,8 @@

{{metric.title}}

-
diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/widgets/availabilityChart.js b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/widgets/availabilityChart.js index f2606ad872..9e73c9b718 100644 --- a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/widgets/availabilityChart.js +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/widgets/availabilityChart.js @@ -54,38 +54,39 @@ // Customize chart color $scope.bgColor = COLOR_MAPPING[$scope.type]; + function countHBaseRole(site, status, role, groups, filed, limit) { + var jobCond = { + site: site, + status: status, + role: role + }; + return METRIC.aggHBaseInstance(jobCond, groups, filed, limit); + } // Ref: jpm widget if need keep refresh the widget function refresh() { $.each($scope.list, function (i, site) { - var hbaseservers = METRIC.hbasehostStatus({site: site.siteName}); - hbaseservers._promise.then(function (res) { - var hmasternum = 0; - var hmasteractivenum = 0; - var regionserverHealthynum = 0; - var regionservertotal = 0; - $.each(res, function (i, server) { - var role = server.tags.role; - var status = server.status; - if (role === "hmaster") { - hmasternum++; - if (status === "active") { - hmasteractivenum++; - } - } else if (role === "regionserver") { - regionservertotal++; - if (status === "live") { - regionserverHealthynum++; - } - } + + countHBaseRole(site.siteId, "active", "hmaster", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.hmasteractivenum = data.value[0]; + }); + }); + countHBaseRole(site.siteId, "standby", "hmaster", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.hmasterstandbynum = data.value[0] + }); + }); + countHBaseRole(site.siteId, "live", "regionserver", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.regionserverhealtynum = data.value[0]; + }); + }); + countHBaseRole(site.siteId, "dead", "regionserver", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.regionserverunhealtynum = data.value[0]; }); - $scope.hbaseinfo = { - hmasternum: hmasternum, - hmasteractivenum: hmasteractivenum, - regionserverHealthynum: regionserverHealthynum, - regionservertotal: regionservertotal - }; }); }); } @@ -101,18 +102,21 @@ '
' + '
' + '

{{type}}

' + - '
' + + '
' + '' + - '{{hbaseinfo.hmasternum}} Masters (' + - '{{hbaseinfo.hmasteractivenum}} Active / ' + - '{{hbaseinfo.hmasternum - hbaseinfo.hmasteractivenum}} Standby)' + + '{{hmasteractivenum+hmasterstandbynum}} Masters (' + + '{{hmasteractivenum}}0 Active / ' + + '{{hmasterstandbynum}}0 Standby)' + '' + '
' + - '
' + + '
' + + ' NO DATA' + + '
' + + '
' + '' + - '{{hbaseinfo.regionservertotal}} RegionServers (' + - '{{hbaseinfo.regionserverHealthynum}} Healthy / ' + - '{{hbaseinfo.regionservertotal - hbaseinfo.regionserverHealthynum}} Unhealthy)' + + '{{regionserverhealtynum+regionserverunhealtynum}} RegionServers (' + + '{{regionserverhealtynum}}0 Healthy / ' + + '{{regionserverunhealtynum}}0 Unhealthy)' + '' + '
' + '
' + From e01c8d978222746d23023cee0e2379d466e7d53a Mon Sep 17 00:00:00 2001 From: chitin Date: Thu, 19 Jan 2017 15:05:44 +0800 Subject: [PATCH 2/3] [MINOR] Format averageload to single digit --- .../main/webapp/app/apps/hadoop_metric/partials/overview.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/partials/overview.html b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/partials/overview.html index 7ba7f0b166..5e3d00caa0 100644 --- a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/partials/overview.html +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/partials/overview.html @@ -46,7 +46,7 @@

From a0a3e5e2c40aecce63055c0608adbd24553e4f9e Mon Sep 17 00:00:00 2001 From: chitin Date: Thu, 19 Jan 2017 17:39:13 +0800 Subject: [PATCH 3/3] [MINOR] Fix metric name in config.json --- .../webapp/app/apps/hadoop_metric/config.json | 18 +++++++++--------- .../hadoop_metric/ctrls/regionDetailCtrl.js | 9 ++++----- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/config.json b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/config.json index 185e7919c2..88b14f6e32 100644 --- a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/config.json +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/config.json @@ -42,15 +42,15 @@ "nonheap" : "hadoop.memory.nonheapmemoryusage.used", "heap" : "hadoop.memory.heapmemoryusage.used", "directmemory" : "hadoop.bufferpool.direct.memoryused", - "GC count" : "hadoop.hbase.jvm.gccount", - "GC TimeMillis" : "hadoop.hbase.jvm.gctimemillis", - "QueueSize" : "hadoop.hbase.ipc.ipc.queuesize", - "NumCallsInGeneralQueue" : "hadoop.hbase.ipc.ipc.numcallsingeneralqueue", - "NumActiveHandler" : "hadoop.hbase.ipc.ipc.numactivehandler", - "IPC Queue Time (99th" : "hadoop.hbase.ipc.ipc.queuecalltime_99th_percentile", - "IPC Process Time (99th" : "hadoop.hbase.ipc.ipc.processcalltime_99th_percentile", - "QueueCallTime_num_ops" : "hadoop.hbase.ipc.ipc.queuecalltime_num_ops", - "ProcessCallTime_num_ops" : "hadoop.hbase.ipc.ipc.processcalltime_num_ops", + "GCCount" : "hadoop.hbase.jvm.gccount", + "GCTimeMillis" : "hadoop.hbase.jvm.gctimemillis", + "QueueSize" : "hadoop.hbase.regionserver.ipc.queuesize", + "NumCallsInGeneralQueue" : "hadoop.hbase.regionserver.ipc.numcallsingeneralqueue", + "NumActiveHandler" : "hadoop.hbase.regionserver.ipc.numactivehandler", + "IPCQueueTime99th" : "hadoop.hbase.regionserver.ipc.queuecalltime_99th_percentile", + "IPCProcessTime99th" : "hadoop.hbase.regionserver.ipc.processcalltime_99th_percentile", + "QueueCallTime_num_ops" : "hadoop.hbase.regionserver.ipc.queuecalltime_num_ops", + "ProcessCallTime_num_ops" : "hadoop.hbase.regionserver.ipc.processcalltime_num_ops", "RegionCount" : "hadoop.hbase.regionserver.server.regioncount", "StoreCount" : "hadoop.hbase.regionserver.server.storecount", "MemStoreSize" : "hadoop.hbase.regionserver.server.memstoresize", diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/regionDetailCtrl.js b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/regionDetailCtrl.js index 1f8c40dbfd..00f2c9957c 100644 --- a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/regionDetailCtrl.js +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/regionDetailCtrl.js @@ -72,7 +72,6 @@ var endTime = Time.endTime(); var metricspromies = []; METRIC.getMetricObj().then(function (res) { - console.log(res.regionserver); var masterMetricList = res.regionserver; for (var metricKey in masterMetricList) { metricspromies.push(generateHbaseMetric(masterMetricList[metricKey], startTime, endTime, metricKey)); @@ -87,13 +86,13 @@ $scope.metricList = []; $scope.metricList.push(mergeSeries("Memory Usage", [seriesObj["nonheap"], seriesObj["heap"]], ["nonheap", "heap"], sizeoption)); $scope.metricList.push(mergeSeries("Direct Memory Usage", [seriesObj["directmemory"]], ["directmemory"], sizeoption)); - $scope.metricList.push(mergeSeries("GC count", [seriesObj["GC count"]], ["GC count"], {})); - $scope.metricList.push(mergeSeries("GC TimeMillis", [seriesObj["GC TimeMillis"]], ["GC TimeMillis"], gctimeoption)); + $scope.metricList.push(mergeSeries("GC count", [seriesObj["GCCount"]], ["GC count"], {})); + $scope.metricList.push(mergeSeries("GC TimeMillis", [seriesObj["GCTimeMillis"]], ["GC TimeMillis"], gctimeoption)); $scope.metricList.push(mergeSeries("QueueSize", [seriesObj["QueueSize"]], ["QueueSize"], {})); $scope.metricList.push(mergeSeries("NumCallsInGeneralQueue", [seriesObj["NumCallsInGeneralQueue"]], ["NumCallsInGeneralQueue"], {})); $scope.metricList.push(mergeSeries("NumActiveHandler", [seriesObj["NumActiveHandler"]], ["NumActiveHandler"], {})); - $scope.metricList.push(mergeSeries("IPC Queue Time (99th)", [seriesObj["IPC Queue Time (99th)"]], ["IPC Queue Time (99th)"], {})); - $scope.metricList.push(mergeSeries("IPC Process Time (99th)", [seriesObj["IPC Process Time (99th)"]], ["IPC Process Time (99th)"], {})); + $scope.metricList.push(mergeSeries("IPC Queue Time (99th)", [seriesObj["IPCQueueTime99th"]], ["IPC Queue Time (99th)"], {})); + $scope.metricList.push(mergeSeries("IPC Process Time (99th)", [seriesObj["IPCProcessTime99th"]], ["IPC Process Time (99th)"], {})); $scope.metricList.push(mergeSeries("QueueCallTime_num_ops", [seriesObj["QueueCallTime_num_ops"]], ["QueueCallTime_num_ops"], {})); $scope.metricList.push(mergeSeries("ProcessCallTime_num_ops", [seriesObj["ProcessCallTime_num_ops"]], ["ProcessCallTime_num_ops"], {})); $scope.metricList.push(mergeSeries("RegionCount", [seriesObj["RegionCount"]], ["RegionCount"], {}));
{{hmasteractivenum}} + {{hmasteractivenum}} + + Active HBase Master{{regionsnum}} + {{regionsnum}} + + Regions
{{hmasterstandbynum}} + + {{hmasterstandbynum}} + 0 + Backup HBase Master {{hmasteraverageload}} - + Average Load
{{regionservertotal}}RegionServers: {{regionserverhealtynum}} Good Health / {{regionservertotal-regionserverhealtynum}} + + + {{regionserverhealtynum+regionserverunhealtynum}} + 0 + RegionServers: + + {{regionserverhealtynum}} + 0 + Good Health / + + {{regionserverunhealtynum}} + 0 Bad Health Backup HBase Master - {{hmasteraverageload}} + {{common.number.format(hmasteraverageload, 0)}} Average Load