From 9c71b0ea05d3dcec51024873667865c695075ac1 Mon Sep 17 00:00:00 2001 From: chitin Date: Thu, 19 Jan 2017 18:47:23 +0800 Subject: [PATCH] [EAGLE-845][IMPROVEMENT] Using aggregation query in backend instead of calculating count in frontend. [IMPROVEMENT] Using aggregation query in backend instead of calculating count in frontend. - Using siteId as query argument. - Using aggregation query in backend instead of calculating count in frontend. Author: chitin Closes #782 from chitin/refactorquery. --- .../webapp/app/apps/hadoop_metric/config.json | 18 ++-- .../app/apps/hadoop_metric/ctrls/overview.js | 96 +++++++++---------- .../hadoop_metric/ctrls/regionDetailCtrl.js | 9 +- .../webapp/app/apps/hadoop_metric/index.js | 20 ++++ .../apps/hadoop_metric/partials/overview.html | 40 ++++++-- .../widgets/availabilityChart.js | 72 +++++++------- 6 files changed, 148 insertions(+), 107 deletions(-) diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/config.json b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/config.json index 185e7919c2..88b14f6e32 100644 --- a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/config.json +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/config.json @@ -42,15 +42,15 @@ "nonheap" : "hadoop.memory.nonheapmemoryusage.used", "heap" : "hadoop.memory.heapmemoryusage.used", "directmemory" : "hadoop.bufferpool.direct.memoryused", - "GC count" : "hadoop.hbase.jvm.gccount", - "GC TimeMillis" : "hadoop.hbase.jvm.gctimemillis", - "QueueSize" : "hadoop.hbase.ipc.ipc.queuesize", - "NumCallsInGeneralQueue" : "hadoop.hbase.ipc.ipc.numcallsingeneralqueue", - "NumActiveHandler" : "hadoop.hbase.ipc.ipc.numactivehandler", - "IPC Queue Time (99th" : "hadoop.hbase.ipc.ipc.queuecalltime_99th_percentile", - "IPC Process Time (99th" : "hadoop.hbase.ipc.ipc.processcalltime_99th_percentile", - "QueueCallTime_num_ops" : "hadoop.hbase.ipc.ipc.queuecalltime_num_ops", - "ProcessCallTime_num_ops" : "hadoop.hbase.ipc.ipc.processcalltime_num_ops", + "GCCount" : "hadoop.hbase.jvm.gccount", + "GCTimeMillis" : "hadoop.hbase.jvm.gctimemillis", + "QueueSize" : "hadoop.hbase.regionserver.ipc.queuesize", + "NumCallsInGeneralQueue" : "hadoop.hbase.regionserver.ipc.numcallsingeneralqueue", + "NumActiveHandler" : "hadoop.hbase.regionserver.ipc.numactivehandler", + "IPCQueueTime99th" : "hadoop.hbase.regionserver.ipc.queuecalltime_99th_percentile", + "IPCProcessTime99th" : "hadoop.hbase.regionserver.ipc.processcalltime_99th_percentile", + "QueueCallTime_num_ops" : "hadoop.hbase.regionserver.ipc.queuecalltime_num_ops", + "ProcessCallTime_num_ops" : "hadoop.hbase.regionserver.ipc.processcalltime_num_ops", "RegionCount" : "hadoop.hbase.regionserver.server.regioncount", "StoreCount" : "hadoop.hbase.regionserver.server.storecount", "MemStoreSize" : "hadoop.hbase.regionserver.server.memstoresize", diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/overview.js b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/overview.js index 8e9140fda4..406adf2dd1 100644 --- a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/overview.js +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/overview.js @@ -69,12 +69,12 @@ host: $scope.defaultHostname }; return METRIC.aggMetricsToEntities(METRIC.hbaseMetricsAggregation(jobCond, name, ["site"], "avg(value)", intervalMin, trendStartTime, trendEndTime), flag) - ._promise.then(function (list) { - var metricFlag = $.map(list, function (metrics) { - return metrics[0].flag; + ._promise.then(function (list) { + var metricFlag = $.map(list, function (metrics) { + return metrics[0].flag; + }); + return [metricFlag, list]; }); - return [metricFlag, list]; - }); }); } @@ -82,7 +82,7 @@ var series = []; $.each(metrics, function (i, metricMap) { - if(typeof metricMap !== 'undefined') { + if (typeof metricMap !== 'undefined') { series.push(METRIC.metricsToSeries(legendName[i], metricMap[0], option)); } }); @@ -93,12 +93,28 @@ }; } + function countHBaseRole(site, status, role, groups, filed, limit) { + var jobCond = { + site: site, + status: status, + role: role + }; + return METRIC.aggHBaseInstance(jobCond, groups, filed, limit); + } + + function sumAllRegions(site, role, groups, filed, limit) { + var jobCond = { + site: site, + role: role + }; + return METRIC.aggHBaseInstance(jobCond, groups, filed, limit); + } + // TODO: Optimize the chart count // TODO: ECharts dynamic refresh series bug: https://github.com/ecomfe/echarts/issues/4033 $scope.refresh = function () { - var hbaseservers = METRIC.hbasehostStatus({site: $scope.site}); var metricspromies = []; METRIC.getMetricObj().then(function (res) { var masterMetricList = res.master; @@ -107,7 +123,7 @@ } $q.all(metricspromies).then(function (resp) { var metricObj = {}; - for(var i=0; i < resp.length; i+=1) { + for (var i = 0; i < resp.length; i += 1) { metricObj[resp[i][0]] = resp[i][1]; } return metricObj; @@ -132,51 +148,31 @@ }); }); - METRIC.regionserverStatus($scope.hostname, $scope.site)._promise.then(function (res) { - $scope.regionstatus = res; + countHBaseRole($scope.site, "active", "hmaster", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.hmasteractivenum = data.value[0]; + }); }); - - hbaseservers._promise.then(function (res) { - var regionserverhealtynum = 0; - var regionservertotal = 0; - var hmasterstandbynum = 0; - var hmasteractivenum = 0; - var regionsnum = 0; - var regionArray = []; - $.each(res, function (i, server) { - var role = server.tags.role; - var status = server.status; - if (role === "regionserver") { - regionservertotal++; - if (status === "live") { - regionserverhealtynum++; - } - regionArray.push(METRIC.regionserverStatus(server.tags.hostname, $scope.site)._promise.then(function (res) { - return res[0].numRegions; - })); - } - else if (role === "hmaster") { - if (status === "active") { - hmasteractivenum++; - } else { - hmasterstandbynum++; - } - - } + countHBaseRole($scope.site, "standby", "hmaster", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.hmasterstandbynum = data.value[0] }); + }); - $q.all(regionArray).then(function (res) { - - for(var i=0; i
- + - + - + - - + @@ -67,7 +88,8 @@

{{metric.title}}

-
diff --git a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/widgets/availabilityChart.js b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/widgets/availabilityChart.js index f2606ad872..9e73c9b718 100644 --- a/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/widgets/availabilityChart.js +++ b/eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/widgets/availabilityChart.js @@ -54,38 +54,39 @@ // Customize chart color $scope.bgColor = COLOR_MAPPING[$scope.type]; + function countHBaseRole(site, status, role, groups, filed, limit) { + var jobCond = { + site: site, + status: status, + role: role + }; + return METRIC.aggHBaseInstance(jobCond, groups, filed, limit); + } // Ref: jpm widget if need keep refresh the widget function refresh() { $.each($scope.list, function (i, site) { - var hbaseservers = METRIC.hbasehostStatus({site: site.siteName}); - hbaseservers._promise.then(function (res) { - var hmasternum = 0; - var hmasteractivenum = 0; - var regionserverHealthynum = 0; - var regionservertotal = 0; - $.each(res, function (i, server) { - var role = server.tags.role; - var status = server.status; - if (role === "hmaster") { - hmasternum++; - if (status === "active") { - hmasteractivenum++; - } - } else if (role === "regionserver") { - regionservertotal++; - if (status === "live") { - regionserverHealthynum++; - } - } + + countHBaseRole(site.siteId, "active", "hmaster", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.hmasteractivenum = data.value[0]; + }); + }); + countHBaseRole(site.siteId, "standby", "hmaster", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.hmasterstandbynum = data.value[0] + }); + }); + countHBaseRole(site.siteId, "live", "regionserver", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.regionserverhealtynum = data.value[0]; + }); + }); + countHBaseRole(site.siteId, "dead", "regionserver", ["site"], "count")._promise.then(function (res) { + $.map(res, function (data) { + $scope.regionserverunhealtynum = data.value[0]; }); - $scope.hbaseinfo = { - hmasternum: hmasternum, - hmasteractivenum: hmasteractivenum, - regionserverHealthynum: regionserverHealthynum, - regionservertotal: regionservertotal - }; }); }); } @@ -101,18 +102,21 @@ '
' + '
' + '

{{type}}

' + - '
' + + '
' + '' + - '{{hbaseinfo.hmasternum}} Masters (' + - '{{hbaseinfo.hmasteractivenum}} Active / ' + - '{{hbaseinfo.hmasternum - hbaseinfo.hmasteractivenum}} Standby)' + + '{{hmasteractivenum+hmasterstandbynum}} Masters (' + + '{{hmasteractivenum}}0 Active / ' + + '{{hmasterstandbynum}}0 Standby)' + '' + '
' + - '
' + + '
' + + ' NO DATA' + + '
' + + '
' + '' + - '{{hbaseinfo.regionservertotal}} RegionServers (' + - '{{hbaseinfo.regionserverHealthynum}} Healthy / ' + - '{{hbaseinfo.regionservertotal - hbaseinfo.regionserverHealthynum}} Unhealthy)' + + '{{regionserverhealtynum+regionserverunhealtynum}} RegionServers (' + + '{{regionserverhealtynum}}0 Healthy / ' + + '{{regionserverunhealtynum}}0 Unhealthy)' + '' + '
' + '
' +

{{hmasteractivenum}} + {{hmasteractivenum}} + + Active HBase Master{{regionsnum}} + {{regionsnum}} + + Regions
{{hmasterstandbynum}} + + {{hmasterstandbynum}} + 0 + Backup HBase Master - {{hmasteraverageload}} - + {{common.number.format(hmasteraverageload, 0)}} + Average Load
{{regionservertotal}}RegionServers: {{regionserverhealtynum}} Good Health / {{regionservertotal-regionserverhealtynum}} + + + {{regionserverhealtynum+regionserverunhealtynum}} + 0 + RegionServers: + + {{regionserverhealtynum}} + 0 + Good Health / + + {{regionserverunhealtynum}} + 0 Bad Health