Skip to content
This repository has been archived by the owner on May 12, 2021. It is now read-only.

Commit

Permalink
[EAGLE-845][IMPROVEMENT] Using aggregation query in backend instead o…
Browse files Browse the repository at this point in the history 
…f calculating count in frontend.

[IMPROVEMENT] Using aggregation query in backend instead of calculating count in frontend.
- Using siteId as query argument.
- Using aggregation query in backend instead of calculating count in frontend.

Author: chitin <chitin1027@gmail.com>

Closes #782 from chitin/refactorquery.
  • Loading branch information
@denglingang @haoch
denglingang authored and haoch committed Jan 19, 2017
1 parent 6537254 commit 9c71b0e
Show file tree
Hide file tree
Showing 6 changed files with 148 additions and 107 deletions.
18 changes: 9 additions & 9 deletions eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,15 @@
"nonheap" : "hadoop.memory.nonheapmemoryusage.used",
"heap" : "hadoop.memory.heapmemoryusage.used",
"directmemory" : "hadoop.bufferpool.direct.memoryused",
"GC count" : "hadoop.hbase.jvm.gccount",
"GC TimeMillis" : "hadoop.hbase.jvm.gctimemillis",
"QueueSize" : "hadoop.hbase.ipc.ipc.queuesize",
"NumCallsInGeneralQueue" : "hadoop.hbase.ipc.ipc.numcallsingeneralqueue",
"NumActiveHandler" : "hadoop.hbase.ipc.ipc.numactivehandler",
"IPC Queue Time (99th" : "hadoop.hbase.ipc.ipc.queuecalltime_99th_percentile",
"IPC Process Time (99th" : "hadoop.hbase.ipc.ipc.processcalltime_99th_percentile",
"QueueCallTime_num_ops" : "hadoop.hbase.ipc.ipc.queuecalltime_num_ops",
"ProcessCallTime_num_ops" : "hadoop.hbase.ipc.ipc.processcalltime_num_ops",
"GCCount" : "hadoop.hbase.jvm.gccount",
"GCTimeMillis" : "hadoop.hbase.jvm.gctimemillis",
"QueueSize" : "hadoop.hbase.regionserver.ipc.queuesize",
"NumCallsInGeneralQueue" : "hadoop.hbase.regionserver.ipc.numcallsingeneralqueue",
"NumActiveHandler" : "hadoop.hbase.regionserver.ipc.numactivehandler",
"IPCQueueTime99th" : "hadoop.hbase.regionserver.ipc.queuecalltime_99th_percentile",
"IPCProcessTime99th" : "hadoop.hbase.regionserver.ipc.processcalltime_99th_percentile",
"QueueCallTime_num_ops" : "hadoop.hbase.regionserver.ipc.queuecalltime_num_ops",
"ProcessCallTime_num_ops" : "hadoop.hbase.regionserver.ipc.processcalltime_num_ops",
"RegionCount" : "hadoop.hbase.regionserver.server.regioncount",
"StoreCount" : "hadoop.hbase.regionserver.server.storecount",
"MemStoreSize" : "hadoop.hbase.regionserver.server.memstoresize",
Expand Down
96 changes: 46 additions & 50 deletions eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/overview.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,20 +69,20 @@
host: $scope.defaultHostname
};
return METRIC.aggMetricsToEntities(METRIC.hbaseMetricsAggregation(jobCond, name, ["site"], "avg(value)", intervalMin, trendStartTime, trendEndTime), flag)
._promise.then(function (list) {
var metricFlag = $.map(list, function (metrics) {
return metrics[0].flag;
._promise.then(function (list) {
var metricFlag = $.map(list, function (metrics) {
return metrics[0].flag;
});
return [metricFlag, list];
});
return [metricFlag, list];
});
});
}

function mergeMetricToOneSeries(metricTitle, metrics, legendName, dataOption, option) {
var series = [];

$.each(metrics, function (i, metricMap) {
if(typeof metricMap !== 'undefined') {
if (typeof metricMap !== 'undefined') {
series.push(METRIC.metricsToSeries(legendName[i], metricMap[0], option));
}
});
Expand All @@ -93,12 +93,28 @@
};
}

function countHBaseRole(site, status, role, groups, filed, limit) {
var jobCond = {
site: site,
status: status,
role: role
};
return METRIC.aggHBaseInstance(jobCond, groups, filed, limit);
}

function sumAllRegions(site, role, groups, filed, limit) {
var jobCond = {
site: site,
role: role
};
return METRIC.aggHBaseInstance(jobCond, groups, filed, limit);
}

// TODO: Optimize the chart count
// TODO: ECharts dynamic refresh series bug: https://github.com/ecomfe/echarts/issues/4033


$scope.refresh = function () {
var hbaseservers = METRIC.hbasehostStatus({site: $scope.site});
var metricspromies = [];
METRIC.getMetricObj().then(function (res) {
var masterMetricList = res.master;
Expand All @@ -107,7 +123,7 @@
}
$q.all(metricspromies).then(function (resp) {
var metricObj = {};
for(var i=0; i < resp.length; i+=1) {
for (var i = 0; i < resp.length; i += 1) {
metricObj[resp[i][0]] = resp[i][1];
}
return metricObj;
Expand All @@ -132,51 +148,31 @@
});
});

METRIC.regionserverStatus($scope.hostname, $scope.site)._promise.then(function (res) {
$scope.regionstatus = res;
countHBaseRole($scope.site, "active", "hmaster", ["site"], "count")._promise.then(function (res) {
$.map(res, function (data) {
$scope.hmasteractivenum = data.value[0];
});
});

hbaseservers._promise.then(function (res) {
var regionserverhealtynum = 0;
var regionservertotal = 0;
var hmasterstandbynum = 0;
var hmasteractivenum = 0;
var regionsnum = 0;
var regionArray = [];
$.each(res, function (i, server) {
var role = server.tags.role;
var status = server.status;
if (role === "regionserver") {
regionservertotal++;
if (status === "live") {
regionserverhealtynum++;
}
regionArray.push(METRIC.regionserverStatus(server.tags.hostname, $scope.site)._promise.then(function (res) {
return res[0].numRegions;
}));
}
else if (role === "hmaster") {
if (status === "active") {
hmasteractivenum++;
} else {
hmasterstandbynum++;
}

}
countHBaseRole($scope.site, "standby", "hmaster", ["site"], "count")._promise.then(function (res) {
$.map(res, function (data) {
$scope.hmasterstandbynum = data.value[0]
});
});

$q.all(regionArray).then(function (res) {

for(var i=0; i<regionArray.length; i++) {
regionsnum+=res[i];
}
$scope.regionsnum = regionsnum;
countHBaseRole($scope.site, "live", "regionserver", ["site"], "count")._promise.then(function (res) {
$.map(res, function (data) {
$scope.regionserverhealtynum = data.value[0];
});
});
countHBaseRole($scope.site, "dead", "regionserver", ["site"], "count")._promise.then(function (res) {
$.map(res, function (data) {
$scope.regionserverunhealtynum = data.value[0];
});
});
sumAllRegions($scope.site, "regionserver", ["site"], "sum(numRegions)")._promise.then(function (res) {
$.map(res, function (data) {
$scope.regionsnum = data.value[0];
});

$scope.regionserverhealtynum = regionserverhealtynum;
$scope.regionservertotal = regionservertotal;
$scope.hmasterstandbynum = hmasterstandbynum;
$scope.hmasteractivenum = hmasteractivenum;
});

activeMasterInfo._promise.then(function (res) {
Expand All @@ -187,7 +183,7 @@
component: "hbasemaster",
host: $scope.defaultHostname
};
METRIC.hbaseMomentMetric(jobCond,"hadoop.hbase.master.server.averageload", 1).then(function (res) {
METRIC.hbaseMomentMetric(jobCond, "hadoop.hbase.master.server.averageload", 1).then(function (res) {
$scope.hmasteraverageload = (typeof res.data.obj[0] !== 'undefined') ? res.data.obj[0].value[0] : "-1";
});
});
Expand Down
9 changes: 4 additions & 5 deletions eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/ctrls/regionDetailCtrl.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@
var endTime = Time.endTime();
var metricspromies = [];
METRIC.getMetricObj().then(function (res) {
console.log(res.regionserver);
var masterMetricList = res.regionserver;
for (var metricKey in masterMetricList) {
metricspromies.push(generateHbaseMetric(masterMetricList[metricKey], startTime, endTime, metricKey));
Expand All @@ -87,13 +86,13 @@
$scope.metricList = [];
$scope.metricList.push(mergeSeries("Memory Usage", [seriesObj["nonheap"], seriesObj["heap"]], ["nonheap", "heap"], sizeoption));
$scope.metricList.push(mergeSeries("Direct Memory Usage", [seriesObj["directmemory"]], ["directmemory"], sizeoption));
$scope.metricList.push(mergeSeries("GC count", [seriesObj["GC count"]], ["GC count"], {}));
$scope.metricList.push(mergeSeries("GC TimeMillis", [seriesObj["GC TimeMillis"]], ["GC TimeMillis"], gctimeoption));
$scope.metricList.push(mergeSeries("GC count", [seriesObj["GCCount"]], ["GC count"], {}));
$scope.metricList.push(mergeSeries("GC TimeMillis", [seriesObj["GCTimeMillis"]], ["GC TimeMillis"], gctimeoption));
$scope.metricList.push(mergeSeries("QueueSize", [seriesObj["QueueSize"]], ["QueueSize"], {}));
$scope.metricList.push(mergeSeries("NumCallsInGeneralQueue", [seriesObj["NumCallsInGeneralQueue"]], ["NumCallsInGeneralQueue"], {}));
$scope.metricList.push(mergeSeries("NumActiveHandler", [seriesObj["NumActiveHandler"]], ["NumActiveHandler"], {}));
$scope.metricList.push(mergeSeries("IPC Queue Time (99th)", [seriesObj["IPC Queue Time (99th)"]], ["IPC Queue Time (99th)"], {}));
$scope.metricList.push(mergeSeries("IPC Process Time (99th)", [seriesObj["IPC Process Time (99th)"]], ["IPC Process Time (99th)"], {}));
$scope.metricList.push(mergeSeries("IPC Queue Time (99th)", [seriesObj["IPCQueueTime99th"]], ["IPC Queue Time (99th)"], {}));
$scope.metricList.push(mergeSeries("IPC Process Time (99th)", [seriesObj["IPCProcessTime99th"]], ["IPC Process Time (99th)"], {}));
$scope.metricList.push(mergeSeries("QueueCallTime_num_ops", [seriesObj["QueueCallTime_num_ops"]], ["QueueCallTime_num_ops"], {}));
$scope.metricList.push(mergeSeries("ProcessCallTime_num_ops", [seriesObj["ProcessCallTime_num_ops"]], ["ProcessCallTime_num_ops"], {}));
$scope.metricList.push(mergeSeries("RegionCount", [seriesObj["RegionCount"]], ["RegionCount"], {}));
Expand Down
20 changes: 20 additions & 0 deletions eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
METRIC.QUERY_HBASE_METRICS = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]{*}&metricName=${metric}&pageSize=${limit}';
METRIC.QUERY_HBASE_METRICS_WITHTIME = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]{*}&metricName=${metric}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}';
METRIC.QUERY_HBASE_INSTANCE = '${baseURL}/rest/entities?query=HbaseServiceInstance[${condition}]{*}&pageSize=${limit}';
METRIC.QUERY_HBASE_INSTANCE_AGG = "${baseURL}/rest/entities?query=HbaseServiceInstance[${condition}]<${groups}>{${field}}&pageSize=${limit}";
METRIC.QUERY_HBASE_METRICS_INTERVAL = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]<${groups}>{${field}}${order}${top}&metricName=${metric}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}&intervalmin=${intervalMin}&timeSeries=true';
/**
* Fetch query content with current site application configuration
Expand Down Expand Up @@ -184,6 +185,25 @@
return METRIC.get(metrics_url);
};

METRIC.aggHBaseInstance = function (condition, groups, field, limit) {
var fields = field.split(/\s*,\s*/);
var fieldStr = $.map(fields, function (field, index) {
var matches = field.match(/^([^\s]*)(\s+.*)?$/);
if (matches[2]) {
orderId = index;
}
return matches[1];
}).join(", ");
var config = {
condition: METRIC.condition(condition),
groups: toFields(groups),
field: fieldStr,
limit: limit || 10000
};
var metrics_url = common.template(getQuery("HBASE_INSTANCE_AGG"), config);
return wrapList(METRIC.get(metrics_url));
};

METRIC.hbaseMetricsAggregation = function (condition, metric, groups, field, intervalMin, startTime, endTime, top, limit) {
var fields = field.split(/\s*,\s*/);
var orderId = -1;
Expand Down
40 changes: 31 additions & 9 deletions eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/partials/overview.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,25 +26,46 @@ <h3 class="box-title">
<div class="box-body">
<table class="table table-striped hadoopMetric-table">
<tr>
<td width="5%">{{hmasteractivenum}}</td>
<td width="5%">
<span ng-show="hmasteractivenum">{{hmasteractivenum}}</span>
<span ng-show="!hmasteractivenum" class="fa fa fa-refresh fa-spin no-animate"></span>
</td>
<th width="45%">Active HBase Master</th>
<td width="5%">{{regionsnum}}</td>
<td width="5%">
<span ng-show="regionsnum">{{regionsnum}}</span>
<span ng-show="!regionsnum" class="fa fa fa-refresh fa-spin no-animate"></span>
</td>
<th width="45%">Regions
</th>
</tr>
<tr>
<td><a ui-sref="backupMasterList({siteId: site})" target="_blank">{{hmasterstandbynum}}</a></td>
<td>
<a ui-sref="backupMasterList({siteId: site})" target="_blank">
<span ng-show="hmasterstandbynum">{{hmasterstandbynum}}</span></a>
<span ng-show="!hmasterstandbynum">0</span>
</td>
<th>Backup HBase Master</th>
<td>
<span ng-show="hmasteraverageload !== '-1'">{{hmasteraverageload}}</span>
<span ng-show="hmasteraverageload === '-1'" class="fa fa-question-circle"></span>
<span ng-show="hmasteraverageload !== '-1'">{{common.number.format(hmasteraverageload, 0)}}</span>
<span ng-show="hmasteraverageload === '-1'" class="fa fa-refresh fa-spin no-animate"></span>
</td>
<th>Average Load</th>
</tr>
<tr>
<td><a ui-sref="regionList({siteId: site})">{{regionservertotal}}</a></td>
<th>RegionServers: <a ui-sref="regionList({siteId: site})">{{regionserverhealtynum}}</a> <span
class="label label-success">Good Health</span> / <a ui-sref="regionList({siteId: site})">{{regionservertotal-regionserverhealtynum}}</a>
<td>
<a ui-sref="regionList({siteId: site})">
<span ng-show="regionserverhealtynum || regionserverunhealtynum">{{regionserverhealtynum+regionserverunhealtynum}}</span></a>
<span ng-show="!regionserverhealtynum && !regionserverunhealtynum">0</span>
</td>
<th>RegionServers:
<a ui-sref="regionList({siteId: site})">
<span ng-show="regionserverhealtynum">{{regionserverhealtynum}}</span></a>
<span ng-show="!regionserverhealtynum">0</span>
<span
class="label label-success">Good Health</span> /
<a ui-sref="regionList({siteId: site})">
<span ng-show="regionserverunhealtynum">{{regionserverunhealtynum}}</span></a>
<span ng-show="!regionserverunhealtynum">0</span>
<span class="label label-danger">Bad Health</span>
</th>
<td></td>
Expand All @@ -67,7 +88,8 @@ <h3 class="box-title">
<div class="col-sm-6 col-md-4 col-lg-3" ng-repeat="metric in metricList track by $index">
<div class="hadoopMetric-chart">
<h3>{{metric.title}}</h3>
<div ng-show="metric.series.length" chart class="hadoopMetric-chart-container" series="metric.series"
<div ng-show="metric.series.length" chart class="hadoopMetric-chart-container"
series="metric.series"
option="metric.option"></div>
<div ng-show="!metric.series.length" class="hadoopMetric-chart-container">
<div class="hadoopMetric-chart-container-withborder">
Expand Down
72 changes: 38 additions & 34 deletions eagle-hadoop-metric/src/main/webapp/app/apps/hadoop_metric/widgets/availabilityChart.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,38 +54,39 @@
// Customize chart color
$scope.bgColor = COLOR_MAPPING[$scope.type];

function countHBaseRole(site, status, role, groups, filed, limit) {
var jobCond = {
site: site,
status: status,
role: role
};
return METRIC.aggHBaseInstance(jobCond, groups, filed, limit);
}

// Ref: jpm widget if need keep refresh the widget

function refresh() {
$.each($scope.list, function (i, site) {
var hbaseservers = METRIC.hbasehostStatus({site: site.siteName});
hbaseservers._promise.then(function (res) {
var hmasternum = 0;
var hmasteractivenum = 0;
var regionserverHealthynum = 0;
var regionservertotal = 0;
$.each(res, function (i, server) {
var role = server.tags.role;
var status = server.status;
if (role === "hmaster") {
hmasternum++;
if (status === "active") {
hmasteractivenum++;
}
} else if (role === "regionserver") {
regionservertotal++;
if (status === "live") {
regionserverHealthynum++;
}
}

countHBaseRole(site.siteId, "active", "hmaster", ["site"], "count")._promise.then(function (res) {
$.map(res, function (data) {
$scope.hmasteractivenum = data.value[0];
});
});
countHBaseRole(site.siteId, "standby", "hmaster", ["site"], "count")._promise.then(function (res) {
$.map(res, function (data) {
$scope.hmasterstandbynum = data.value[0]
});
});
countHBaseRole(site.siteId, "live", "regionserver", ["site"], "count")._promise.then(function (res) {
$.map(res, function (data) {
$scope.regionserverhealtynum = data.value[0];
});
});
countHBaseRole(site.siteId, "dead", "regionserver", ["site"], "count")._promise.then(function (res) {
$.map(res, function (data) {
$scope.regionserverunhealtynum = data.value[0];
});
$scope.hbaseinfo = {
hmasternum: hmasternum,
hmasteractivenum: hmasteractivenum,
regionserverHealthynum: regionserverHealthynum,
regionservertotal: regionservertotal
};
});
});
}
Expand All @@ -101,18 +102,21 @@
'<div class="small-box hadoopMetric-widget bg-{{bgColor}}">' +
'<div class="inner">' +
'<h3>{{type}}</h3>' +
'<div class="hadoopMetric-widget-detail">' +
'<div ng-show="hmasteractivenum" class="hadoopMetric-widget-detail">' +
'<a ui-sref="HadoopMetric({siteId: site.siteName})">' +
'<span>{{hbaseinfo.hmasternum}}</span> Masters (' +
'<span>{{hbaseinfo.hmasteractivenum}}</span> Active / ' +
'<span>{{hbaseinfo.hmasternum - hbaseinfo.hmasteractivenum}}</span> Standby)' +
'<span>{{hmasteractivenum+hmasterstandbynum}}</span> Masters (' +
'<span ng-show="hmasteractivenum">{{hmasteractivenum}}</span><span ng-show="!hmasteractivenum">0</span> Active / ' +
'<span ng-show="hmasterstandbynum">{{hmasterstandbynum}}</span><span ng-show="!hmasterstandbynum">0</span> Standby)' +
'</a>' +
'</div>' +
'<div class="hadoopMetric-widget-detail">' +
'<div ng-show="!hmasteractivenum" class="hadoopMetric-widget-detail">' +
'<span class="fa fa-question-circle"></span><span> NO DATA</span>' +
'</div>' +
'<div ng-show="hmasteractivenum" class="hadoopMetric-widget-detail">' +
'<a ui-sref="regionList({siteId: site.siteName})">' +
'<span>{{hbaseinfo.regionservertotal}}</span> RegionServers (' +
'<span>{{hbaseinfo.regionserverHealthynum}}</span> Healthy / ' +
'<span>{{hbaseinfo.regionservertotal - hbaseinfo.regionserverHealthynum}}</span> Unhealthy)' +
'<span>{{regionserverhealtynum+regionserverunhealtynum}}</span> RegionServers (' +
'<span ng-show="regionserverhealtynum">{{regionserverhealtynum}}</span><span ng-show="!regionserverhealtynum">0</span> Healthy / ' +
'<span ng-show="regionserverunhealtynum">{{regionserverunhealtynum}}</span><span ng-show="!regionserverunhealtynum">0</span> Unhealthy)' +
'</a>' +
'</div>' +
'</div>' +
Expand Down

0 comments on commit 9c71b0e

Please sign in to comment.