Skip to content
Permalink
Browse files
[AMBARI-25077] Grafana on a workload cluster should show visualizatio…
…n of only it's own cluster. (#12)
  • Loading branch information
avijayanhwx committed Jan 2, 2019
1 parent c7dcf2b commit 9f800dac83fb6761e4627a7030e839642148224c
Showing 6 changed files with 102 additions and 23 deletions.
@@ -59,7 +59,7 @@ define([

//We get a list of components and their associated metrics.
this.initMetricAppidMapping = function () {
return this.doAmbariRequest({url: '/ws/v1/timeline/metrics/metadata'})
return this.doAmbariRequest({url: '/metadata'})
.then(function (items) {
items = items.data;
allMetrics = {};
@@ -227,7 +227,7 @@ define([
var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + target.metric + metricTransform +
url: '?metricNames=' + target.metric + metricTransform +
metricAggregator + "&hostname=" + target.hosts + '&appId=' + target.app + instanceId + '&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -268,7 +268,7 @@ define([
var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + target.metric + metricTransform
url: '?metricNames=' + target.metric + metricTransform
+ metricAggregator + '&hostname=' + tHost + '&appId=' + target.app + instanceId + '&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -302,7 +302,7 @@ define([
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
var templatedComponent = (_.isEmpty(tComponent)) ? target.app : tComponent;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + target.metric + metricTransform
url: '?metricNames=' + target.metric + metricTransform
+ metricAggregator + '&hostname=' + target.templatedHost + '&appId=' + templatedComponent + instanceId
+ '&startTime=' + from + '&endTime=' + to + precision + topN + seriesAggregator
}).then(
@@ -317,7 +317,7 @@ define([
var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + encodeURIComponent(target.queue) + metricTransform
url: '?metricNames=' + encodeURIComponent(target.queue) + metricTransform
+ metricAggregator + '&appId=resourcemanager' + instanceId + '&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -330,7 +330,7 @@ define([
+ target.precision;
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + target.hbMetric + instanceId + '&appId=hbase&startTime='
url: '?metricNames=' + target.hbMetric + instanceId + '&appId=hbase&startTime='
+ from + '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -345,7 +345,7 @@ define([
var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + target.kbMetric + metricTransform + instanceId
url: '?metricNames=' + target.kbMetric + metricTransform + instanceId
+ metricAggregator + '&appId=kafka_broker&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -361,7 +361,7 @@ define([
var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + target.nnMetric + metricTransform + instanceId
url: '?metricNames=' + target.nnMetric + metricTransform + instanceId
+ metricAggregator + '&appId=namenode&startTime=' + from + '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -377,7 +377,7 @@ define([
var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + target.sTopoMetric + metricTransform + instanceId
url: '?metricNames=' + target.sTopoMetric + metricTransform + instanceId
+ metricAggregator + '&appId=nimbus&startTime=' + from + '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -393,7 +393,7 @@ define([
var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + target.sCoreMetric + metricTransform + instanceId
url: '?metricNames=' + target.sCoreMetric + metricTransform + instanceId
+ metricAggregator + '&appId=ambari-infra-solr&startTime=' + from + '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -408,7 +408,7 @@ define([
var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + target.sCollectionMetric + metricTransform + instanceId
url: '?metricNames=' + target.sCollectionMetric + metricTransform + instanceId
+ metricAggregator + '&appId=ambari-infra-solr&startTime=' + from + '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -424,7 +424,7 @@ define([
var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator;
return self.doAmbariRequest({
url: '/ws/v1/timeline/metrics?metricNames=' + target.sDataSourceMetric + metricTransform + instanceId
url: '?metricNames=' + target.sDataSourceMetric + metricTransform + instanceId
+ metricAggregator + '&appId=druid&startTime=' + from + '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -1111,7 +1111,7 @@ define([
*/
this.testDatasource = function () {
return this.doAmbariRequest({
url: '/ws/v1/timeline/metrics/metadata',
url: '/metadata',
method: 'GET'
}).then(function (response) {
console.log(response);
@@ -1161,7 +1161,7 @@ define([
}
return this.doAmbariRequest({
method: 'GET',
url: '/ws/v1/timeline/metrics/instances?' + 'appId=' + app
url: '/instance?' + 'appId=' + app
}).then(function (response) {
var clusters = [];
var data = response.data;
@@ -1188,7 +1188,7 @@ define([
}
return this.doAmbariRequest({
method: 'GET',
url: '/ws/v1/timeline/metrics/instances?' + 'appId=' + app + '&instanceId=' + cluster
url: '/instance?' + 'appId=' + app + '&instanceId=' + cluster
}).then(function (response) {
var hosts = [];
var data = response.data;
@@ -17,7 +17,6 @@
*/
package org.apache.ambari.metrics.core.timeline;

import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;

@@ -87,6 +86,7 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
private TimelineMetricMetadataManager metricMetadataManager;
private MetricCollectorHAController haController;
private boolean containerMetricsDisabled = false;
private String defaultInstanceId = "";

/**
* Construct the service.
@@ -218,6 +218,7 @@ private synchronized void initializeSubsystem() {
"delay = " + initDelay + ", delay = " + delay);
}
containerMetricsDisabled = configuration.isContainerMetricsDisabled();
defaultInstanceId = configuration.getDefaultInstanceId();
isInitialized = true;
}

@@ -258,7 +259,7 @@ public TimelineMetrics getTimelineMetrics(List<String> metricNames,
List<String> transientMetricNames = new ArrayList<>();

if (configuration.getTimelineMetricsMultipleClusterSupport() && StringUtils.isEmpty(instanceId)) {
instanceId = DEFAULT_INSTANCE_ID;
instanceId = this.defaultInstanceId;
}

List<byte[]> uuids = metricMetadataManager.getUuidsForGetMetricQuery(metricFunctions.keySet(),
@@ -29,7 +29,6 @@
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_SECOND_TABLE_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CONTAINER_METRICS_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DATE_TIERED_COMPACTION_POLICY;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.FIFO_COMPACTION_POLICY_CLASS;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_MAX_RETRIES;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_RESULT_LIMIT;
@@ -203,6 +202,7 @@ public class PhoenixHBaseAccessor {
private TimelineMetricMetadataManager metadataManagerInstance;
private Set<String> eventMetricPatterns = new HashSet<>();
private boolean supportMultipleClusterMetrics = false;
private String defaultInstanceId = "";

private Map<String, Integer> tableTTL = new HashMap<>();

@@ -263,6 +263,7 @@ public PhoenixHBaseAccessor(PhoenixConnectionProvider dataSource) {
tableTTL.put(METRIC_TRANSIENT_TABLE_NAME, metricsConf.getInt(METRICS_TRANSIENT_TABLE_TTL, 7 * 86400)); //7 days

this.supportMultipleClusterMetrics = Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS, "false"));
this.defaultInstanceId = configuration.getDefaultInstanceId();

if (cacheEnabled) {
LOG.debug("Initialising and starting metrics cache committer thread...");
@@ -903,6 +904,10 @@ public void insertMetricRecordsWithMetadata(TimelineMetricMetadataManager metada

boolean acceptMetric = TimelineMetricsFilter.acceptMetric(tm);

if (supportMultipleClusterMetrics && StringUtils.isEmpty(tm.getInstanceId())) {
tm.setInstanceId(defaultInstanceId);
}

// Write to metadata cache on successful write to store
if (metadataManager != null) {
metadataManager.putIfModifiedTimelineMetricMetadata(
@@ -912,9 +917,6 @@ public void insertMetricRecordsWithMetadata(TimelineMetricMetadataManager metada
tm.getHostName(), tm.getAppId());

if (!tm.getAppId().equals("FLUME_HANDLER")) {
if (supportMultipleClusterMetrics && StringUtils.isEmpty(tm.getInstanceId())) {
tm.setInstanceId(DEFAULT_INSTANCE_ID);
}
metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(), tm.getHostName());
}
}
@@ -292,6 +292,9 @@ public class TimelineMetricConfiguration {
public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS =
"timeline.metrics.support.multiple.clusters";

public static final String TIMELINE_METRICS_DEFAULT_INSTANCE_ID =
"timeline.metrics.default.instanceid";

public static final String TIMELINE_METRICS_EVENT_METRIC_PATTERNS =
"timeline.metrics.downsampler.event.metric.patterns";

@@ -512,6 +515,13 @@ public boolean getTimelineMetricsMultipleClusterSupport() {
return false;
}

public String getDefaultInstanceId() {
if (metricsConf != null) {
return metricsConf.get(TIMELINE_METRICS_DEFAULT_INSTANCE_ID, DEFAULT_INSTANCE_ID);
}
return DEFAULT_INSTANCE_ID;
}

public String getTimelineServiceRpcAddress() {
String defaultRpcAddress = "0.0.0.0:60200";
if (metricsConf != null) {
@@ -154,7 +154,7 @@ public class PhoenixTransactSQL {
"START_TIME UNSIGNED_LONG, " +
"SUPPORTS_AGGREGATION BOOLEAN, " +
"IS_WHITELISTED BOOLEAN " +
"CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID)) " +
"CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID, INSTANCE_ID)) " +
"DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";

public static final String CREATE_HOSTED_APPS_METADATA_TABLE_SQL =
@@ -36,6 +36,7 @@
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
@@ -287,6 +288,31 @@ public TimelineMetrics getTimelineMetrics(
}
}

@GET
@Path("metrics/{instanceId}")
@Produces({ MediaType.APPLICATION_JSON })
public TimelineMetrics getTimelineMetricsForInstance(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("instanceId") String instanceId,
@QueryParam("metricNames") String metricNames,
@QueryParam("appId") String appId,
@QueryParam("hostname") String hostname,
@QueryParam("startTime") String startTime,
@QueryParam("endTime") String endTime,
@QueryParam("precision") String precision,
@QueryParam("limit") String limit,
@QueryParam("grouped") String grouped,
@QueryParam("topN") String topN,
@QueryParam("topNFunction") String topNFunction,
@QueryParam("isBottomN") String isBottomN,
@QueryParam("seriesAggregateFunction") String seriesAggregateFunction
) {

return getTimelineMetrics(req, res, metricNames, appId, instanceId, hostname, startTime, endTime, precision, limit,
grouped, topN, topNFunction, isBottomN, seriesAggregateFunction);
}

@GET
@Path("/metrics/summary")
@Produces({ MediaType.APPLICATION_JSON })
@@ -324,6 +350,28 @@ public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(
}
}

@GET
@Path("/metrics/{instanceId}/metadata")
@Produces({ MediaType.APPLICATION_JSON })
public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadataForInstance(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@QueryParam("appId") String appId,
@QueryParam("metricName") String metricPattern,
@QueryParam("includeAll") String includeBlacklistedMetrics
) {
init(res);

try {
return timelineMetricStore.getTimelineMetricMetadata(
parseStr(appId),
parseStr(metricPattern),
parseBoolean(includeBlacklistedMetrics));
} catch (Exception e) {
throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
}
}

@GET
@Path("/metrics/hosts")
@Produces({ MediaType.APPLICATION_JSON })
@@ -341,7 +389,7 @@ public Map<String, Set<String>> getHostedAppsMetadata(
}

@GET
@Path("/metrics/instances")
@Path("/metrics/instance")
@Produces({ MediaType.APPLICATION_JSON })
public Map<String, Map<String, Set<String>>> getClusterHostsMetadata(
@Context HttpServletRequest req,
@@ -358,6 +406,24 @@ public Map<String, Map<String, Set<String>>> getClusterHostsMetadata(
}
}

@GET
@Path("/metrics/{instanceId}/instance")
@Produces({ MediaType.APPLICATION_JSON })
public Map<String, Map<String, Set<String>>> getClusterHostsMetadataForInstance(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@QueryParam("appId") String appId,
@PathParam("instanceId") String instanceId
) {
init(res);

try {
return timelineMetricStore.getInstanceHostsMetadata(instanceId, appId);
} catch (Exception e) {
throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
}
}

@GET
@Path("/metrics/uuid")
@Produces({ MediaType.APPLICATION_JSON })

0 comments on commit 9f800da

Please sign in to comment.