Skip to content

Commit

Permalink
YARN-3360. Add JMX metrics to TimelineDataManager (Jason Lowe via jea…
Browse files Browse the repository at this point in the history
…gles)
  • Loading branch information
Jonathan Eagles committed Jun 24, 2015
1 parent 2236b57 commit 4c659dd
Show file tree
Hide file tree
Showing 7 changed files with 303 additions and 0 deletions.
2 changes: 2 additions & 0 deletions hadoop-yarn-project/CHANGES.txt
Expand Up @@ -60,6 +60,8 @@ Release 2.8.0 - UNRELEASED

NEW FEATURES

YARN-3360. Add JMX metrics to TimelineDataManager (Jason Lowe via jeagles)

YARN-3345. Add non-exclusive node label API. (Wangda Tan via jianhe)

YARN-3365. Enhanced NodeManager to support using the 'tc' tool via
Expand Down
Expand Up @@ -31,6 +31,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
Expand All @@ -56,6 +57,7 @@ public class TimelineDataManager extends AbstractService {
@VisibleForTesting
public static final String DEFAULT_DOMAIN_ID = "DEFAULT";

private TimelineDataManagerMetrics metrics;
private TimelineStore store;
private TimelineACLsManager timelineACLsManager;

Expand All @@ -69,6 +71,7 @@ public TimelineDataManager(TimelineStore store,

@Override
protected void serviceInit(Configuration conf) throws Exception {
metrics = TimelineDataManagerMetrics.create();
TimelineDomain domain = store.getDomain("DEFAULT");
// it is okay to reuse an existing domain even if it was created by another
// user of the timeline server before, because it allows everybody to access.
Expand Down Expand Up @@ -130,6 +133,38 @@ public TimelineEntities getEntities(
Long limit,
EnumSet<Field> fields,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrGetEntitiesOps();
try {
TimelineEntities entities = doGetEntities(
entityType,
primaryFilter,
secondaryFilter,
windowStart,
windowEnd,
fromId,
fromTs,
limit,
fields,
callerUGI);
metrics.incrGetEntitiesTotal(entities.getEntities().size());
return entities;
} finally {
metrics.addGetEntitiesTime(Time.monotonicNow() - startTime);
}
}

private TimelineEntities doGetEntities(
String entityType,
NameValuePair primaryFilter,
Collection<NameValuePair> secondaryFilter,
Long windowStart,
Long windowEnd,
String fromId,
Long fromTs,
Long limit,
EnumSet<Field> fields,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineEntities entities = null;
entities = store.getEntities(
entityType,
Expand Down Expand Up @@ -161,6 +196,20 @@ public TimelineEntity getEntity(
String entityId,
EnumSet<Field> fields,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrGetEntityOps();
try {
return doGetEntity(entityType, entityId, fields, callerUGI);
} finally {
metrics.addGetEntityTime(Time.monotonicNow() - startTime);
}
}

private TimelineEntity doGetEntity(
String entityType,
String entityId,
EnumSet<Field> fields,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineEntity entity = null;
entity =
store.getEntity(entityId, entityType, fields);
Expand Down Expand Up @@ -190,6 +239,32 @@ public TimelineEvents getEvents(
Long windowEnd,
Long limit,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrGetEventsOps();
try {
TimelineEvents events = doGetEvents(
entityType,
entityIds,
eventTypes,
windowStart,
windowEnd,
limit,
callerUGI);
metrics.incrGetEventsTotal(events.getAllEvents().size());
return events;
} finally {
metrics.addGetEventsTime(Time.monotonicNow() - startTime);
}
}

private TimelineEvents doGetEvents(
String entityType,
SortedSet<String> entityIds,
SortedSet<String> eventTypes,
Long windowStart,
Long windowEnd,
Long limit,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineEvents events = null;
events = store.getEntityTimelines(
entityType,
Expand Down Expand Up @@ -236,9 +311,22 @@ public TimelineEvents getEvents(
public TimelinePutResponse postEntities(
TimelineEntities entities,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrPostEntitiesOps();
try {
return doPostEntities(entities, callerUGI);
} finally {
metrics.addPostEntitiesTime(Time.monotonicNow() - startTime);
}
}

private TimelinePutResponse doPostEntities(
TimelineEntities entities,
UserGroupInformation callerUGI) throws YarnException, IOException {
if (entities == null) {
return new TimelinePutResponse();
}
metrics.incrPostEntitiesTotal(entities.getEntities().size());
TimelineEntities entitiesToPut = new TimelineEntities();
List<TimelinePutResponse.TimelinePutError> errors =
new ArrayList<TimelinePutResponse.TimelinePutError>();
Expand Down Expand Up @@ -303,6 +391,17 @@ public TimelinePutResponse postEntities(
*/
public void putDomain(TimelineDomain domain,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrPutDomainOps();
try {
doPutDomain(domain, callerUGI);
} finally {
metrics.addPutDomainTime(Time.monotonicNow() - startTime);
}
}

private void doPutDomain(TimelineDomain domain,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineDomain existingDomain =
store.getDomain(domain.getId());
if (existingDomain != null) {
Expand All @@ -329,6 +428,17 @@ public void putDomain(TimelineDomain domain,
*/
public TimelineDomain getDomain(String domainId,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrGetDomainOps();
try {
return doGetDomain(domainId, callerUGI);
} finally {
metrics.addGetDomainTime(Time.monotonicNow() - startTime);
}
}

private TimelineDomain doGetDomain(String domainId,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineDomain domain = store.getDomain(domainId);
if (domain != null) {
if (timelineACLsManager.checkAccess(callerUGI, domain)) {
Expand All @@ -344,6 +454,19 @@ public TimelineDomain getDomain(String domainId,
*/
public TimelineDomains getDomains(String owner,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrGetDomainsOps();
try {
TimelineDomains domains = doGetDomains(owner, callerUGI);
metrics.incrGetDomainsTotal(domains.getDomains().size());
return domains;
} finally {
metrics.addGetDomainsTime(Time.monotonicNow() - startTime);
}
}

private TimelineDomains doGetDomains(String owner,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineDomains domains = store.getDomains(owner);
boolean hasAccess = true;
if (domains.getDomains().size() > 0) {
Expand Down
@@ -0,0 +1,174 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.timeline;

import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableRate;

/** This class tracks metrics for the TimelineDataManager. */
@Metrics(about="Metrics for TimelineDataManager", context="yarn")
public class TimelineDataManagerMetrics {
@Metric("getEntities calls")
MutableCounterLong getEntitiesOps;

@Metric("Entities returned via getEntities")
MutableCounterLong getEntitiesTotal;

@Metric("getEntities processing time")
MutableRate getEntitiesTime;

@Metric("getEntity calls")
MutableCounterLong getEntityOps;

@Metric("getEntity processing time")
MutableRate getEntityTime;

@Metric("getEvents calls")
MutableCounterLong getEventsOps;

@Metric("Events returned via getEvents")
MutableCounterLong getEventsTotal;

@Metric("getEvents processing time")
MutableRate getEventsTime;

@Metric("postEntities calls")
MutableCounterLong postEntitiesOps;

@Metric("Entities posted via postEntities")
MutableCounterLong postEntitiesTotal;

@Metric("postEntities processing time")
MutableRate postEntitiesTime;

@Metric("putDomain calls")
MutableCounterLong putDomainOps;

@Metric("putDomain processing time")
MutableRate putDomainTime;

@Metric("getDomain calls")
MutableCounterLong getDomainOps;

@Metric("getDomain processing time")
MutableRate getDomainTime;

@Metric("getDomains calls")
MutableCounterLong getDomainsOps;

@Metric("Domains returned via getDomains")
MutableCounterLong getDomainsTotal;

@Metric("getDomains processing time")
MutableRate getDomainsTime;

@Metric("Total calls")
public long totalOps() {
return getEntitiesOps.value() +
getEntityOps.value() +
getEventsOps.value() +
postEntitiesOps.value() +
putDomainOps.value() +
getDomainOps.value() +
getDomainsOps.value();
}

TimelineDataManagerMetrics() {
}

public static TimelineDataManagerMetrics create() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(new TimelineDataManagerMetrics());
}

public void incrGetEntitiesOps() {
getEntitiesOps.incr();
}

public void incrGetEntitiesTotal(long delta) {
getEntitiesTotal.incr(delta);
}

public void addGetEntitiesTime(long msec) {
getEntitiesTime.add(msec);
}

public void incrGetEntityOps() {
getEntityOps.incr();
}

public void addGetEntityTime(long msec) {
getEntityTime.add(msec);
}

public void incrGetEventsOps() {
getEventsOps.incr();
}

public void incrGetEventsTotal(long delta) {
getEventsTotal.incr(delta);
}

public void addGetEventsTime(long msec) {
getEventsTime.add(msec);
}

public void incrPostEntitiesOps() {
postEntitiesOps.incr();
}

public void incrPostEntitiesTotal(long delta) {
postEntitiesTotal.incr(delta);
}

public void addPostEntitiesTime(long msec) {
postEntitiesTime.add(msec);
}

public void incrPutDomainOps() {
putDomainOps.incr();
}

public void addPutDomainTime(long msec) {
putDomainTime.add(msec);
}

public void incrGetDomainOps() {
getDomainOps.incr();
}

public void addGetDomainTime(long msec) {
getDomainTime.add(msec);
}

public void incrGetDomainsOps() {
getDomainsOps.incr();
}

public void incrGetDomainsTotal(long delta) {
getDomainsTotal.incr(delta);
}

public void addGetDomainsTime(long msec) {
getDomainsTime.add(msec);
}
}
Expand Up @@ -67,6 +67,7 @@ public static void setup() throws Exception {
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
dataManager =
new TimelineDataManager(store, aclsManager);
dataManager.init(conf);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager =
new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
Expand Down
Expand Up @@ -98,6 +98,7 @@ public void setup() throws Exception {
TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration());
TimelineDataManager dataManager =
new TimelineDataManager(store, aclsManager);
dataManager.init(conf);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
historyManager =
new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
Expand Down

0 comments on commit 4c659dd

Please sign in to comment.