Skip to content

Commit

Permalink
YARN-11272. Add OpDurations For Reservation
Browse files Browse the repository at this point in the history
  • Loading branch information
slfan1989 committed Aug 23, 2022
1 parent a1a870c commit 4c27aad
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 10 deletions.
35 changes: 35 additions & 0 deletions ...rg/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,21 @@ public final class ZKFederationStateStoreOpDurations implements MetricsSource {
@Metric("Duration for a get PolicyConfigurations call")
private MutableRate getPoliciesConfigurations;

@Metric("Duration for a add reservation homeSubCluster call")
private MutableRate addReservationHomeSubCluster;

@Metric("Duration for a get reservation homeSubCluster call")
private MutableRate getReservationHomeSubCluster;

@Metric("Duration for a get reservations homeSubCluster call")
private MutableRate getReservationsHomeSubCluster;

@Metric("Duration for a delete reservation homeSubCluster call")
private MutableRate deleteReservationHomeSubCluster;

@Metric("Duration for a update reservation homeSubCluster call")
private MutableRate updateReservationHomeSubCluster;

protected static final MetricsInfo RECORD_INFO =
info("ZKFederationStateStoreOpDurations", "Durations of ZKFederationStateStore calls");

Expand Down Expand Up @@ -152,4 +167,24 @@ public void addSetPolicyConfigurationDuration(long startTime, long endTime) {
public void addGetPoliciesConfigurationsDuration(long startTime, long endTime) {
getPoliciesConfigurations.add(endTime - startTime);
}

public void addReservationHomeSubClusterDuration(long startTime, long endTime) {
addReservationHomeSubCluster.add(endTime - startTime);
}

public void addGetReservationHomeSubClusterDuration(long startTime, long endTime) {
getReservationHomeSubCluster.add(endTime - startTime);
}

public void addGetReservationsHomeSubClusterDuration(long startTime, long endTime) {
getReservationsHomeSubCluster.add(endTime - startTime);
}

public void addDeleteReservationHomeSubClusterDuration(long startTime, long endTime) {
deleteReservationHomeSubCluster.add(endTime - startTime);
}

public void addUpdateReservationHomeSubClusterDuration(long startTime, long endTime) {
updateReservationHomeSubCluster.add(endTime - startTime);
}
}
25 changes: 17 additions & 8 deletions ...va/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import java.util.TimeZone;

import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.curator.ZKCuratorManager;
import org.apache.hadoop.yarn.api.records.ApplicationId;
Expand Down Expand Up @@ -718,8 +717,8 @@ private SubClusterId getReservation(final ReservationId reservationId)
}
}
return subClusterId;
}
}

@VisibleForTesting
public ZKFederationStateStoreOpDurations getOpDurations() {
return opDurations;
Expand All @@ -729,6 +728,7 @@ public ZKFederationStateStoreOpDurations getOpDurations() {
public AddReservationHomeSubClusterResponse addReservationHomeSubCluster(
AddReservationHomeSubClusterRequest request) throws YarnException {

long start = clock.getTime();
FederationReservationHomeSubClusterStoreInputValidator.validate(request);
ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster();
ReservationId reservationId = reservationHomeSubCluster.getReservationId();
Expand All @@ -749,14 +749,16 @@ public AddReservationHomeSubClusterResponse addReservationHomeSubCluster(
String errMsg = "Cannot check app home subcluster for " + reservationId;
FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
}

long end = clock.getTime();
opDurations.addReservationHomeSubClusterDuration(start, end);
return AddReservationHomeSubClusterResponse.newInstance(homeSubCluster);
}

@Override
public GetReservationHomeSubClusterResponse getReservationHomeSubCluster(
GetReservationHomeSubClusterRequest request) throws YarnException {

long start = clock.getTime();
FederationReservationHomeSubClusterStoreInputValidator.validate(request);
ReservationId reservationId = request.getReservationId();
SubClusterId homeSubCluster = getReservation(reservationId);
Expand All @@ -768,13 +770,15 @@ public GetReservationHomeSubClusterResponse getReservationHomeSubCluster(

ReservationHomeSubCluster reservationHomeSubCluster =
ReservationHomeSubCluster.newInstance(reservationId, homeSubCluster);

long end = clock.getTime();
opDurations.addGetReservationHomeSubClusterDuration(start, end);
return GetReservationHomeSubClusterResponse.newInstance(reservationHomeSubCluster);
}

@Override
public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster(
GetReservationsHomeSubClusterRequest request) throws YarnException {
long start = clock.getTime();
List<ReservationHomeSubCluster> result = new ArrayList<>();

try {
Expand All @@ -789,14 +793,15 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster(
String errMsg = "Cannot get apps: " + e.getMessage();
FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
}

long end = clock.getTime();
opDurations.addGetReservationsHomeSubClusterDuration(start, end);
return GetReservationsHomeSubClusterResponse.newInstance(result);
}

@Override
public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster(
DeleteReservationHomeSubClusterRequest request) throws YarnException {

long start = clock.getTime();
FederationReservationHomeSubClusterStoreInputValidator.validate(request);
ReservationId reservationId = request.getReservationId();
String reservationZNode = getNodePath(reservationsZNode, reservationId.toString());
Expand All @@ -820,14 +825,16 @@ public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster(
String errMsg = "Cannot delete reservation: " + e.getMessage();
FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
}

long end = clock.getTime();
opDurations.addDeleteReservationHomeSubClusterDuration(start, end);
return DeleteReservationHomeSubClusterResponse.newInstance();
}

@Override
public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
UpdateReservationHomeSubClusterRequest request) throws YarnException {

long start = clock.getTime();
FederationReservationHomeSubClusterStoreInputValidator.validate(request);
ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster();
ReservationId reservationId = reservationHomeSubCluster.getReservationId();
Expand All @@ -840,6 +847,8 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(

SubClusterId newSubClusterId = reservationHomeSubCluster.getHomeSubCluster();
putReservation(reservationId, newSubClusterId, true);
long end = clock.getTime();
opDurations.addUpdateReservationHomeSubClusterDuration(start, end);
return UpdateReservationHomeSubClusterResponse.newInstance();
}
}
4 changes: 2 additions & 2 deletions ...ava/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -664,7 +664,7 @@ public void testAddReservationHomeSubClusterReservationAlreadyExists() throws Ex
AddReservationHomeSubClusterRequest request2 =
AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster2);
AddReservationHomeSubClusterResponse response =
stateStore.addReservationHomeSubCluster(request2);
stateStore.addReservationHomeSubCluster(request2);

Assert.assertNotNull(response);
Assert.assertEquals(subClusterId1, response.getHomeSubCluster());
Expand All @@ -673,7 +673,7 @@ public void testAddReservationHomeSubClusterReservationAlreadyExists() throws Ex

@Test
public void testAddReservationHomeSubClusterAppAlreadyExistsInTheSameSC()
throws Exception {
throws Exception {

ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
Expand Down
15 changes: 15 additions & 0 deletions ...rg/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,11 @@ public void testMetricsInited() throws Exception {
zkStateStoreOpDurations.addGetPolicyConfigurationDuration(start, end);
zkStateStoreOpDurations.addSetPolicyConfigurationDuration(start, end);
zkStateStoreOpDurations.addGetPoliciesConfigurationsDuration(start, end);
zkStateStoreOpDurations.addReservationHomeSubClusterDuration(start, end);
zkStateStoreOpDurations.addGetReservationHomeSubClusterDuration(start, end);
zkStateStoreOpDurations.addGetReservationsHomeSubClusterDuration(start, end);
zkStateStoreOpDurations.addDeleteReservationHomeSubClusterDuration(start, end);
zkStateStoreOpDurations.addUpdateReservationHomeSubClusterDuration(start, end);

zkStateStoreOpDurations.getMetrics(collector, true);
assertEquals("Incorrect number of perf metrics", 1, collector.getRecords().size());
Expand All @@ -137,6 +142,11 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record, "GetPolicyConfigurationAvgTime", expectAvgTime);
MetricsRecords.assertMetric(record, "SetPolicyConfigurationAvgTime", expectAvgTime);
MetricsRecords.assertMetric(record, "GetPoliciesConfigurationsAvgTime", expectAvgTime);
MetricsRecords.assertMetric(record, "AddReservationHomeSubClusterAvgTime", expectAvgTime);
MetricsRecords.assertMetric(record, "GetReservationHomeSubClusterAvgTime", expectAvgTime);
MetricsRecords.assertMetric(record, "GetReservationsHomeSubClusterAvgTime", expectAvgTime);
MetricsRecords.assertMetric(record, "DeleteReservationHomeSubClusterAvgTime", expectAvgTime);
MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterAvgTime", expectAvgTime);

long expectOps = 1;
MetricsRecords.assertMetric(record, "AddAppHomeSubClusterNumOps", expectOps);
Expand All @@ -152,5 +162,10 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record, "GetPolicyConfigurationNumOps", expectOps);
MetricsRecords.assertMetric(record, "SetPolicyConfigurationNumOps", expectOps);
MetricsRecords.assertMetric(record, "GetPoliciesConfigurationsNumOps", expectOps);
MetricsRecords.assertMetric(record, "AddReservationHomeSubClusterNumOps", expectOps);
MetricsRecords.assertMetric(record, "GetReservationHomeSubClusterNumOps", expectOps);
MetricsRecords.assertMetric(record, "GetReservationsHomeSubClusterNumOps", expectOps);
MetricsRecords.assertMetric(record, "DeleteReservationHomeSubClusterNumOps", expectOps);
MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps", expectOps);
}
}

0 comments on commit 4c27aad

Please sign in to comment.