From 4c27aaddba1465b95686c80b7751c0cfe5bb8b33 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 23 Aug 2022 13:29:10 +0800 Subject: [PATCH] YARN-11272. Add OpDurations For Reservation --- .../ZKFederationStateStoreOpDurations.java | 35 +++++++++++++++++++ .../impl/ZookeeperFederationStateStore.java | 25 ++++++++----- .../impl/FederationStateStoreBaseTest.java | 4 +-- .../TestZookeeperFederationStateStore.java | 15 ++++++++ 4 files changed, 69 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java index 6ce5e2ef46185..113e4850a5709 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java @@ -74,6 +74,21 @@ public final class ZKFederationStateStoreOpDurations implements MetricsSource { @Metric("Duration for a get PolicyConfigurations call") private MutableRate getPoliciesConfigurations; + @Metric("Duration for a add reservation homeSubCluster call") + private MutableRate addReservationHomeSubCluster; + + @Metric("Duration for a get reservation homeSubCluster call") + private MutableRate getReservationHomeSubCluster; + + @Metric("Duration for a get reservations homeSubCluster call") + private MutableRate getReservationsHomeSubCluster; + + @Metric("Duration for a delete reservation homeSubCluster call") + private MutableRate deleteReservationHomeSubCluster; + + @Metric("Duration for a update reservation homeSubCluster call") + private MutableRate updateReservationHomeSubCluster; + protected static final MetricsInfo RECORD_INFO = info("ZKFederationStateStoreOpDurations", "Durations of ZKFederationStateStore calls"); @@ -152,4 +167,24 @@ public void addSetPolicyConfigurationDuration(long startTime, long endTime) { public void addGetPoliciesConfigurationsDuration(long startTime, long endTime) { getPoliciesConfigurations.add(endTime - startTime); } + + public void addReservationHomeSubClusterDuration(long startTime, long endTime) { + addReservationHomeSubCluster.add(endTime - startTime); + } + + public void addGetReservationHomeSubClusterDuration(long startTime, long endTime) { + getReservationHomeSubCluster.add(endTime - startTime); + } + + public void addGetReservationsHomeSubClusterDuration(long startTime, long endTime) { + getReservationsHomeSubCluster.add(endTime - startTime); + } + + public void addDeleteReservationHomeSubClusterDuration(long startTime, long endTime) { + deleteReservationHomeSubCluster.add(endTime - startTime); + } + + public void addUpdateReservationHomeSubClusterDuration(long startTime, long endTime) { + updateReservationHomeSubCluster.add(endTime - startTime); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index a68c52bdbe34d..d710dacd54087 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -26,7 +26,6 @@ import java.util.TimeZone; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.commons.lang3.NotImplementedException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.curator.ZKCuratorManager; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -718,8 +717,8 @@ private SubClusterId getReservation(final ReservationId reservationId) } } return subClusterId; - } - + } + @VisibleForTesting public ZKFederationStateStoreOpDurations getOpDurations() { return opDurations; @@ -729,6 +728,7 @@ public ZKFederationStateStoreOpDurations getOpDurations() { public AddReservationHomeSubClusterResponse addReservationHomeSubCluster( AddReservationHomeSubClusterRequest request) throws YarnException { + long start = clock.getTime(); FederationReservationHomeSubClusterStoreInputValidator.validate(request); ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster(); ReservationId reservationId = reservationHomeSubCluster.getReservationId(); @@ -749,7 +749,8 @@ public AddReservationHomeSubClusterResponse addReservationHomeSubCluster( String errMsg = "Cannot check app home subcluster for " + reservationId; FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } - + long end = clock.getTime(); + opDurations.addReservationHomeSubClusterDuration(start, end); return AddReservationHomeSubClusterResponse.newInstance(homeSubCluster); } @@ -757,6 +758,7 @@ public AddReservationHomeSubClusterResponse addReservationHomeSubCluster( public GetReservationHomeSubClusterResponse getReservationHomeSubCluster( GetReservationHomeSubClusterRequest request) throws YarnException { + long start = clock.getTime(); FederationReservationHomeSubClusterStoreInputValidator.validate(request); ReservationId reservationId = request.getReservationId(); SubClusterId homeSubCluster = getReservation(reservationId); @@ -768,13 +770,15 @@ public GetReservationHomeSubClusterResponse getReservationHomeSubCluster( ReservationHomeSubCluster reservationHomeSubCluster = ReservationHomeSubCluster.newInstance(reservationId, homeSubCluster); - + long end = clock.getTime(); + opDurations.addGetReservationHomeSubClusterDuration(start, end); return GetReservationHomeSubClusterResponse.newInstance(reservationHomeSubCluster); } @Override public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( GetReservationsHomeSubClusterRequest request) throws YarnException { + long start = clock.getTime(); List result = new ArrayList<>(); try { @@ -789,14 +793,15 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( String errMsg = "Cannot get apps: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } - + long end = clock.getTime(); + opDurations.addGetReservationsHomeSubClusterDuration(start, end); return GetReservationsHomeSubClusterResponse.newInstance(result); } @Override public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( DeleteReservationHomeSubClusterRequest request) throws YarnException { - + long start = clock.getTime(); FederationReservationHomeSubClusterStoreInputValidator.validate(request); ReservationId reservationId = request.getReservationId(); String reservationZNode = getNodePath(reservationsZNode, reservationId.toString()); @@ -820,7 +825,8 @@ public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( String errMsg = "Cannot delete reservation: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } - + long end = clock.getTime(); + opDurations.addDeleteReservationHomeSubClusterDuration(start, end); return DeleteReservationHomeSubClusterResponse.newInstance(); } @@ -828,6 +834,7 @@ public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( UpdateReservationHomeSubClusterRequest request) throws YarnException { + long start = clock.getTime(); FederationReservationHomeSubClusterStoreInputValidator.validate(request); ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster(); ReservationId reservationId = reservationHomeSubCluster.getReservationId(); @@ -840,6 +847,8 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( SubClusterId newSubClusterId = reservationHomeSubCluster.getHomeSubCluster(); putReservation(reservationId, newSubClusterId, true); + long end = clock.getTime(); + opDurations.addUpdateReservationHomeSubClusterDuration(start, end); return UpdateReservationHomeSubClusterResponse.newInstance(); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java index c88638967e144..296e4846ea4c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java @@ -664,7 +664,7 @@ public void testAddReservationHomeSubClusterReservationAlreadyExists() throws Ex AddReservationHomeSubClusterRequest request2 = AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster2); AddReservationHomeSubClusterResponse response = - stateStore.addReservationHomeSubCluster(request2); + stateStore.addReservationHomeSubCluster(request2); Assert.assertNotNull(response); Assert.assertEquals(subClusterId1, response.getHomeSubCluster()); @@ -673,7 +673,7 @@ public void testAddReservationHomeSubClusterReservationAlreadyExists() throws Ex @Test public void testAddReservationHomeSubClusterAppAlreadyExistsInTheSameSC() - throws Exception { + throws Exception { ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); SubClusterId subClusterId1 = SubClusterId.newInstance("SC1"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java index 584f3355ff5d6..272394b6b285b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java @@ -115,6 +115,11 @@ public void testMetricsInited() throws Exception { zkStateStoreOpDurations.addGetPolicyConfigurationDuration(start, end); zkStateStoreOpDurations.addSetPolicyConfigurationDuration(start, end); zkStateStoreOpDurations.addGetPoliciesConfigurationsDuration(start, end); + zkStateStoreOpDurations.addReservationHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addGetReservationHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addGetReservationsHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addDeleteReservationHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addUpdateReservationHomeSubClusterDuration(start, end); zkStateStoreOpDurations.getMetrics(collector, true); assertEquals("Incorrect number of perf metrics", 1, collector.getRecords().size()); @@ -137,6 +142,11 @@ public void testMetricsInited() throws Exception { MetricsRecords.assertMetric(record, "GetPolicyConfigurationAvgTime", expectAvgTime); MetricsRecords.assertMetric(record, "SetPolicyConfigurationAvgTime", expectAvgTime); MetricsRecords.assertMetric(record, "GetPoliciesConfigurationsAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "AddReservationHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "GetReservationHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "GetReservationsHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "DeleteReservationHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterAvgTime", expectAvgTime); long expectOps = 1; MetricsRecords.assertMetric(record, "AddAppHomeSubClusterNumOps", expectOps); @@ -152,5 +162,10 @@ public void testMetricsInited() throws Exception { MetricsRecords.assertMetric(record, "GetPolicyConfigurationNumOps", expectOps); MetricsRecords.assertMetric(record, "SetPolicyConfigurationNumOps", expectOps); MetricsRecords.assertMetric(record, "GetPoliciesConfigurationsNumOps", expectOps); + MetricsRecords.assertMetric(record, "AddReservationHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "GetReservationHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "GetReservationsHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "DeleteReservationHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps", expectOps); } } \ No newline at end of file