diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java index d1d971f3f4b..fd7b18bb681 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java @@ -106,11 +106,7 @@ OMClientResponse getResponse() { private final Daemon daemon; private final OMMetadataManager omMetadataManager; - private final AtomicLong flushedTransactionCount = new AtomicLong(0); - private final AtomicLong flushIterations = new AtomicLong(0); private final AtomicBoolean isRunning = new AtomicBoolean(false); - private final OzoneManagerDoubleBufferMetrics ozoneManagerDoubleBufferMetrics; - private long maxFlushedTransactionsInOneIteration; private final Consumer updateLastAppliedIndex; private final boolean isRatisEnabled; @@ -189,6 +185,13 @@ public OzoneManagerDoubleBuffer build() { } } + private final OzoneManagerDoubleBufferMetrics metrics = OzoneManagerDoubleBufferMetrics.create(); + + /** Accumulative count (for testing and debug only). */ + private final AtomicLong flushedTransactionCount = new AtomicLong(); + /** The number of flush iterations (for testing and debug only). */ + private final AtomicLong flushIterations = new AtomicLong(); + @SuppressWarnings("checkstyle:parameternumber") private OzoneManagerDoubleBuffer(OMMetadataManager omMetadataManager, Consumer updateLastAppliedIndex, @@ -203,8 +206,6 @@ private OzoneManagerDoubleBuffer(OMMetadataManager omMetadataManager, this.unFlushedTransactions = new Semaphore(maxUnFlushedTransactions); this.omMetadataManager = omMetadataManager; this.updateLastAppliedIndex = updateLastAppliedIndex; - this.ozoneManagerDoubleBufferMetrics = - OzoneManagerDoubleBufferMetrics.create(); this.flushNotifier = flushNotifier; isRunning.set(true); // Daemon thread which runs in background and flushes transactions to DB. @@ -354,8 +355,7 @@ private void flushBatch(Queue buffer) throws IOException { () -> omMetadataManager.getStore() .commitBatchOperation(batchOperation)); - ozoneManagerDoubleBufferMetrics.updateFlushTime( - Time.monotonicNow() - startTime); + metrics.updateFlushTime(Time.monotonicNow() - startTime); } // Complete futures first and then do other things. @@ -367,14 +367,10 @@ private void flushBatch(Queue buffer) throws IOException { .forEach(f -> f.complete(null)); } - flushedTransactionCount.addAndGet(flushedTransactionsSize); - flushIterations.incrementAndGet(); - - if (LOG.isDebugEnabled()) { - LOG.debug("Sync iteration {} flushed transactions in this iteration {}", - flushIterations.get(), - flushedTransactionsSize); - } + final long accumulativeCount = flushedTransactionCount.addAndGet(flushedTransactionsSize); + final long flushedIterations = flushIterations.incrementAndGet(); + LOG.debug("Sync iteration: {}, size in this iteration: {}, accumulative count: {}", + flushedIterations, flushedTransactionsSize, accumulativeCount); // Clean up committed transactions. cleanupCache(cleanupEpochs); @@ -386,7 +382,7 @@ private void flushBatch(Queue buffer) throws IOException { updateLastAppliedIndex.accept(lastTransaction); // set metrics. - updateMetrics(flushedTransactionsSize); + metrics.updateFlush(flushedTransactionsSize); } private String addToBatch(Queue buffer, BatchOperation batchOperation) { @@ -492,25 +488,6 @@ private void cleanupCache(Map> cleanupEpochs) { private synchronized void clearReadyBuffer() { readyBuffer.clear(); } - /** - * Update OzoneManagerDoubleBuffer metrics values. - */ - private void updateMetrics(int flushedTransactionsSize) { - ozoneManagerDoubleBufferMetrics.incrTotalNumOfFlushOperations(); - ozoneManagerDoubleBufferMetrics.incrTotalSizeOfFlushedTransactions( - flushedTransactionsSize); - ozoneManagerDoubleBufferMetrics.setAvgFlushTransactionsInOneIteration( - (float) ozoneManagerDoubleBufferMetrics - .getTotalNumOfFlushedTransactions() / - ozoneManagerDoubleBufferMetrics.getTotalNumOfFlushOperations()); - if (maxFlushedTransactionsInOneIteration < flushedTransactionsSize) { - maxFlushedTransactionsInOneIteration = flushedTransactionsSize; - ozoneManagerDoubleBufferMetrics - .setMaxNumberOfTransactionsFlushedInOneIteration( - flushedTransactionsSize); - } - ozoneManagerDoubleBufferMetrics.updateQueueSize(flushedTransactionsSize); - } /** * Stop OM DoubleBuffer flush thread. @@ -520,7 +497,7 @@ private void updateMetrics(int flushedTransactionsSize) { @SuppressWarnings("squid:S2142") public void stop() { stopDaemon(); - ozoneManagerDoubleBufferMetrics.unRegister(); + metrics.unRegister(); } @VisibleForTesting @@ -553,22 +530,6 @@ private void terminate(Throwable t, int status, OMResponse omResponse) { ExitUtils.terminate(status, message.toString(), t, LOG); } - /** - * Returns the flushed transaction count to OM DB. - * @return flushedTransactionCount - */ - public long getFlushedTransactionCount() { - return flushedTransactionCount.get(); - } - - /** - * Returns total number of flush iterations run by sync thread. - * @return flushIterations - */ - public long getFlushIterations() { - return flushIterations.get(); - } - /** * Add OmResponseBufferEntry to buffer. */ @@ -623,8 +584,20 @@ private synchronized void swapCurrentAndReadyBuffer() { } @VisibleForTesting - public OzoneManagerDoubleBufferMetrics getOzoneManagerDoubleBufferMetrics() { - return ozoneManagerDoubleBufferMetrics; + OzoneManagerDoubleBufferMetrics getMetrics() { + return metrics; + } + + /** @return the flushed transaction count to OM DB. */ + @VisibleForTesting + long getFlushedTransactionCountForTesting() { + return flushedTransactionCount.get(); + } + + /** @return total number of flush iterations run by sync thread. */ + @VisibleForTesting + long getFlushIterationsForTesting() { + return flushIterations.get(); } @VisibleForTesting diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java index 45767ec7d07..62e320e1e06 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java @@ -20,10 +20,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.protobuf.ServiceException; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -37,16 +35,13 @@ import org.apache.hadoop.ozone.om.OzoneManagerPrepareState; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; -import org.apache.hadoop.ozone.om.ratis.metrics.OzoneManagerStateMachineMetrics; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.lock.OMLockDetails; import org.apache.hadoop.ozone.om.response.DummyOMClientResponse; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos - .OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos - .OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocolPB.OzoneManagerRequestHandler; import org.apache.hadoop.ozone.protocolPB.RequestHandler; import org.apache.hadoop.security.UserGroupInformation; @@ -90,7 +85,6 @@ public class OzoneManagerStateMachine extends BaseStateMachine { LoggerFactory.getLogger(OzoneManagerStateMachine.class); private final SimpleStateMachineStorage storage = new SimpleStateMachineStorage(); - private final OzoneManagerRatisServer omRatisServer; private final OzoneManager ozoneManager; private RequestHandler handler; private RaftGroupId raftGroupId; @@ -106,14 +100,10 @@ public class OzoneManagerStateMachine extends BaseStateMachine { /** The last index skipped by {@link #notifyTermIndexUpdated(long, long)}. */ private volatile long lastSkippedIndex = RaftLog.INVALID_LOG_INDEX; - private OzoneManagerStateMachineMetrics metrics; - - public OzoneManagerStateMachine(OzoneManagerRatisServer ratisServer, boolean isTracingEnabled) throws IOException { - this.omRatisServer = ratisServer; this.isTracingEnabled = isTracingEnabled; - this.ozoneManager = omRatisServer.getOzoneManager(); + this.ozoneManager = ratisServer.getOzoneManager(); loadSnapshotInfoFromDB(); this.threadPrefix = ozoneManager.getThreadNamePrefix(); @@ -132,7 +122,6 @@ public OzoneManagerStateMachine(OzoneManagerRatisServer ratisServer, .setNameFormat(threadPrefix + "InstallSnapshotThread").build(); this.installSnapshotExecutor = HadoopExecutors.newSingleThreadExecutor(installSnapshotThreadFactory); - this.metrics = OzoneManagerStateMachineMetrics.create(); } /** @@ -270,7 +259,14 @@ public TransactionContext startTransaction( ctxt.setException(ioe); return ctxt; } - return handleStartTransactionRequests(raftClientRequest, omRequest); + + return TransactionContext.newBuilder() + .setClientRequest(raftClientRequest) + .setStateMachine(this) + .setServerRole(RaftProtos.RaftPeerRole.LEADER) + .setLogData(raftClientRequest.getMessage().getContent()) + .setStateMachineContext(omRequest) + .build(); } @Override @@ -499,18 +495,9 @@ public CompletableFuture notifyInstallSnapshotFromLeader( LOG.info("Received install snapshot notification from OM leader: {} with " + "term index: {}", leaderNodeId, firstTermIndexInLog); - CompletableFuture future = CompletableFuture.supplyAsync( + return CompletableFuture.supplyAsync( () -> ozoneManager.installSnapshotFromLeader(leaderNodeId), installSnapshotExecutor); - return future; - } - - /** - * Notifies the state machine that the raft peer is no longer leader. - */ - @Override - public void notifyNotLeader(Collection pendingEntries) - throws IOException { } @Override @@ -530,29 +517,10 @@ public void close() { } } - /** - * Handle the RaftClientRequest and return TransactionContext object. - * @param raftClientRequest - * @param omRequest - * @return TransactionContext - */ - private TransactionContext handleStartTransactionRequests( - RaftClientRequest raftClientRequest, OMRequest omRequest) { - - return TransactionContext.newBuilder() - .setClientRequest(raftClientRequest) - .setStateMachine(this) - .setServerRole(RaftProtos.RaftPeerRole.LEADER) - .setLogData(raftClientRequest.getMessage().getContent()) - .setStateMachineContext(omRequest) - .build(); - } - /** * Submits write request to OM and returns the response Message. * @param request OMRequest * @return response from OM - * @throws ServiceException */ private OMResponse runCommand(OMRequest request, TermIndex termIndex) { try { @@ -635,23 +603,10 @@ public OzoneManagerRequestHandler getHandler() { return (OzoneManagerRequestHandler) this.handler; } - @VisibleForTesting - public void setRaftGroupId(RaftGroupId raftGroupId) { - this.raftGroupId = raftGroupId; - } - - @VisibleForTesting - public OzoneManagerStateMachineMetrics getMetrics() { - return this.metrics; - } - public void stop() { ozoneManagerDoubleBuffer.stop(); HadoopExecutors.shutdown(executorService, LOG, 5, TimeUnit.SECONDS); HadoopExecutors.shutdown(installSnapshotExecutor, LOG, 5, TimeUnit.SECONDS); - if (metrics != null) { - metrics.unRegister(); - } } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerDoubleBufferMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerDoubleBufferMetrics.java index f77eda081a7..351f1852893 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerDoubleBufferMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerDoubleBufferMetrics.java @@ -131,6 +131,17 @@ public void updateQueueSize(long size) { queueSize.add(size); } + public void updateFlush(int flushedTransactionsInOneIteration) { + incrTotalNumOfFlushOperations(); + incrTotalSizeOfFlushedTransactions(flushedTransactionsInOneIteration); + setAvgFlushTransactionsInOneIteration(getTotalNumOfFlushedTransactions() / (float)getTotalNumOfFlushOperations()); + final long max = getMaxNumberOfTransactionsFlushedInOneIteration(); + if (flushedTransactionsInOneIteration > max) { + maxNumberOfTransactionsFlushedInOneIteration.incr(flushedTransactionsInOneIteration - max); + } + updateQueueSize(flushedTransactionsInOneIteration); + } + @VisibleForTesting public MutableStat getQueueSize() { return queueSize; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerStateMachineMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerStateMachineMetrics.java deleted file mode 100644 index 51d26ef7ac0..00000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerStateMachineMetrics.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.ratis.metrics; - -import org.apache.hadoop.metrics2.MetricsCollector; -import org.apache.hadoop.metrics2.MetricsRecordBuilder; -import org.apache.hadoop.metrics2.MetricsSource; -import org.apache.hadoop.metrics2.MetricsSystem; -import org.apache.hadoop.metrics2.annotation.Metric; -import org.apache.hadoop.metrics2.annotation.Metrics; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.metrics2.lib.MetricsRegistry; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.ozone.OzoneConsts; - -/** - * Class which maintains metrics related to OzoneManager state machine. - */ -@Metrics(about = "OzoneManagerStateMachine Metrics", context = OzoneConsts.OZONE) -public final class OzoneManagerStateMachineMetrics implements MetricsSource { - - private static final String SOURCE_NAME = - OzoneManagerStateMachineMetrics.class.getSimpleName(); - private MetricsRegistry registry; - private static OzoneManagerStateMachineMetrics instance; - - @Metric(about = "Number of apply transactions in applyTransactionMap.") - private MutableCounterLong applyTransactionMapSize; - - @Metric(about = "Number of ratis transactions in ratisTransactionMap.") - private MutableCounterLong ratisTransactionMapSize; - - private OzoneManagerStateMachineMetrics() { - registry = new MetricsRegistry(SOURCE_NAME); - } - - public static synchronized OzoneManagerStateMachineMetrics create() { - if (instance != null) { - return instance; - } else { - MetricsSystem ms = DefaultMetricsSystem.instance(); - OzoneManagerStateMachineMetrics metrics = new OzoneManagerStateMachineMetrics(); - instance = ms.register(SOURCE_NAME, "OzoneManager StateMachine Metrics", - metrics); - return instance; - } - } - - public void updateApplyTransactionMapSize(long size) { - this.applyTransactionMapSize.incr( - Math.negateExact(applyTransactionMapSize.value()) + size); - } - - public void updateRatisTransactionMapSize(long size) { - this.ratisTransactionMapSize.incr( - Math.negateExact(ratisTransactionMapSize.value()) + size); - } - - public void unRegister() { - MetricsSystem ms = DefaultMetricsSystem.instance(); - ms.unregisterSource(SOURCE_NAME); - } - - public void getMetrics(MetricsCollector collector, boolean all) { - MetricsRecordBuilder rb = collector.addRecord(SOURCE_NAME); - - applyTransactionMapSize.snapshot(rb, all); - ratisTransactionMapSize.snapshot(rb, all); - rb.endRecord(); - } -} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java index 202234a0d43..21205c4dc33 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java @@ -81,16 +81,13 @@ class TestOzoneManagerDoubleBuffer { private OzoneManagerDoubleBuffer doubleBuffer; private OzoneManager ozoneManager; - private OmMetadataManagerImpl omMetadataManager; private S3SecretLockedManager secretManager; - private CreateSnapshotResponse snapshotResponse1 = - mock(CreateSnapshotResponse.class); - private CreateSnapshotResponse snapshotResponse2 = - mock(CreateSnapshotResponse.class); - private OMResponse omKeyResponse = mock(OMResponse.class); - private OMResponse omBucketResponse = mock(OMResponse.class); - private OMResponse omSnapshotResponse1 = mock(OMResponse.class); - private OMResponse omSnapshotResponse2 = mock(OMResponse.class); + private final CreateSnapshotResponse snapshotResponse1 = mock(CreateSnapshotResponse.class); + private final CreateSnapshotResponse snapshotResponse2 = mock(CreateSnapshotResponse.class); + private final OMResponse omKeyResponse = mock(OMResponse.class); + private final OMResponse omBucketResponse = mock(OMResponse.class); + private final OMResponse omSnapshotResponse1 = mock(OMResponse.class); + private final OMResponse omSnapshotResponse2 = mock(OMResponse.class); private static OMClientResponse omKeyCreateResponse = mock(OMKeyCreateResponse.class); private static OMClientResponse omBucketCreateResponse = @@ -104,10 +101,6 @@ class TestOzoneManagerDoubleBuffer { private OzoneManagerDoubleBuffer.FlushNotifier flushNotifier; private OzoneManagerDoubleBuffer.FlushNotifier spyFlushNotifier; - private static String userPrincipalId1 = "alice@EXAMPLE.COM"; - private static String userPrincipalId2 = "messi@EXAMPLE.COM"; - private static String userPrincipalId3 = "ronaldo@EXAMPLE.COM"; - @BeforeEach public void setup() throws IOException { OMMetrics omMetrics = OMMetrics.create(); @@ -117,8 +110,8 @@ public void setup() throws IOException { ozoneManager = mock(OzoneManager.class); when(ozoneManager.getMetrics()).thenReturn(omMetrics); - omMetadataManager = - new OmMetadataManagerImpl(ozoneConfiguration, ozoneManager); + + final OmMetadataManagerImpl omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, ozoneManager); when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); when(ozoneManager.getMaxUserVolumeCount()).thenReturn(10L); AuditLogger auditLogger = mock(AuditLogger.class); @@ -244,12 +237,10 @@ public void testOzoneManagerDoubleBuffer( // Flush the current buffer. doubleBuffer.flushCurrentBuffer(); - assertEquals(expectedFlushCounts, doubleBuffer.getFlushIterations()); - assertEquals(expectedFlushedTransactionCount, - doubleBuffer.getFlushedTransactionCount()); + assertEquals(expectedFlushCounts, doubleBuffer.getFlushIterationsForTesting()); + assertEquals(expectedFlushedTransactionCount, doubleBuffer.getFlushedTransactionCountForTesting()); - OzoneManagerDoubleBufferMetrics bufferMetrics = - doubleBuffer.getOzoneManagerDoubleBufferMetrics(); + final OzoneManagerDoubleBufferMetrics bufferMetrics = doubleBuffer.getMetrics(); assertEquals(expectedFlushCountsInMetric, bufferMetrics.getTotalNumOfFlushOperations()); @@ -259,6 +250,9 @@ public void testOzoneManagerDoubleBuffer( bufferMetrics.getMaxNumberOfTransactionsFlushedInOneIteration()); assertEquals(expectedAvgFlushTransactionsInMetric, bufferMetrics.getAvgFlushTransactionsInOneIteration(), 0.001); + + // reset max + bufferMetrics.setMaxNumberOfTransactionsFlushedInOneIteration(0); } @Test @@ -318,6 +312,10 @@ public void testAwaitFlush() throws Exception { @Test public void testS3SecretCacheSizePostDoubleBufferFlush() throws IOException { + final String userPrincipalId1 = "alice@EXAMPLE.COM"; + final String userPrincipalId2 = "messi@EXAMPLE.COM"; + final String userPrincipalId3 = "ronaldo@EXAMPLE.COM"; + // Create a secret for "alice". // This effectively makes alice an S3 admin. KerberosName.setRuleMechanism(DEFAULT_MECHANISM); @@ -338,9 +336,9 @@ public void testS3SecretCacheSizePostDoubleBufferFlush() throws IOException { doubleBuffer.stopDaemon(); // Create 3 secrets and store them in the cache and double buffer. - processSuccessSecretRequest(userPrincipalId1, 1, true); - processSuccessSecretRequest(userPrincipalId2, 2, true); - processSuccessSecretRequest(userPrincipalId3, 3, true); + processSuccessSecretRequest(userPrincipalId1, 1); + processSuccessSecretRequest(userPrincipalId2, 2); + processSuccessSecretRequest(userPrincipalId3, 3); S3SecretCache cache = secretManager.cache(); // Check if all the three secrets are cached. @@ -357,8 +355,7 @@ public void testS3SecretCacheSizePostDoubleBufferFlush() throws IOException { assertNull(cache.get(userPrincipalId1)); } finally { // cleanup metrics - OzoneManagerDoubleBufferMetrics metrics = - doubleBuffer.getOzoneManagerDoubleBufferMetrics(); + final OzoneManagerDoubleBufferMetrics metrics = doubleBuffer.getMetrics(); metrics.setMaxNumberOfTransactionsFlushedInOneIteration(0); metrics.setAvgFlushTransactionsInOneIteration(0); metrics.incrTotalSizeOfFlushedTransactions( @@ -368,10 +365,7 @@ public void testS3SecretCacheSizePostDoubleBufferFlush() throws IOException { } } - private void processSuccessSecretRequest( - String userPrincipalId, - int txLogIndex, - boolean shouldHaveResponse) throws IOException { + private void processSuccessSecretRequest(String userPrincipalId, int txLogIndex) throws IOException { S3GetSecretRequest s3GetSecretRequest = new S3GetSecretRequest( new S3GetSecretRequest( diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java index 635f86f3ab5..ee2e9043a36 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java @@ -96,8 +96,7 @@ public void stop() { public void testDoubleBufferWithDummyResponse() throws Exception { String volumeName = UUID.randomUUID().toString(); int bucketCount = 100; - OzoneManagerDoubleBufferMetrics metrics = - doubleBuffer.getOzoneManagerDoubleBufferMetrics(); + final OzoneManagerDoubleBufferMetrics metrics = doubleBuffer.getMetrics(); // As we have not flushed/added any transactions, all metrics should have // value zero. @@ -113,11 +112,11 @@ public void testDoubleBufferWithDummyResponse() throws Exception { 100, 60000); assertThat(metrics.getTotalNumOfFlushOperations()).isGreaterThan(0); - assertEquals(bucketCount, doubleBuffer.getFlushedTransactionCount()); + assertEquals(bucketCount, doubleBuffer.getFlushedTransactionCountForTesting()); assertThat(metrics.getMaxNumberOfTransactionsFlushedInOneIteration()).isGreaterThan(0); assertEquals(bucketCount, omMetadataManager.countRowsInTable( omMetadataManager.getBucketTable())); - assertThat(doubleBuffer.getFlushIterations()).isGreaterThan(0); + assertThat(doubleBuffer.getFlushIterationsForTesting()).isGreaterThan(0); assertThat(metrics.getFlushTime().lastStat().numSamples()).isGreaterThan(0); assertThat(metrics.getAvgFlushTransactionsInOneIteration()).isGreaterThan(0); assertEquals(bucketCount, (long) metrics.getQueueSize().lastStat().total()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java index 7178868dcf3..006777141a6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithOMResponse.java @@ -173,9 +173,9 @@ public void testDoubleBufferWithMixOfTransactions() throws Exception { final int deleteCount = 5; // We are doing +1 for volume transaction. - GenericTestUtils.waitFor(() -> - doubleBuffer.getFlushedTransactionCount() == - (bucketCount + deleteCount + 1), 100, 120000); + GenericTestUtils.waitFor( + () -> doubleBuffer.getFlushedTransactionCountForTesting() == bucketCount + deleteCount + 1, + 100, 120000); assertEquals(1, omMetadataManager.countRowsInTable( omMetadataManager.getVolumeTable())); @@ -251,8 +251,9 @@ public void testDoubleBufferWithMixOfTransactionsParallel() throws Exception { final int deleteCount = 10; // We are doing +1 for volume transaction. - GenericTestUtils.waitFor(() -> doubleBuffer.getFlushedTransactionCount() - == (bucketCount + deleteCount + 2), 100, 120000); + GenericTestUtils.waitFor( + () -> doubleBuffer.getFlushedTransactionCountForTesting() == bucketCount + deleteCount + 2, + 100, 120000); assertEquals(2, omMetadataManager.countRowsInTable( omMetadataManager.getVolumeTable())); @@ -398,8 +399,8 @@ private void testDoubleBuffer(int volumeCount, int bucketsPerVolume) int expectedBuckets = bucketsPerVolume * volumeCount; long expectedTransactions = volumeCount + expectedBuckets; - GenericTestUtils.waitFor(() -> - expectedTransactions == doubleBuffer.getFlushedTransactionCount(), + GenericTestUtils.waitFor( + () -> expectedTransactions == doubleBuffer.getFlushedTransactionCountForTesting(), 100, volumeCount * 500); GenericTestUtils.waitFor(() -> @@ -411,7 +412,7 @@ private void testDoubleBuffer(int volumeCount, int bucketsPerVolume) assertRowCount(expectedBuckets, omMetadataManager.getBucketTable()), 300, volumeCount * 300); - assertThat(doubleBuffer.getFlushIterations()).isGreaterThan(0); + assertThat(doubleBuffer.getFlushIterationsForTesting()).isGreaterThan(0); } private boolean assertRowCount(int expected, Table table) {