HDDS-2609. Move plain unit tests out of integration tests (#260)

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/pipeline/package-info.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+/**
+ * Package info tests.
+ */
+package org.apache.hadoop.hdds.scm.pipeline;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java

@@ -32,6 +32,7 @@
 import org.apache.hadoop.ozone.client.io.KeyOutputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -685,11 +686,11 @@ public void testWriteWithExceedingMaxBufferLimit() throws Exception {
 
   private OzoneOutputStream createKey(String keyName, ReplicationType type,
       long size) throws Exception {
-    return ContainerTestHelper
+    return TestHelper
         .createKey(keyName, type, size, objectStore, volumeName, bucketName);
   }
   private void validateData(String keyName, byte[] data) throws Exception {
-    ContainerTestHelper
+    TestHelper
         .validateData(keyName, data, objectStore, volumeName, bucketName);
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStreamWithFailures.java

@@ -36,6 +36,7 @@
 import org.apache.hadoop.ozone.client.io.KeyOutputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
 import org.apache.ratis.protocol.GroupMismatchException;
 import org.apache.ratis.protocol.RaftRetryFailureException;
 import org.junit.After;
@@ -221,7 +222,7 @@ public void testWatchForCommitWithCloseContainerException()
         (XceiverClientRatis) blockOutputStream.getXceiverClient();
     Assert.assertEquals(3, raftClient.getCommitInfoMap().size());
     // Close the containers on the Datanode and write more data
-    ContainerTestHelper.waitForContainerClose(key, cluster);
+    TestHelper.waitForContainerClose(key, cluster);
     // 4 writeChunks = maxFlushSize + 2 putBlocks  will be discarded here
     // once exception is hit
     key.write(data1);
@@ -624,7 +625,7 @@ public void testFailureWithPrimeSizedData() throws Exception {
         (XceiverClientRatis) blockOutputStream.getXceiverClient();
     Assert.assertEquals(3, raftClient.getCommitInfoMap().size());
     // Close the containers on the Datanode and write more data
-    ContainerTestHelper.waitForContainerClose(key, cluster);
+    TestHelper.waitForContainerClose(key, cluster);
     key.write(data1);
 
     // As a part of handling the exception, 2 failed writeChunks  will be
@@ -745,7 +746,7 @@ public void testExceptionDuringClose() throws Exception {
         (XceiverClientRatis) blockOutputStream.getXceiverClient();
     Assert.assertEquals(3, raftClient.getCommitInfoMap().size());
     // Close the containers on the Datanode and write more data
-    ContainerTestHelper.waitForContainerClose(key, cluster);
+    TestHelper.waitForContainerClose(key, cluster);
     key.write(data1);
 
     // commitInfoMap will remain intact as there is no server failure
@@ -879,7 +880,7 @@ public void testWatchForCommitWithSingleNodeRatis() throws Exception {
         (XceiverClientRatis) blockOutputStream.getXceiverClient();
     Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
     // Close the containers on the Datanode and write more data
-    ContainerTestHelper.waitForContainerClose(key, cluster);
+    TestHelper.waitForContainerClose(key, cluster);
     // 4 writeChunks = maxFlushSize + 2 putBlocks  will be discarded here
     // once exception is hit
     key.write(data1);
@@ -1206,13 +1207,13 @@ private OzoneOutputStream createKey(String keyName, ReplicationType type,
 
   private OzoneOutputStream createKey(String keyName, ReplicationType type,
       long size, ReplicationFactor factor) throws Exception {
-    return ContainerTestHelper
+    return TestHelper
         .createKey(keyName, type, factor, size, objectStore, volumeName,
             bucketName);
   }
 
   private void validateData(String keyName, byte[] data) throws Exception {
-    ContainerTestHelper
+    TestHelper
         .validateData(keyName, data, objectStore, volumeName, bucketName);
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java

@@ -37,6 +37,7 @@
 import org.apache.hadoop.ozone.client.io.OzoneInputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -338,7 +339,7 @@ public void testMultiBlockWrites3() throws Exception {
 
   private void waitForContainerClose(OzoneOutputStream outputStream)
       throws Exception {
-    ContainerTestHelper
+    TestHelper
         .waitForContainerClose(outputStream, cluster);
   }
 
@@ -392,12 +393,12 @@ public void testDiscardPreallocatedBlocks() throws Exception {
 
   private OzoneOutputStream createKey(String keyName, ReplicationType type,
       long size) throws Exception {
-    return ContainerTestHelper
+    return TestHelper
         .createKey(keyName, type, size, objectStore, volumeName, bucketName);
   }
 
   private void validateData(String keyName, byte[] data) throws Exception {
-    ContainerTestHelper
+    TestHelper
         .validateData(keyName, data, objectStore, volumeName, bucketName);
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java

@@ -37,7 +37,7 @@
 import org.apache.hadoop.ozone.client.OzoneClientFactory;
 import org.apache.hadoop.ozone.client.io.KeyOutputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
 
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -213,7 +213,7 @@ public void testContainerReplication() throws Exception {
     }
     // This will try to read the data from the dn to which the container got
     // replicated after the container got closed.
-    ContainerTestHelper
+    TestHelper
         .validateData(keyName, testData, objectStore, volumeName, bucketName);
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java

@@ -30,7 +30,7 @@
 import org.apache.hadoop.ozone.client.OzoneClientFactory;
 import org.apache.hadoop.ozone.client.io.KeyOutputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
 import org.apache.hadoop.ozone.container.common.transport.server.ratis.ContainerStateMachine;
 import org.apache.hadoop.ozone.container.common.transport.server.ratis.RatisServerConfiguration;
 import org.apache.hadoop.ozone.om.OzoneManager;
@@ -160,7 +160,7 @@ public void testContainerStateMachineFailures() throws Exception {
   public void testRatisSnapshotRetention() throws Exception {
 
     ContainerStateMachine stateMachine =
-        (ContainerStateMachine) ContainerTestHelper.getStateMachine(cluster);
+        (ContainerStateMachine) TestHelper.getStateMachine(cluster);
     SimpleStateMachineStorage storage =
         (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
     Assert.assertNull(storage.findLatestSnapshot());
@@ -181,7 +181,7 @@ public void testRatisSnapshotRetention() throws Exception {
         conf.getObject(RatisServerConfiguration.class);
 
     stateMachine =
-        (ContainerStateMachine) ContainerTestHelper.getStateMachine(cluster);
+        (ContainerStateMachine) TestHelper.getStateMachine(cluster);
     storage = (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
     Path parentPath = storage.findLatestSnapshot().getFile().getPath();
     int numSnapshots = parentPath.getParent().toFile().listFiles().length;
@@ -200,12 +200,12 @@ public void testRatisSnapshotRetention() throws Exception {
       key.write(("ratis" + i).getBytes());
     }
     stateMachine =
-        (ContainerStateMachine) ContainerTestHelper.getStateMachine(cluster);
+        (ContainerStateMachine) TestHelper.getStateMachine(cluster);
     storage = (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
     parentPath = storage.findLatestSnapshot().getFile().getPath();
     numSnapshots = parentPath.getParent().toFile().listFiles().length;
     Assert.assertTrue(Math.abs(ratisServerConfiguration
         .getNumSnapshotsRetained() - numSnapshots) <= 1);
   }
 
-}
+}

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java

@@ -38,6 +38,7 @@
 import org.apache.hadoop.ozone.client.io.KeyOutputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
 import org.apache.hadoop.ozone.container.common.impl.ContainerData;
 import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
 import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
@@ -305,7 +306,7 @@ public void testApplyTransactionFailure() throws Exception {
         (KeyValueContainerData) containerData;
     key.close();
     ContainerStateMachine stateMachine =
-        (ContainerStateMachine) ContainerTestHelper.getStateMachine(cluster);
+        (ContainerStateMachine) TestHelper.getStateMachine(cluster);
     SimpleStateMachineStorage storage =
         (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
     Path parentPath = storage.findLatestSnapshot().getFile().getPath();
@@ -383,7 +384,7 @@ public void testApplyTransactionIdempotencyWithClosedContainer()
     Assert.assertTrue(containerData instanceof KeyValueContainerData);
     key.close();
     ContainerStateMachine stateMachine =
-        (ContainerStateMachine) ContainerTestHelper.getStateMachine(cluster);
+        (ContainerStateMachine) TestHelper.getStateMachine(cluster);
     SimpleStateMachineStorage storage =
         (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
     Path parentPath = storage.findLatestSnapshot().getFile().getPath();
@@ -454,7 +455,7 @@ public void testWriteStateMachineDataIdempotencyWithClosedContainer()
     Assert.assertTrue(containerData instanceof KeyValueContainerData);
     key.close();
     ContainerStateMachine stateMachine =
-        (ContainerStateMachine) ContainerTestHelper.getStateMachine(cluster);
+        (ContainerStateMachine) TestHelper.getStateMachine(cluster);
     SimpleStateMachineStorage storage =
         (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
     Path parentPath = storage.findLatestSnapshot().getFile().getPath();
@@ -619,4 +620,4 @@ public void testValidateBCSIDOnDnRestart() throws Exception {
             .getContainerState()
             == ContainerProtos.ContainerDataProto.State.UNHEALTHY);
   }
-}
+}

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java

@@ -39,6 +39,7 @@
 import org.apache.hadoop.ozone.client.io.KeyOutputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -245,7 +246,7 @@ public void testContainerExclusionWithClosedContainerException()
     containerIdList.add(containerId);
 
     // below check will assert if the container does not get closed
-    ContainerTestHelper
+    TestHelper
         .waitForContainerClose(cluster, containerIdList.toArray(new Long[0]));
 
     // This write will hit ClosedContainerException and this container should
@@ -404,12 +405,12 @@ public void testPipelineExclusionWithPipelineFailure() throws Exception {
 
   private OzoneOutputStream createKey(String keyName, ReplicationType type,
       long size) throws Exception {
-    return ContainerTestHelper
+    return TestHelper
         .createKey(keyName, type, size, objectStore, volumeName, bucketName);
   }
 
   private void validateData(String keyName, byte[] data) throws Exception {
-    ContainerTestHelper
+    TestHelper
         .validateData(keyName, data, objectStore, volumeName, bucketName);
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java

@@ -31,6 +31,7 @@
 import org.apache.hadoop.ozone.client.io.KeyInputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -115,7 +116,7 @@ private String getKeyName() {
 
   private OzoneOutputStream createKey(String keyName, ReplicationType type,
       long size) throws Exception {
-    return ContainerTestHelper
+    return TestHelper
         .createKey(keyName, type, size, objectStore, volumeName, bucketName);
   }
 
@@ -126,7 +127,7 @@ public void testSeekRandomly() throws Exception {
         .getXceiverClientMetrics();
 
     String keyName = getKeyName();
-    OzoneOutputStream key = ContainerTestHelper.createKey(keyName,
+    OzoneOutputStream key = TestHelper.createKey(keyName,
         ReplicationType.RATIS, 0, objectStore, volumeName, bucketName);
 
     // write data of more than 2 blocks.
@@ -230,7 +231,7 @@ public void testSeek() throws Exception {
         ContainerProtos.Type.ReadChunk);
 
     String keyName = getKeyName();
-    OzoneOutputStream key = ContainerTestHelper.createKey(keyName,
+    OzoneOutputStream key = TestHelper.createKey(keyName,
         ReplicationType.RATIS, 0, objectStore, volumeName, bucketName);
 
     // write data spanning 3 chunks

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java

@@ -35,6 +35,7 @@
 import org.apache.hadoop.ozone.client.io.KeyOutputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -208,12 +209,12 @@ public void testMultiBlockWritesWithIntermittentDnFailures()
 
   private OzoneOutputStream createKey(String keyName, ReplicationType type,
       long size) throws Exception {
-    return ContainerTestHelper
+    return TestHelper
         .createKey(keyName, type, size, objectStore, volumeName, bucketName);
   }
 
   private void validateData(String keyName, byte[] data) throws Exception {
-    ContainerTestHelper
+    TestHelper
         .validateData(keyName, data, objectStore, volumeName, bucketName);
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnException.java

@@ -36,6 +36,7 @@
 import org.apache.hadoop.ozone.client.io.KeyOutputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.TestHelper;
 import org.apache.ratis.protocol.GroupMismatchException;
 import org.junit.After;
 import org.junit.Assert;
@@ -148,7 +149,7 @@ public void testGroupMismatchExceptionHandling() throws Exception {
     Pipeline pipeline =
         cluster.getStorageContainerManager().getPipelineManager()
             .getPipeline(container.getPipelineID());
-    ContainerTestHelper.waitForPipelineClose(key, cluster, true);
+    TestHelper.waitForPipelineClose(key, cluster, true);
     key.flush();
     Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream
         .getIoException()) instanceof GroupMismatchException);
@@ -196,7 +197,7 @@ public void testMaxRetriesByOzoneClient() throws Exception {
     OutputStream stream = entries.get(0).getOutputStream();
     Assert.assertTrue(stream instanceof BlockOutputStream);
     BlockOutputStream blockOutputStream = (BlockOutputStream) stream;
-    ContainerTestHelper.waitForContainerClose(key, cluster);
+    TestHelper.waitForContainerClose(key, cluster);
     try {
       key.write(data1);
       Assert.fail("Expected exception not thrown");
@@ -222,12 +223,12 @@ public void testMaxRetriesByOzoneClient() throws Exception {
 
   private OzoneOutputStream createKey(String keyName, ReplicationType type,
       long size) throws Exception {
-    return ContainerTestHelper
+    return TestHelper
         .createKey(keyName, type, size, objectStore, volumeName, bucketName);
   }
 
   private void validateData(String keyName, byte[] data) throws Exception {
-    ContainerTestHelper
+    TestHelper
         .validateData(keyName, data, objectStore, volumeName, bucketName);
   }
 }