HDDS-6303. Merge from master to resolve CI workflow issues & hrpcOmTr…

…ansport support (apache#3074)

* HDDS-6149. Remove unused keytabs (apache#2960)

* HDDS-6094. Some unit tests are skipped due to using JUnit4 runner (apache#2909)

* HDDS-6075. OzoneConfiguration constructor overrides input configuration keys. (apache#2921)

* HDDS-4177. SCM Container DB bootstrap on Recon startup (apache#2942)

* HDDS-6086. Compute MD5MD5CRC file checksum using chunk checksums from DataNodes (apache#2919)

* HDDS-6148. Validate ContainerBalancerConfiguration before start ContainerBalancer (apache#2957)

* HDDS-6161. SCM StateMachine failing to reinitialize doesn't terminate the process. (apache#2971)

* HDDS-6134. Move replication-specific config to ReplicationServer (apache#2943)

* HDDS-4010. S3G startup fails when multiple service ids are configured. (apache#2976)

* HDDS-6170. Add metrics to replication manager to track container health states (apache#2975)

* HDDS-3231. Cleanup KeyManagerImpl (apache#2961)

* HDDS-5927. Improve defaults in ContainerBalancerConfiguration (apache#2892)

* HDDS-6157. More consistent synchronization in InputStreams (apache#2965)

* HDDS-4743. [FSO] Add FSO variant of ITestOzoneContractDistcp. (apache#2980)

* HDDS-6114. Intermittent error due to Failed to init RocksDB (apache#2947)

* HDDS-6175. Use s3Auth during proxy during decrypt in RpcClient. (apache#2981)

* HDDS-6175. Use s3Auth during proxy during decrypt in RpcClient.

* HDDS-5740. Enable ratis by default for SCM. (apache#2637)

* HDDS-6183. Intermittent failure in TestKeyDeletingService.checkIfDeleteServiceWithFailingSCM. (apache#2991)

* HDDS-4190. Intermittent failure in TestOMVolumeSetOwnerRequest and TestOMVolumeSetQuotaRequest. (apache#2982)

* HDDS-6120. Compute block checksum using chunk checksums (apache#2930)

* HDDS-6147. Add ability in OM to get limited delta updates (apache#2956)

* HDDS-6195. Remove unused jmh-core dependency. (apache#2997)

* HDDS-6167. Update ozone-runner version to 20211202-1 (apache#2969)

* HDDS-6171. Create an API to change Bucket Owner (apache#2988)

* HDDS-6163. Fix PATH in docker image (apache#2967)

* HDDS-6202. Avoid using jmh-generator-annprocess since it is GPL2.0. (apache#2998)

* HDDS-6135. SCM Container DB bootstrap on Recon startup for SCM HA. (apache#2972)

* HDDS-6109. Preserve the underlying exception raised in client lib. (apache#2989)

* HDDS-3408. Rename ChunkLayOutVersion to ContainerLayoutVersion. (apache#2983)

* HDDS-6203. CleanUp incomplete gz files during Container move (apache#3000)

* HDDS-6216. Move OMOpenKeysDeleteRequest to package om.request.key (apache#3011)

* HDDS-6191. Intermittent failure in TestDeleteWithSlowFollower (apache#3015)

* HDDS-6128. CLI tool that downloads all the block replicas and creates a manifest file (apache#2987)

* HDDS-6177. Extend container info command to include replica details  (apache#2995)

* HDDS-6211. [Docs] Image styling on deployed site does not replicate local builds. (apache#3007)

* HDDS-6219. Switch to RATIS ReplicationType from STAND_ALONE in our tests. (apache#3014)

* HDDS-6192. feature/Observability.md translated to Chinese (apache#2994)

* HDDS-6205. Add CLI command to display the latest Replication Manager report (apache#3013)

* HDDS-6227. Test helpers should observe naming conditions (apache#3020)

* HDDS-6239. ozonesecure-mr failing with No URLs in mirrorlist (apache#3029)

* HDDS-6201. Fix NPE for DataScanner with scanned container deleted by others. (apache#3005)

* HDDS-5529. For any IOexception from @REPLicated method we should throw it (apache#2788)

* HDDS-6181. Change SCMHAInvocationHandler#invokeRatis() logging to TRACE (apache#2992)

* HDDS-6206. Application errors must not flood system log (apache#3001)

* HDDS-6245. Add BucketLayout logging to Audit Logs (apache#3040)

* HDDS-6238 Reduce memory requirements for list keys. (apache#3032)

* HDDS-2919. Intermittent failure in TestRDBStore (apache#3028)

* HDDS-6253. Unnecessary duplicate smoketest after defaulting to FSO (apache#3036)

* HDDS-6204. Cleanup handling malformed authorization header (apache#2999)

* HDDS-6169. Selective checks: skip junit tests on ozone-runner image update (apache#2974)

* HDDS-6270. Use a dedicated file instead of /etc/passwd for xcompat acceptance test (apache#3050)

* HDDS-6273. Amend doc SecuringTDE.md (apache#3047)

* HDDS-6140. Selective checks: skip unit check for integration-test changes (apache#2948)

* HDDS-6215. Recon get limited delta updates from OM (apache#3009)

* HDDS-6125. Recon get limited delta updates from OM

* HDDS-6215. Fix unit test

* trigger new CI check

* HDDS-6215. Fix typo

* trigger new CI check

Co-authored-by: Symious <yiyang0203@gmail.com>

* HDDS-6226. Run tests for selective CI checks in CI (apache#3019)

* HDDS-6247. Avoid logging stack trace for user input problems (apache#3039)

* HDDS-6208. New checkstyle: WhitespaceAround (apache#3003)

* HDDS-6289. Upgrade acceptance test log flooded with parse error (apache#3063)

Co-authored-by: Siyao Meng <50227127+smengcl@users.noreply.github.com>

* Trigger Build

* Fix integration test for added configuation field for selecting OmTransport for s3 gateway - TestOzoneConfigurationFields (added config key not in xml).

Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com>
Co-authored-by: Lokesh Jain <ljain@apache.org>
Co-authored-by: Aswin Shakil Balasubramanian <aswinshakilbalu@gmail.com>
Co-authored-by: Wei-Chiu Chuang <weichiu@apache.org>
Co-authored-by: Symious <yiyang0203@foxmail.com>
Co-authored-by: Bharat Viswanadham <bharat@apache.org>
Co-authored-by: Stephen O'Donnell <stephen.odonnell@gmail.com>
Co-authored-by: GeorgeJahad <github@blackbirdsystems.net>
Co-authored-by: Siddhant Sangwan <siddhantsangwan027@gmail.com>
Co-authored-by: Jyotinder Singh <jyotindrsingh@gmail.com>
Co-authored-by: Shashikant Banerjee <shashikant@apache.org>
Co-authored-by: Tsz-Wo Nicholas Sze <szetszwo@apache.org>
Co-authored-by: Kiyoshi Mizumaru <kiyoshi.mizumaru@gmail.com>
Co-authored-by: Ritesh H Shukla <kerneltime@gmail.com>
Co-authored-by: Nibiru <axcsd3692@qq.com>
Co-authored-by: Kaijie Chen <chen@kaijie.org>
Co-authored-by: Zita Dombi <50611074+dombizita@users.noreply.github.com>
Co-authored-by: Istvan Fajth <pifta@cloudera.com>
Co-authored-by: steinsgateted <71066027+steinsgateted@users.noreply.github.com>
Co-authored-by: Gui Hecheng <markgui@tencent.com>
Co-authored-by: Jackson Yao <jacksonyao@tencent.com>
Co-authored-by: Keyi Song <72794035+sky76093016@users.noreply.github.com>
Co-authored-by: UENISHI Kota <kuenishi@users.noreply.github.com>
Co-authored-by: Siyao Meng <50227127+smengcl@users.noreply.github.com>
Co-authored-by: Symious <yiyang0203@gmail.com>

.github/workflows/post-commit.yml

@@ -125,6 +125,12 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v2
+        if: matrix.check != 'bats'
+      - name: Checkout project with history
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+        if: matrix.check == 'bats'
       - name: Cache for maven dependencies
         uses: actions/cache@v2
         with:

.gitignore

@@ -68,6 +68,7 @@ hadoop-ozone/recon/node_modules
 .mvn
 
 .dev-tools
-
+dev-support/ci/bats-assert
+dev-support/ci/bats-support
 
 hadoop-ozone/dist/src/main/license/current.txt

CONTRIBUTING.md

@@ -20,7 +20,8 @@ We welcome contributions of:
       * Unit Tests (JUnit / Java)
       * Acceptance Tests (docker + robot framework)
       * Blockade tests (python + blockade) 
-      * Performance: We have multiple type of load generator / benchmark tools (`ozone freon`, `ozone genesis`), which can be used to test cluster and report problems.
+      * Performance: We have multiple type of load generator / benchmark tools (`ozone freon`),
+        which can be used to test cluster and report problems.
  * **Bug reports** pointing out broken functionality, docs, or suggestions for improvements are always welcome!
 
 ## Who To Contact

dev-support/ci/selective_ci_checks.bats

@@ -66,6 +66,17 @@ load bats-assert/load.bash
   assert_output -p needs-kubernetes-tests=true
 }
 
+@test "runner image update" {
+  run dev-support/ci/selective_ci_checks.sh b95eeba82a
+
+  assert_output -p 'basic-checks=["rat"]'
+  assert_output -p needs-build=true
+  assert_output -p needs-compose-tests=true
+  assert_output -p needs-dependency-check=true
+  assert_output -p needs-integration-tests=false
+  assert_output -p needs-kubernetes-tests=true
+}
+
 @test "check script" {
   run dev-support/ci/selective_ci_checks.sh 316899152
 
@@ -77,10 +88,21 @@ load bats-assert/load.bash
   assert_output -p needs-kubernetes-tests=true
 }
 
+@test "integration and unit" {
+  run dev-support/ci/selective_ci_checks.sh 9aebf6e25
+
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
+  assert_output -p needs-build=false
+  assert_output -p needs-compose-tests=false
+  assert_output -p needs-dependency-check=false
+  assert_output -p needs-integration-tests=true
+  assert_output -p needs-kubernetes-tests=false
+}
+
 @test "integration only" {
   run dev-support/ci/selective_ci_checks.sh 61396ba9f
 
-  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
+  assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]'
   assert_output -p needs-build=false
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false

dev-support/ci/selective_ci_checks.sh

@@ -219,7 +219,10 @@ function get_count_compose_files() {
     start_end::group_start "Count compose files"
     local pattern_array=(
         "^hadoop-ozone/dev-support/checks/acceptance.sh"
-        "^hadoop-ozone/dist/src/main/compose"
+        "^hadoop-ozone/dist"
+    )
+    local ignore_array=(
+        "^hadoop-ozone/dist/src/main/k8s"
     )
     filter_changed_files true
     COUNT_COMPOSE_CHANGED_FILES=${match_count}
@@ -258,7 +261,10 @@ function get_count_kubernetes_files() {
     start_end::group_start "Count kubernetes files"
     local pattern_array=(
         "^hadoop-ozone/dev-support/checks/kubernetes.sh"
-        "^hadoop-ozone/dist/src/main/k8s"
+        "^hadoop-ozone/dist"
+    )
+    local ignore_array=(
+        "^hadoop-ozone/dist/src/main/compose"
     )
     filter_changed_files true
     COUNT_KUBERNETES_CHANGED_FILES=${match_count}
@@ -332,6 +338,9 @@ function check_needs_checkstyle() {
         "pom.xml"
         "src/..../java"
     )
+    local ignore_array=(
+        "^hadoop-ozone/dist"
+    )
     filter_changed_files
 
     if [[ ${match_count} != "0" ]]; then
@@ -373,6 +382,9 @@ function check_needs_findbugs() {
         "pom.xml"
         "src/..../java"
     )
+    local ignore_array=(
+        "^hadoop-ozone/dist"
+    )
     filter_changed_files
 
     if [[ ${match_count} != "0" ]]; then
@@ -391,6 +403,11 @@ function check_needs_unit_test() {
         "src/..../java"
         "src/..../resources"
     )
+    local ignore_array=(
+        "^hadoop-ozone/dist"
+        "^hadoop-ozone/fault-injection-test/mini-chaos-tests"
+        "^hadoop-ozone/integration-test"
+    )
     filter_changed_files
 
     if [[ ${match_count} != "0" ]]; then

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java

@@ -159,7 +159,7 @@ public void connect(String encodedToken) throws Exception {
 
   private synchronized void connectToDatanode(DatanodeDetails dn)
       throws IOException {
-    if (isConnected(dn)){
+    if (isConnected(dn)) {
       return;
     }
     // read port from the data node, on failure use default configured
@@ -269,10 +269,10 @@ public ContainerCommandResponseProto sendCommand(
         Thread.currentThread().interrupt();
       }
     }
-    try{
+    try {
       for (Map.Entry<DatanodeDetails,
               CompletableFuture<ContainerCommandResponseProto> >
-              entry : futureHashMap.entrySet()){
+              entry : futureHashMap.entrySet()) {
         responseProtoHashMap.put(entry.getKey(), entry.getValue().get());
       }
     } catch (InterruptedException e) {
@@ -538,7 +538,7 @@ public void onCompleted() {
   }
 
   private synchronized void checkOpen(DatanodeDetails dn)
-      throws IOException{
+      throws IOException {
     if (closed) {
       throw new IOException("This channel is not connected.");
     }

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java

@@ -206,8 +206,8 @@ public static void verifyKeyName(String keyName) {
     if (keyName == null) {
       throw new IllegalArgumentException("Key name is null");
     }
-    if(!OzoneConsts.KEYNAME_ILLEGAL_CHARACTER_CHECK_REGEX
-            .matcher(keyName).matches()){
+    if (!OzoneConsts.KEYNAME_ILLEGAL_CHARACTER_CHECK_REGEX
+            .matcher(keyName).matches()) {
       throw new IllegalArgumentException("Invalid key name: " + keyName);
     }
   }

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java

@@ -152,7 +152,7 @@ public synchronized void initialize() throws IOException {
         // retry according to retry policy.
         chunks = getChunkInfos();
         break;
-      } catch(SCMSecurityException ex) {
+      } catch (SCMSecurityException ex) {
         throw ex;
       } catch (StorageContainerException ex) {
         refreshPipeline(ex);
@@ -340,9 +340,9 @@ synchronized int readWithStrategy(ByteReaderStrategy strategy) throws
         } else {
           throw e;
         }
-      } catch(SCMSecurityException ex) {
+      } catch (SCMSecurityException ex) {
         throw ex;
-      } catch(IOException ex) {
+      } catch (IOException ex) {
         // We got a IOException which might be due
         // to DN down or connectivity issue.
         if (shouldRetryRead(ex)) {
@@ -512,7 +512,7 @@ synchronized long getBlockPosition() {
   }
 
   @Override
-  public void unbuffer() {
+  public synchronized void unbuffer() {
     storePosition();
     releaseClient();
 

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java

@@ -487,7 +487,7 @@ public void flush() throws IOException {
       } catch (Throwable e) {
         String msg = "Failed to flush. error: " + e.getMessage();
         LOG.error(msg, e);
-        throw new RuntimeException(msg, e);
+        throw e;
       }
     }
   }
@@ -553,7 +553,7 @@ public void close() throws IOException {
       } catch (Throwable e) {
         String msg = "Failed to flush. error: " + e.getMessage();
         LOG.error(msg, e);
-        throw new RuntimeException(msg, e);
+        throw e;
       } finally {
         cleanup(false);
       }
@@ -708,7 +708,7 @@ private void handleInterruptedException(Exception ex,
       boolean processExecutionException)
       throws IOException {
     LOG.error("Command execution was interrupted.");
-    if(processExecutionException) {
+    if (processExecutionException) {
       handleExecutionException(ex);
     } else {
       throw new IOException(EXCEPTION_MSG + ex.toString(), ex);

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BufferPool.java

@@ -46,7 +46,7 @@ public BufferPool(int bufferSize, int capacity) {
   }
 
   public BufferPool(int bufferSize, int capacity,
-      Function<ByteBuffer, ByteString> byteStringConversion){
+      Function<ByteBuffer, ByteString> byteStringConversion) {
     this.capacity = capacity;
     this.bufferSize = bufferSize;
     bufferList = new ArrayList<>(capacity);

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/RatisBlockOutputStream.java

@@ -97,7 +97,7 @@ void releaseBuffersOnException() {
 
   @Override
   XceiverClientReply sendWatchForCommit(boolean bufferFull) throws IOException {
-    return bufferFull? commitWatcher.watchOnFirstIndex()
+    return bufferFull ? commitWatcher.watchOnFirstIndex()
         : commitWatcher.watchOnLastIndex();
   }
 

hadoop-hdds/common/pom.xml

@@ -166,6 +166,21 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>junit-jupiter-api</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.vintage</groupId>
+      <artifactId>junit-vintage-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>io.jaegertracing</groupId>
       <artifactId>jaeger-client</artifactId>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -151,7 +151,7 @@ public final class HddsConfigKeys {
    */
   public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration";
   // Limit Certificate duration to a max value of 5 years.
-  public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D";
+  public static final String HDDS_X509_MAX_DURATION_DEFAULT = "P1865D";
   public static final String HDDS_X509_SIGNATURE_ALGO =
       "hdds.x509.signature.algorithm";
   public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java

@@ -632,7 +632,7 @@ public static long getShutDownTimeOut(ConfigurationSource conf) {
    * Utility method to round up bytes into the nearest MB.
    */
   public static int roundupMb(long bytes) {
-    return (int)Math.ceil((double) bytes/(double) ONE_MB);
+    return (int)Math.ceil((double) bytes / (double) ONE_MB);
   }
 
   /**

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java

@@ -151,6 +151,6 @@ public static String createStartupShutdownMessage(VersionInfo versionInfo,
 
   public static String appendIfNotPresent(String str, char c) {
     Preconditions.checkNotNull(str, "Input string is null");
-    return str.isEmpty() || str.charAt(str.length() - 1) != c ? str + c: str;
+    return str.isEmpty() || str.charAt(str.length() - 1) != c ? str + c : str;
   }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/annotation/InterfaceAudience.java

@@ -50,7 +50,7 @@ public final class InterfaceAudience {
    */
   @Documented
   @Retention(RetentionPolicy.RUNTIME)
-  public @interface Public {};
+  public @interface Public { };
 
   /**
    * Intended only for the project(s) specified in the annotation.
@@ -67,7 +67,7 @@ public final class InterfaceAudience {
    */
   @Documented
   @Retention(RetentionPolicy.RUNTIME)
-  public @interface Private {};
+  public @interface Private { };
 
-  private InterfaceAudience() {} // Audience can't exist on its own
+  private InterfaceAudience() { } // Audience can't exist on its own
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java

@@ -43,7 +43,7 @@ public final class OzoneQuota {
   public static final String OZONE_QUOTA_TB = "TB";
 
   /** Quota Units.*/
-  public enum Units {B, KB, MB, GB, TB}
+  public enum Units { B, KB, MB, GB, TB }
 
   // Quota to decide how many buckets can be created.
   private long quotaInNamespace;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/QuotaList.java

@@ -28,13 +28,14 @@ public class QuotaList {
   private ArrayList<OzoneQuota.Units> unitQuota;
   private ArrayList<Long> sizeQuota;
 
-  public QuotaList(){
+  public QuotaList() {
     ozoneQuota = new ArrayList<String>();
     unitQuota = new ArrayList<OzoneQuota.Units>();
     sizeQuota = new ArrayList<Long>();
   }
 
-  public void addQuotaList(String oQuota, OzoneQuota.Units uQuota, Long sQuota){
+  public void addQuotaList(
+      String oQuota, OzoneQuota.Units uQuota, Long sQuota) {
     ozoneQuota.add(oQuota);
     unitQuota.add(uQuota);
     sizeQuota.add(sQuota);
@@ -52,15 +53,15 @@ public ArrayList<OzoneQuota.Units> getUnitQuotaArray() {
     return this.unitQuota;
   }
 
-  public OzoneQuota.Units getUnits(String oQuota){
+  public OzoneQuota.Units getUnits(String oQuota) {
     return unitQuota.get(ozoneQuota.indexOf(oQuota));
   }
 
-  public Long getQuotaSize(OzoneQuota.Units uQuota){
+  public Long getQuotaSize(OzoneQuota.Units uQuota) {
     return sizeQuota.get(unitQuota.indexOf(uQuota));
   }
 
-  public OzoneQuota.Units getQuotaUnit(Long sQuota){
+  public OzoneQuota.Units getQuotaUnit(Long sQuota) {
     return unitQuota.get(sizeQuota.indexOf(sQuota));
   }
 

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationFactor.java

@@ -46,7 +46,7 @@ public enum ReplicationFactor {
    * @return ReplicationFactor
    */
   public static ReplicationFactor valueOf(int value) {
-    if(value == 1) {
+    if (value == 1) {
       return ONE;
     }
     if (value == 3) {