Merge master into upgrade

.github/workflows/post-commit.yml

@@ -56,6 +56,12 @@ jobs:
           name: ozone-bin
           path: hadoop-ozone/dist/target/hadoop-ozone*.tar.gz
           retention-days: 1
+      - name: Delete temporary build artifacts before caching
+        run: |
+          #Never cache local artifacts
+          rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds*
+          rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone*
+        if: always()
   bats:
     runs-on: ubuntu-18.04
     steps:
@@ -218,12 +224,6 @@ jobs:
           name: acceptance-${{ matrix.suite }}
           path: /mnt/ozone/target/acceptance
         continue-on-error: true
-      - name: Delete temporary build artifacts before caching
-        run: |
-          #Never cache local artifacts
-          rm -rf ~/.m2/repository/org/apache/hadoop/hdds
-          rm -rf ~/.m2/repository/org/apache/hadoop/ozone
-        if: always()
   integration:
     runs-on: ubuntu-18.04
     strategy:
@@ -335,9 +335,3 @@ jobs:
           name: kubernetes
           path: /mnt/ozone/target/kubernetes
         continue-on-error: true
-      - name: Delete temporary build artifacts before caching
-        run: |
-          #Never cache local artifacts
-          rm -rf ~/.m2/repository/org/apache/hadoop/hdds
-          rm -rf ~/.m2/repository/org/apache/hadoop/ozone
-        if: always()

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/JavaUtils.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds;
+
+/**
+ * Various reusable utility methods related to Java.
+ */
+public final class JavaUtils {
+  // "1.8"->8, "9"->9, "10"->10
+  private static final int JAVA_SPEC_VER = Math.max(8, Integer.parseInt(
+      System.getProperty("java.specification.version").split("\\.")[0]));
+
+  /**
+   * Query to see if major version of Java specification of the system
+   * is equal or greater than the parameter.
+   *
+   * @param version 8, 9, 10 etc.
+   * @return comparison with system property, always true for 8
+   */
+  public static boolean isJavaVersionAtLeast(int version) {
+    return JAVA_SPEC_VER >= version;
+  }
+
+  /**
+   * Private constructor.
+   */
+  private JavaUtils() {
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java

@@ -24,8 +24,8 @@
  * The replication type to be used while writing key into ozone.
  */
 public enum ReplicationType {
-    RATIS,
-    STAND_ALONE,
+  RATIS,
+  STAND_ALONE,
   CHAINED;
 
   public static ReplicationType fromProto(

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java

@@ -24,6 +24,7 @@
 import java.util.Set;
 import java.util.UUID;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.hdds.DatanodeVersions;
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
@@ -369,6 +370,7 @@ public static DatanodeDetails getFromProtoBuf(
    * Returns a DatanodeDetails protobuf message from a datanode ID.
    * @return HddsProtos.DatanodeDetailsProto
    */
+  @JsonIgnore
   public HddsProtos.DatanodeDetailsProto getProtoBufMessage() {
     return toProto(CURRENT_VERSION);
   }
@@ -431,6 +433,7 @@ public HddsProtos.DatanodeDetailsProto.Builder toProtoBuilder(
    * Returns a ExtendedDatanodeDetails protobuf message from a datanode ID.
    * @return HddsProtos.ExtendedDatanodeDetailsProto
    */
+  @JsonIgnore
   public HddsProtos.ExtendedDatanodeDetailsProto getExtendedProtoBufMessage() {
     HddsProtos.ExtendedDatanodeDetailsProto.Builder extendedBuilder =
         HddsProtos.ExtendedDatanodeDetailsProto.newBuilder()

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -246,6 +246,11 @@ public final class ScmConfigKeys {
   public static final int OZONE_SCM_HEARTBEAT_RPC_RETRY_COUNT_DEFAULT =
       15;
 
+  public static final String OZONE_SCM_HEARTBEAT_RPC_RETRY_INTERVAL =
+      "ozone.scm.heartbeat.rpc-retry-interval";
+  public static final String OZONE_SCM_HEARTBEAT_RPC_RETRY_INTERVAL_DEFAULT =
+      "1s";
+
   /**
    * Defines how frequently we will log the missing of heartbeat to a specific
    * SCM. In the default case we will write a warning message for each 10

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java

@@ -288,6 +288,17 @@ StatusAndMessages finalizeScmUpgrade(String upgradeClientID)
       throws IOException;
 
   StatusAndMessages queryUpgradeFinalizationProgress(String upgradeClientID,
-                                                     boolean force)
-      throws IOException;
+      boolean force) throws IOException;
+
+  /**
+   * Get usage information of datanode by ipaddress or uuid.
+   *
+   * @param ipaddress datanode ipaddress String
+   * @param uuid datanode uuid String
+   * @return List of DatanodeUsageInfo. Each element contains info such as
+   * capacity, SCMused, and remaining space.
+   * @throws IOException
+   */
+  List<HddsProtos.DatanodeUsageInfo> getDatanodeUsageInfo(String ipaddress,
+      String uuid) throws IOException;
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java

@@ -199,6 +199,7 @@ public void updateLastUsedTime() {
     lastUsed = Instant.ofEpochMilli(Time.now());
   }
 
+  @JsonIgnore
   public HddsProtos.ContainerInfoProto getProtobuf() {
     HddsProtos.ContainerInfoProto.Builder builder =
         HddsProtos.ContainerInfoProto.newBuilder();

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineID.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hdds.scm.pipeline;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 
 import java.util.UUID;
@@ -45,6 +46,7 @@ public UUID getId() {
     return id;
   }
 
+  @JsonIgnore
   public HddsProtos.PipelineID getProtobuf() {
     HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder()
         .setMostSigBits(id.getMostSignificantBits())

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java

@@ -18,20 +18,20 @@
 package org.apache.hadoop.hdds.scm.protocol;
 
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.ScmConfig;
 import org.apache.hadoop.hdds.scm.ScmInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
 import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.security.KerberosInfo;
 
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
-import org.apache.hadoop.security.KerberosInfo;
 
 /**
  * ContainerLocationProtocol is used by an HDFS node to find the set of nodes
@@ -246,6 +246,17 @@ StatusAndMessages finalizeScmUpgrade(String upgradeClientID)
       throws IOException;
 
   StatusAndMessages queryUpgradeFinalizationProgress(String upgradeClientID,
-                                                     boolean force)
-      throws IOException;
+      boolean force) throws IOException;
+
+  /**
+   * Get Datanode usage information by ip or uuid.
+   *
+   * @param ipaddress - datanode IP address String
+   * @param uuid - datanode UUID String
+   * @return List of DatanodeUsageInfo. Each element contains info such as
+   * capacity, SCMused, and remaining space.
+   * @throws IOException
+   */
+  List<HddsProtos.DatanodeUsageInfo> getDatanodeUsageInfo(String ipaddress,
+      String uuid) throws IOException;
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -231,6 +231,8 @@ public enum Units {TB, GB, MB, KB, B}
   // instance gets stored.
   public static final String OM_CONTEXT_ATTRIBUTE = "ozone.om";
 
+  public static final String SCM_CONTEXT_ATTRIBUTE = "ozone.scm";
+
   private OzoneConsts() {
     // Never Constructed
   }
@@ -398,5 +400,7 @@ private OzoneConsts() {
   public static final String OM_RATIS_SNAPSHOT_DIR = "snapshot";
 
   public static final long DEFAULT_OM_UPDATE_ID = -1L;
+  // CRL Sequence Id
+  public static final String CRL_SEQUENCE_ID_KEY = "CRL_SEQUENCE_ID";
 
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java

@@ -75,8 +75,10 @@ private static Function<ByteBuffer, ByteString> newChecksumByteBufferFunction(
   /** The algorithms for {@link ChecksumType}. */
   enum Algorithm {
     NONE(() -> data -> ByteString.EMPTY),
-    CRC32(() -> newChecksumByteBufferFunction(PureJavaCrc32ByteBuffer::new)),
-    CRC32C(() -> newChecksumByteBufferFunction(PureJavaCrc32CByteBuffer::new)),
+    CRC32(() ->
+        newChecksumByteBufferFunction(ChecksumByteBufferFactory::crc32Impl)),
+    CRC32C(() ->
+        newChecksumByteBufferFunction(ChecksumByteBufferFactory::crc32CImpl)),
     SHA256(() -> newMessageDigestFunction("SHA-256")),
     MD5(() -> newMessageDigestFunction("MD5"));
 

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/ChecksumByteBufferFactory.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.common;
+
+import org.apache.hadoop.hdds.JavaUtils;
+import org.apache.hadoop.util.PureJavaCrc32C;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+/**
+ * Class containing factories for creating various checksum impls.
+ */
+public final class ChecksumByteBufferFactory {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ChecksumByteBufferImpl.class);
+
+  private static volatile boolean useJava9Crc32C
+      = JavaUtils.isJavaVersionAtLeast(9);
+
+  public static class Java9Crc32CFactory {
+    private static final MethodHandle NEW_CRC32C_MH;
+
+    static {
+      MethodHandle newCRC32C = null;
+      try {
+        newCRC32C = MethodHandles.publicLookup()
+            .findConstructor(
+                Class.forName("java.util.zip.CRC32C"),
+                MethodType.methodType(void.class)
+            );
+      } catch (ReflectiveOperationException e) {
+        // Should not reach here.
+        throw new RuntimeException(e);
+      }
+      NEW_CRC32C_MH = newCRC32C;
+    }
+
+    public static java.util.zip.Checksum createChecksum() {
+      try {
+        // Should throw nothing
+        return (Checksum) NEW_CRC32C_MH.invoke();
+      } catch (Throwable t) {
+        throw (t instanceof RuntimeException) ? (RuntimeException) t
+            : new RuntimeException(t);
+      }
+    }
+  };
+
+  public static ChecksumByteBuffer crc32Impl() {
+    return new ChecksumByteBufferImpl(new CRC32());
+  }
+
+  public static ChecksumByteBuffer crc32CImpl() {
+    if (useJava9Crc32C) {
+      try {
+        return new ChecksumByteBufferImpl(Java9Crc32CFactory.createChecksum());
+      } catch (Throwable e) {
+        // should not happen
+        LOG.error("CRC32C creation failed, switching to PureJavaCrc32C", e);
+        useJava9Crc32C = false;
+      }
+    }
+    return new ChecksumByteBufferImpl(new PureJavaCrc32C());
+  }
+
+  /**
+   * Private Constructor.
+   */
+  private ChecksumByteBufferFactory() {
+  }
+}