HDDS-212. Introduce NodeStateManager to manage the state of Datanodes…

… in SCM. Contributed by Nanda kumar.
apache · Jul 4, 2018 · 71df8c2 · 71df8c2
1 parent 3b63715
commit 71df8c2
Show file tree

Hide file tree

Showing 31 changed files with 1,288 additions and 918 deletions.
diff --git a/...hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java b/...hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java
@@ -37,7 +37,6 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.util.EnumSet;
 import java.util.List;
 import java.util.UUID;
 
@@ -234,14 +233,14 @@ public ContainerWithPipeline createContainer(HddsProtos.ReplicationType type,
   /**
    * Returns a set of Nodes that meet a query criteria.
    *
-   * @param nodeStatuses - A set of criteria that we want the node to have.
+   * @param nodeStatuses - Criteria that we want the node to have.
    * @param queryScope - Query scope - Cluster or pool.
    * @param poolName - if it is pool, a pool name is required.
    * @return A set of nodes that meet the requested criteria.
    * @throws IOException
    */
   @Override
-  public HddsProtos.NodePool queryNode(EnumSet<HddsProtos.NodeState>
+  public List<HddsProtos.Node> queryNode(HddsProtos.NodeState
       nodeStatuses, HddsProtos.QueryScope queryScope, String poolName)
       throws IOException {
     return storageContainerLocationClient.queryNode(nodeStatuses, queryScope,
@@ -458,7 +457,8 @@ public void closeContainer(long containerId)
    */
   @Override
   public long getContainerSize(long containerID) throws IOException {
-    // TODO : Fix this, it currently returns the capacity but not the current usage.
+    // TODO : Fix this, it currently returns the capacity
+    // but not the current usage.
     long size = getContainerSizeB();
     if (size == -1) {
       throw new IOException("Container size unknown!");

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
@@ -35,7 +35,7 @@
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public final class DatanodeDetails implements Comparable<DatanodeDetails> {
+public class DatanodeDetails implements Comparable<DatanodeDetails> {
 
   /**
    * DataNode's unique identifier in the cluster.
@@ -63,6 +63,13 @@ private DatanodeDetails(String uuid, String ipAddress, String hostName,
     this.ports = ports;
   }
 
+  protected DatanodeDetails(DatanodeDetails datanodeDetails) {
+    this.uuid = datanodeDetails.uuid;
+    this.ipAddress = datanodeDetails.ipAddress;
+    this.hostName = datanodeDetails.hostName;
+    this.ports = datanodeDetails.ports;
+  }
+
   /**
    * Returns the DataNode UUID.
    *
@@ -238,7 +245,7 @@ public static Builder newBuilder() {
   /**
    * Builder class for building DatanodeDetails.
    */
-  public static class Builder {
+  public static final class Builder {
     private String id;
     private String ipAddress;
     private String hostName;
@@ -324,7 +331,7 @@ public static Port newPort(Port.Name name, Integer value) {
   /**
    * Container to hold DataNode Port details.
    */
-  public static class Port {
+  public static final class Port {
 
     /**
      * Ports that are supported in DataNode.

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -165,10 +165,6 @@ public final class ScmConfigKeys {
   public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =
       "10m";
 
-  public static final String OZONE_SCM_MAX_HB_COUNT_TO_PROCESS =
-      "ozone.scm.max.hb.count.to.process";
-  public static final int OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT = 5000;
-
   public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL =
       "ozone.scm.heartbeat.thread.interval";
   public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_DEFAULT =

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -26,7 +26,6 @@
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 
 import java.io.IOException;
-import java.util.EnumSet;
 import java.util.List;
 
 /**
@@ -150,13 +149,13 @@ ContainerWithPipeline createContainer(HddsProtos.ReplicationType type,
 
   /**
    * Returns a set of Nodes that meet a query criteria.
-   * @param nodeStatuses - A set of criteria that we want the node to have.
+   * @param nodeStatuses - Criteria that we want the node to have.
    * @param queryScope - Query scope - Cluster or pool.
    * @param poolName - if it is pool, a pool name is required.
    * @return A set of nodes that meet the requested criteria.
    * @throws IOException
    */
-  HddsProtos.NodePool queryNode(EnumSet<HddsProtos.NodeState> nodeStatuses,
+  List<HddsProtos.Node> queryNode(HddsProtos.NodeState nodeStatuses,
       HddsProtos.QueryScope queryScope, String poolName) throws IOException;
 
   /**

diff --git a/...n/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/...n/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -26,7 +26,6 @@
     .StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
 
 import java.io.IOException;
-import java.util.EnumSet;
 import java.util.List;
 
 /**
@@ -94,10 +93,10 @@ List<ContainerInfo> listContainer(long startContainerID, int count)
 
   /**
    *  Queries a list of Node Statuses.
-   * @param nodeStatuses
+   * @param state
    * @return List of Datanodes.
    */
-  HddsProtos.NodePool queryNode(EnumSet<HddsProtos.NodeState> nodeStatuses,
+  List<HddsProtos.Node> queryNode(HddsProtos.NodeState state,
       HddsProtos.QueryScope queryScope, String poolName) throws IOException;
 
   /**

diff --git a/...he/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/...he/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -59,7 +59,6 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.EnumSet;
 import java.util.List;
 
 /**
@@ -215,20 +214,19 @@ public void deleteContainer(long containerID)
    * @return List of Datanodes.
    */
   @Override
-  public HddsProtos.NodePool queryNode(EnumSet<HddsProtos.NodeState>
+  public List<HddsProtos.Node> queryNode(HddsProtos.NodeState
       nodeStatuses, HddsProtos.QueryScope queryScope, String poolName)
       throws IOException {
     // TODO : We support only cluster wide query right now. So ignoring checking
     // queryScope and poolName
     Preconditions.checkNotNull(nodeStatuses);
-    Preconditions.checkState(nodeStatuses.size() > 0);
     NodeQueryRequestProto request = NodeQueryRequestProto.newBuilder()
-        .addAllQuery(nodeStatuses)
+        .setState(nodeStatuses)
         .setScope(queryScope).setPoolName(poolName).build();
     try {
       NodeQueryResponseProto response =
           rpcProxy.queryNode(NULL_RPC_CONTROLLER, request);
-      return response.getDatanodes();
+      return response.getDatanodesList();
     } catch (ServiceException e) {
       throw  ProtobufHelper.getRemoteException(e);
     }

diff --git a/...pache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java b/...pache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -57,7 +57,6 @@
     .StorageContainerLocationProtocolProtos.SCMListContainerResponseProto;
 
 import java.io.IOException;
-import java.util.EnumSet;
 import java.util.List;
 
 /**
@@ -171,13 +170,12 @@ public SCMDeleteContainerResponseProto deleteContainer(
       StorageContainerLocationProtocolProtos.NodeQueryRequestProto request)
       throws ServiceException {
     try {
-      EnumSet<HddsProtos.NodeState> nodeStateEnumSet = EnumSet.copyOf(request
+      HddsProtos.NodeState nodeState = request.getState();
-          .getQueryList());
+      List<HddsProtos.Node> datanodes = impl.queryNode(nodeState,
-      HddsProtos.NodePool datanodes = impl.queryNode(nodeStateEnumSet,
           request.getScope(), request.getPoolName());
       return StorageContainerLocationProtocolProtos
           .NodeQueryResponseProto.newBuilder()
-          .setDatanodes(datanodes)
+          .addAllDatanodes(datanodes)
           .build();
     } catch (Exception e) {
       throw new ServiceException(e);

diff --git a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
@@ -118,26 +118,13 @@ message ObjectStageChangeResponseProto {
  match the NodeState that we are requesting.
 */
 message NodeQueryRequestProto {
-
+  required NodeState state = 1;
-
-  // Repeated, So we can specify more than one status type.
-  // These NodeState types are additive for now, in the sense that
-  // if you specify HEALTHY and FREE_NODE members --
-  // Then you get all healthy node which are not raft members.
-  //
-  // if you specify all healthy and dead nodes, you will get nothing
-  // back. Server is not going to dictate what combinations make sense,
-  // it is entirely up to the caller.
-  // TODO: Support operators like OR and NOT. Currently it is always an
-  // implied AND.
-
-  repeated NodeState query = 1;
   required QueryScope scope = 2;
   optional string poolName = 3; // if scope is pool, then pool name is needed.
 }
 
 message NodeQueryResponseProto {
-  required NodePool datanodes = 1;
+  repeated Node datanodes = 1;
 }
 
 /**
@@ -194,7 +181,7 @@ service StorageContainerLocationProtocolService {
   /**
   * Returns a set of Nodes that meet a criteria.
   */
-  rpc queryNode(NodeQueryRequestProto)  returns (NodeQueryResponseProto);
+  rpc queryNode(NodeQueryRequestProto) returns (NodeQueryResponseProto);
 
   /**
   * Notify from client when begin or finish container or pipeline operations on datanodes.

diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -69,14 +69,11 @@ enum NodeType {
  * and getNodeCount.
  */
 enum NodeState {
-    HEALTHY             = 1;
+    HEALTHY = 1;
-    STALE               = 2;
+    STALE = 2;
-    DEAD                = 3;
+    DEAD = 3;
-    DECOMMISSIONING     = 4;
+    DECOMMISSIONING = 4;
-    DECOMMISSIONED      = 5;
+    DECOMMISSIONED = 5;
-    RAFT_MEMBER         = 6;
-    FREE_NODE           = 7; // Not a member in raft.
-    INVALID             = 8;
 }
 
 enum QueryScope {

diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -773,17 +773,6 @@
       The keytab file for Kerberos authentication in SCM.
     </description>
   </property>
-  <property>
-    <name>ozone.scm.max.hb.count.to.process</name>
-    <value>5000</value>
-    <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
-    <description>
-      The maximum number of heartbeat to process per loop of the
-      heartbeat process thread. Please see
-      ozone.scm.heartbeat.thread.interval
-      for more info.
-    </description>
-  </property>
   <property>
     <name>ozone.scm.names</name>
     <value/>

diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -258,17 +258,6 @@ public static long getDeadNodeInterval(Configuration conf) {
     return deadNodeIntervalMs;
   }
 
-  /**
-   * Returns the maximum number of heartbeat to process per loop of the process
-   * thread.
-   * @param conf Configuration
-   * @return - int -- Number of HBs to process
-   */
-  public static int getMaxHBToProcessPerLoop(Configuration conf) {
-    return conf.getInt(ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS,
-        ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT);
-  }
-
   /**
    * Timeout value for the RPC from Datanode to SCM, primarily used for
    * Heartbeats and container reports.

diff --git a/...-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java b/...-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
@@ -59,10 +59,8 @@ RegisteredCommand register(DatanodeDetails datanodeDetails,
   /**
    * Send heartbeat to indicate the datanode is alive and doing well.
    * @param datanodeDetails - Datanode ID.
-   * @param nodeReport - node report.
    * @return SCMheartbeat response list
    */
-  List<SCMCommand> sendHeartbeat(DatanodeDetails datanodeDetails,
+  List<SCMCommand> processHeartbeat(DatanodeDetails datanodeDetails);
-      NodeReportProto nodeReport);
 
 }