HDDS-4821. Use SCM service ID in SCMBlockClient and SCM Client.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -277,8 +277,8 @@ public final class ScmConfigKeys {
   // able to send back a new list to the datanodes.
   public static final String OZONE_SCM_NAMES = "ozone.scm.names";
 
-  public static final String OZONE_SCM_INTERNAL_SERVICE_ID =
-      "ozone.scm.internal.service.id";
+  public static final String OZONE_SCM_DEFAULT_SERVICE_ID =
+      "ozone.scm.default.service.id";
 
   public static final String OZONE_SCM_SERVICE_IDS_KEY =
       "ozone.scm.service.ids";

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java → hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java

@@ -18,8 +18,9 @@
 
 package org.apache.hadoop.hdds.scm.ha;
 
+
 import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigurationException;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.server.ServerUtils;
@@ -29,9 +30,10 @@
 import java.io.File;
 import java.nio.file.Paths;
 import java.util.Collection;
-
 import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_DIR;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY;
 
 /**
  * Utility class used by SCM HA.
@@ -52,7 +54,7 @@ public static boolean isSCMHAEnabled(ConfigurationSource conf) {
   /**
    * Get a collection of all scmNodeIds for the given scmServiceId.
    */
-  public static Collection<String> getSCMNodeIds(Configuration conf,
+  public static Collection<String> getSCMNodeIds(ConfigurationSource conf,
                                                  String scmServiceId) {
     String key = addSuffix(ScmConfigKeys.OZONE_SCM_NODES_KEY, scmServiceId);
     return conf.getTrimmedStringCollection(key);
@@ -101,4 +103,32 @@ public static String getSCMRatisSnapshotDirectory(ConfigurationSource conf) {
     }
     return snapshotDir;
   }
+
+  /**
+   * Get SCM ServiceId from OzoneConfiguration.
+   * @param conf
+   * @return SCM service id if defined, else null.
+   */
+  public static String getScmServiceId(ConfigurationSource conf) {
+
+    String localScmServiceId = conf.getTrimmed(
+        ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID);
+
+    Collection< String > scmServiceIds;
+
+    if (localScmServiceId == null) {
+      // There is no default scm service id is being set, fall back to ozone
+      // .scm.service.ids.
+      scmServiceIds = conf.getTrimmedStringCollection(
+          OZONE_SCM_SERVICE_IDS_KEY);
+      if (scmServiceIds.size() > 1) {
+        throw new ConfigurationException("When multiple SCM Service Ids are " +
+            "configured," + OZONE_SCM_DEFAULT_SERVICE_ID + " need to be " +
+            "defined");
+      } else if (scmServiceIds.size() == 1) {
+        localScmServiceId = scmServiceIds.iterator().next();
+      }
+    }
+    return localScmServiceId;
+  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java

@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.ha;
+
+import org.apache.hadoop.hdds.conf.ConfigurationException;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.OptionalInt;
+
+import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
+import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NODES_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.SCM_DUMMY_NODEID;
+import static org.apache.hadoop.ozone.OzoneConsts.SCM_DUMMY_SERVICE_ID;
+
+/**
+ * SCM Node Info.
+ */
+public class SCMNodeInfo {
+
+  private static final Logger LOG = LoggerFactory.getLogger(SCMNodeInfo.class);
+  private String serviceId;
+  private String nodeId;
+  private String blockClientAddress;
+  private String scmClientAddress;
+  private String scmSecurityAddress;
+  private String scmDatanodeAddress;
+
+  /**
+   * Build SCM Node information from configuration.
+   * @param conf
+   * @return
+   */
+  public static List<SCMNodeInfo> buildNodeInfo(ConfigurationSource conf) {
+
+    // First figure out scm client address from HA style config.
+    // If service Id is not defined, fall back to non-HA config.
+
+    List<SCMNodeInfo> scmNodeInfoList = new ArrayList<>();
+    String scmServiceId = SCMHAUtils.getScmServiceId(conf);
+    if (scmServiceId != null) {
+      ArrayList< String > scmNodeIds = new ArrayList<>(
+          SCMHAUtils.getSCMNodeIds(conf, scmServiceId));
+      if (scmNodeIds.size() == 0) {
+        throw new ConfigurationException(
+            String.format("Configuration does not have any value set for %s " +
+                    "for the SCM serviceId %s. List of SCM Node ID's should " +
+                    "be specified for an SCM HA service", OZONE_SCM_NODES_KEY,
+                scmServiceId));
+      }
+
+      for (String scmNodeId : scmNodeIds) {
+        String addressKey = ConfUtils.addKeySuffixes(
+            OZONE_SCM_ADDRESS_KEY, scmServiceId, scmNodeId);
+        String scmAddress = conf.get(addressKey);
+        if (scmAddress == null) {
+          throw new ConfigurationException(addressKey + "is not defined");
+        }
+
+        // Get port from Address Key if defined, else fall back to port key.
+        int scmClientPort = getPort(conf, scmServiceId, scmNodeId,
+            OZONE_SCM_CLIENT_ADDRESS_KEY, OZONE_SCM_CLIENT_PORT_KEY,
+            OZONE_SCM_CLIENT_PORT_DEFAULT);
+
+        int scmBlockClientPort = getPort(conf, scmServiceId, scmNodeId,
+            OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
+            OZONE_SCM_BLOCK_CLIENT_PORT_KEY,
+            OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT);
+
+        int scmSecurityPort = getPort(conf, scmServiceId, scmNodeId,
+            OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY,
+            OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+            OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT);
+
+        int scmDatanodePort = getPort(conf, scmServiceId, scmNodeId,
+            OZONE_SCM_DATANODE_ADDRESS_KEY, OZONE_SCM_DATANODE_PORT_KEY,
+            OZONE_SCM_DATANODE_PORT_DEFAULT);
+
+        scmNodeInfoList.add(new SCMNodeInfo(scmServiceId, scmNodeId,
+            buildAddress(scmAddress, scmBlockClientPort),
+            buildAddress(scmAddress, scmClientPort),
+            buildAddress(scmAddress, scmSecurityPort),
+            buildAddress(scmAddress, scmDatanodePort)));
+      }
+      return scmNodeInfoList;
+    } else {
+      scmServiceId = SCM_DUMMY_SERVICE_ID;
+
+      // Following current approach of fall back to
+      // OZONE_SCM_CLIENT_ADDRESS_KEY to figure out hostname.
+
+      String scmBlockClientAddress = getHostNameFromConfigKeys(conf,
+          OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
+          OZONE_SCM_CLIENT_ADDRESS_KEY).orElse(null);
+
+      String scmClientAddress = getHostNameFromConfigKeys(conf,
+          OZONE_SCM_CLIENT_ADDRESS_KEY).orElse(null);
+
+      String scmSecurityClientAddress =
+          getHostNameFromConfigKeys(conf,
+              OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY,
+              OZONE_SCM_CLIENT_ADDRESS_KEY).orElse(null);
+
+      String scmDatanodeAddress =
+          getHostNameFromConfigKeys(conf,
+              OZONE_SCM_DATANODE_ADDRESS_KEY,
+              OZONE_SCM_CLIENT_ADDRESS_KEY).orElse(null);
+
+      int scmBlockClientPort = getPortNumberFromConfigKeys(conf,
+          OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY)
+          .orElse(conf.getInt(OZONE_SCM_BLOCK_CLIENT_PORT_KEY,
+              OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
+
+      int scmClientPort = getPortNumberFromConfigKeys(conf,
+          OZONE_SCM_CLIENT_ADDRESS_KEY)
+          .orElse(conf.getInt(OZONE_SCM_CLIENT_PORT_KEY,
+              OZONE_SCM_CLIENT_PORT_DEFAULT));
+
+      int scmSecurityPort = getPortNumberFromConfigKeys(conf,
+          OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY)
+          .orElse(conf.getInt(OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+              OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT));
+
+      int scmDatanodePort = getPortNumberFromConfigKeys(conf,
+          OZONE_SCM_DATANODE_ADDRESS_KEY)
+          .orElse(conf.getInt(OZONE_SCM_DATANODE_PORT_KEY,
+              OZONE_SCM_DATANODE_PORT_DEFAULT));
+
+      scmNodeInfoList.add(new SCMNodeInfo(scmServiceId,
+          SCM_DUMMY_NODEID,
+          scmBlockClientAddress == null ? null :
+              buildAddress(scmBlockClientAddress, scmBlockClientPort),
+          scmClientAddress == null ? null :
+              buildAddress(scmClientAddress, scmClientPort),
+          scmSecurityClientAddress == null ? null :
+              buildAddress(scmSecurityClientAddress, scmSecurityPort),
+          scmDatanodeAddress == null ? null :
+              buildAddress(scmDatanodeAddress, scmDatanodePort)));
+
+      return scmNodeInfoList;
+
+    }
+
+  }
+
+  private static String buildAddress(String address, int port) {
+    return new StringBuilder().append(address).append(":")
+        .append(port).toString();
+  }
+
+  private static int getPort(ConfigurationSource conf,
+      String scmServiceId, String scmNodeId, String configKey,
+      String portKey, int defaultPort) {
+    String suffixKey = ConfUtils.addKeySuffixes(configKey, scmServiceId,
+        scmNodeId);
+    OptionalInt port = getPortNumberFromConfigKeys(conf, suffixKey);
+
+    if (port.isPresent()) {
+      LOG.info("ConfigKey {} is deprecated, For configuring different " +
+          "ports for each SCM use PortConfigKey {} appended with serviceId " +
+          "and nodeId", configKey, portKey);
+      return port.getAsInt();
+    } else {
+      return conf.getInt(ConfUtils.addKeySuffixes(portKey, scmServiceId,
+          scmNodeId), defaultPort);
+    }
+  }
+
+  /**
+   * SCM Node Info which contains information about scm service address.
+   * @param serviceId
+   * @param nodeId
+   * @param blockClientAddress
+   * @param scmClientAddress
+   * @param scmSecurityAddress
+   * @param scmDatanodeAddress
+   */
+  public SCMNodeInfo(String serviceId, String nodeId,
+      String blockClientAddress, String scmClientAddress,
+      String scmSecurityAddress, String scmDatanodeAddress) {
+    this.serviceId = serviceId;
+    this.nodeId = nodeId;
+    this.blockClientAddress = blockClientAddress;
+    this.scmClientAddress = scmClientAddress;
+    this.scmSecurityAddress = scmSecurityAddress;
+    this.scmDatanodeAddress = scmDatanodeAddress;
+  }
+
+  public String getServiceId() {
+    return serviceId;
+  }
+
+  public String getNodeId() {
+    return nodeId;
+  }
+
+  public String getBlockClientAddress() {
+    return blockClientAddress;
+  }
+
+  public String getScmClientAddress() {
+    return scmClientAddress;
+  }
+
+  public String getScmSecurityAddress() {
+    return scmSecurityAddress;
+  }
+
+  public String getScmDatanodeAddress() {
+    return scmDatanodeAddress;
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/package-info.java

@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.ha;
+/**
+ Utility slasses for SCM HA.
+ */

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/package-info.java

@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.server;
+
+/**
+ * Server Util classes.
+ */

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -404,5 +404,11 @@ private OzoneConsts() {
   public static final String OM_RATIS_SNAPSHOT_DIR = "snapshot";
   public static final String SCM_RATIS_SNAPSHOT_DIR = "snapshot";
 
-  public static final long DEFAULT_OM_UPDATE_ID = -1L;  
+  public static final long DEFAULT_OM_UPDATE_ID = -1L;
+
+
+  // SCM default service Id and node Id in non-HA where config is not defined
+  // in non-HA style.
+  public static final String SCM_DUMMY_NODEID = "scmNodeId";
+  public static final String SCM_DUMMY_SERVICE_ID = "scmServiceId";
 }