Skip to content

Commit

Permalink
HDDS-4730. Use separate Ratis admin and client ports (#1887)
Browse files Browse the repository at this point in the history
  • Loading branch information
@adoroszlai
adoroszlai committed Feb 8, 2021
1 parent 8585fba commit 375da4d
Show file tree
Hide file tree
Showing 16 changed files with 566 additions and 108 deletions.
35 changes: 35 additions & 0 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DatanodeVersions.java
View file
@@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds;

/**
* Versioning for datanode.
*/
public final class DatanodeVersions {

public static final int DEFAULT_VERSION = 0;

public static final int SEPARATE_RATIS_PORTS_AVAILABLE = 1;

// this should always point to the latest version
public static final int CURRENT_VERSION = SEPARATE_RATIS_PORTS_AVAILABLE;

private DatanodeVersions() {
// no instances
}
}
53 changes: 50 additions & 3 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
View file
Expand Up @@ -25,6 +25,7 @@
import java.util.UUID;

import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.hdds.DatanodeVersions;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
Expand Down Expand Up @@ -73,6 +74,8 @@ public class DatanodeDetails extends NodeImpl implements
private String buildDate;
private HddsProtos.NodeOperationalState persistedOpState;
private long persistedOpStateExpiryEpochSec = 0;
private int initialVersion;
private int currentVersion;

/**
* Constructs DatanodeDetails instance. DatanodeDetails.Builder is used
Expand All @@ -96,7 +99,8 @@ private DatanodeDetails(UUID uuid, String ipAddress, String hostName,
String networkLocation, List<Port> ports, String certSerialId,
String version, long setupTime, String revision, String buildDate,
HddsProtos.NodeOperationalState persistedOpState,
long persistedOpStateExpiryEpochSec) {
long persistedOpStateExpiryEpochSec,
int initialVersion, int currentVersion) {
super(hostName, networkLocation, NetConstants.NODE_COST_DEFAULT);
this.uuid = uuid;
this.uuidString = uuid.toString();
Expand All @@ -110,6 +114,8 @@ private DatanodeDetails(UUID uuid, String ipAddress, String hostName,
this.buildDate = buildDate;
this.persistedOpState = persistedOpState;
this.persistedOpStateExpiryEpochSec = persistedOpStateExpiryEpochSec;
this.initialVersion = initialVersion;
this.currentVersion = currentVersion;
}

public DatanodeDetails(DatanodeDetails datanodeDetails) {
Expand All @@ -129,6 +135,8 @@ public DatanodeDetails(DatanodeDetails datanodeDetails) {
this.persistedOpState = datanodeDetails.getPersistedOpState();
this.persistedOpStateExpiryEpochSec =
datanodeDetails.getPersistedOpStateExpiryEpochSec();
this.initialVersion = datanodeDetails.getInitialVersion();
this.currentVersion = datanodeDetails.getCurrentVersion();
}

/**
Expand Down Expand Up @@ -263,6 +271,10 @@ public Port getPort(Port.Name name) {
return port;
}
}
// if no separate admin/server port, return single Ratis one for compat
if (name == Name.RATIS_ADMIN || name == Name.RATIS_SERVER) {
return getPort(Name.RATIS);
}
return null;
}

Expand Down Expand Up @@ -440,6 +452,28 @@ public HddsProtos.ExtendedDatanodeDetailsProto getExtendedProtoBufMessage() {
return extendedBuilder.build();
}

/**
* @return the version this datanode was initially created with
*/
public int getInitialVersion() {
return initialVersion;
}

public void setInitialVersion(int initialVersion) {
this.initialVersion = initialVersion;
}

/**
* @return the version this datanode was last started with
*/
public int getCurrentVersion() {
return currentVersion;
}

public void setCurrentVersion(int currentVersion) {
this.currentVersion = currentVersion;
}

@Override
public String toString() {
return uuid.toString() + "{" +
Expand Down Expand Up @@ -498,6 +532,8 @@ public static final class Builder {
private String buildDate;
private HddsProtos.NodeOperationalState persistedOpState;
private long persistedOpStateExpiryEpochSec = 0;
private int initialVersion;
private int currentVersion = DatanodeVersions.CURRENT_VERSION;

/**
* Default private constructor. To create Builder instance use
Expand Down Expand Up @@ -683,6 +719,16 @@ public Builder setPersistedOpStateExpiry(long expiry){
return this;
}

public Builder setInitialVersion(int v) {
this.initialVersion = v;
return this;
}

public Builder setCurrentVersion(int v) {
this.currentVersion = v;
return this;
}

/**
* Builds and returns DatanodeDetails instance.
*
Expand All @@ -695,7 +741,8 @@ public DatanodeDetails build() {
}
DatanodeDetails dn = new DatanodeDetails(id, ipAddress, hostName,
networkLocation, ports, certSerialId, version, setupTime, revision,
buildDate, persistedOpState, persistedOpStateExpiryEpochSec);
buildDate, persistedOpState, persistedOpStateExpiryEpochSec,
initialVersion, currentVersion);
if (networkName != null) {
dn.setNetworkName(networkName);
}
Expand Down Expand Up @@ -724,7 +771,7 @@ public static final class Port {
* Ports that are supported in DataNode.
*/
public enum Name {
STANDALONE, RATIS, REST, REPLICATION;
STANDALONE, RATIS, REST, REPLICATION, RATIS_ADMIN, RATIS_SERVER;

public static final Set<Name> ALL_PORTS = ImmutableSet.copyOf(
Name.values());
Expand Down
36 changes: 21 additions & 15 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
View file
Expand Up @@ -31,6 +31,7 @@
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
import org.apache.hadoop.hdds.ratis.retrypolicy.RetryPolicyCreator;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
Expand All @@ -40,9 +41,9 @@
import org.apache.ratis.RaftConfigKeys;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.client.RaftClientConfigKeys;
import org.apache.ratis.conf.Parameters;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
import org.apache.ratis.grpc.GrpcFactory;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.RaftGroup;
Expand Down Expand Up @@ -94,30 +95,32 @@ public static UUID toDatanodeId(RaftProtos.RaftPeerProto peerId) {
return toDatanodeId(RaftPeerId.valueOf(peerId.getId()));
}

private static String toRaftPeerAddressString(DatanodeDetails id) {
return id.getIpAddress() + ":" +
id.getPort(DatanodeDetails.Port.Name.RATIS).getValue();
private static String toRaftPeerAddress(DatanodeDetails id, Port.Name port) {
return id.getIpAddress() + ":" + id.getPort(port).getValue();
}

public static RaftPeerId toRaftPeerId(DatanodeDetails id) {
return RaftPeerId.valueOf(toRaftPeerIdString(id));
}

public static RaftPeer toRaftPeer(DatanodeDetails id) {
return RaftPeer.newBuilder()
.setId(toRaftPeerId(id))
.setAddress(toRaftPeerAddressString(id))
.build();
return raftPeerBuilderFor(id).build();
}

public static RaftPeer toRaftPeer(DatanodeDetails id, int priority) {
return RaftPeer.newBuilder()
.setId(toRaftPeerId(id))
.setAddress(toRaftPeerAddressString(id))
return raftPeerBuilderFor(id)
.setPriority(priority)
.build();
}

private static RaftPeer.Builder raftPeerBuilderFor(DatanodeDetails dn) {
return RaftPeer.newBuilder()
.setId(toRaftPeerId(dn))
.setAddress(toRaftPeerAddress(dn, Port.Name.RATIS_SERVER))
.setAdminAddress(toRaftPeerAddress(dn, Port.Name.RATIS_ADMIN))
.setClientAddress(toRaftPeerAddress(dn, Port.Name.RATIS));
}

private static List<RaftPeer> toRaftPeers(Pipeline pipeline) {
return toRaftPeers(pipeline.getNodes());
}
Expand Down Expand Up @@ -224,7 +227,9 @@ private static RaftClient newRaftClient(RpcType rpcType, RaftPeerId leader,

// TODO: GRPC TLS only for now, netty/hadoop RPC TLS support later.
if (tlsConfig != null && rpcType == SupportedRpcType.GRPC) {
builder.setParameters(GrpcFactory.newRaftParameters(tlsConfig));
Parameters parameters = new Parameters();
GrpcConfigKeys.Client.setTlsConf(parameters, tlsConfig);
builder.setParameters(parameters);
}
return builder.build();
}
Expand Down Expand Up @@ -254,9 +259,10 @@ private static boolean isClientConfig(String key) {
}

private static boolean isGrpcClientConfig(String key) {
return key.startsWith(GrpcConfigKeys.PREFIX) && !key
.startsWith(GrpcConfigKeys.TLS.PREFIX) && !key
.startsWith(GrpcConfigKeys.Server.PREFIX);
return key.startsWith(GrpcConfigKeys.PREFIX) &&
!key.startsWith(GrpcConfigKeys.TLS.PREFIX) &&
!key.startsWith(GrpcConfigKeys.Admin.PREFIX) &&
!key.startsWith(GrpcConfigKeys.Server.PREFIX);
}
/**
* Set all server properties matching with prefix
Expand Down
12 changes: 12 additions & 0 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
View file
Expand Up @@ -64,6 +64,18 @@ public final class OzoneConfigKeys {
public static final String DFS_CONTAINER_RATIS_IPC_PORT =
"dfs.container.ratis.ipc";
public static final int DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT = 9858;
/**
* Ratis Port where containers listen to admin requests.
*/
public static final String DFS_CONTAINER_RATIS_ADMIN_PORT =
"dfs.container.ratis.admin.port";
public static final int DFS_CONTAINER_RATIS_ADMIN_PORT_DEFAULT = 9857;
/**
* Ratis Port where containers listen to server-to-server requests.
*/
public static final String DFS_CONTAINER_RATIS_SERVER_PORT =
"dfs.container.ratis.server.port";
public static final int DFS_CONTAINER_RATIS_SERVER_PORT_DEFAULT = 9856;

/**
* When set to true, allocate a random free port for ozone container, so that
Expand Down
14 changes: 13 additions & 1 deletion hadoop-hdds/common/src/main/resources/ozone-default.xml
View file
Expand Up @@ -167,8 +167,20 @@
<property>
<name>dfs.container.ratis.ipc</name>
<value>9858</value>
<tag>OZONE, CONTAINER, PIPELINE, RATIS</tag>
<description>The ipc port number of container for clients.</description>
</property>
<property>
<name>dfs.container.ratis.admin.port</name>
<value>9857</value>
<tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag>
<description>The ipc port number of container.</description>
<description>The ipc port number of container for admin requests.</description>
</property>
<property>
<name>dfs.container.ratis.server.port</name>
<value>9856</value>
<tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag>
<description>The ipc port number of container for server-server communication.</description>
</property>
<property>
<name>dfs.container.ratis.ipc.random.port</name>
Expand Down
8 changes: 7 additions & 1 deletion hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
View file
Expand Up @@ -33,6 +33,7 @@

import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.hdds.DFSConfigKeysLegacy;
import org.apache.hadoop.hdds.DatanodeVersions;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.cli.GenericCli;
Expand Down Expand Up @@ -209,6 +210,7 @@ public void start() {
datanodeDetails.setRevision(
HddsVersionInfo.HDDS_VERSION_INFO.getRevision());
datanodeDetails.setBuildDate(HddsVersionInfo.HDDS_VERSION_INFO.getDate());
datanodeDetails.setCurrentVersion(DatanodeVersions.CURRENT_VERSION);
TracingUtil.initTracing(
"HddsDatanodeService." + datanodeDetails.getUuidString()
.substring(0, 8), conf);
Expand Down Expand Up @@ -434,7 +436,11 @@ private DatanodeDetails initializeDatanodeDetails()
} else {
// There is no datanode.id file, this might be the first time datanode
// is started.
return DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build();
DatanodeDetails details = DatanodeDetails.newBuilder()
.setUuid(UUID.randomUUID()).build();
details.setInitialVersion(DatanodeVersions.CURRENT_VERSION);
details.setCurrentVersion(DatanodeVersions.CURRENT_VERSION);
return details;
}
}

Expand Down

0 comments on commit 375da4d

Please sign in to comment.