Skip to content

Commit

Permalink
#6085 [YW] Master brought up in read replica cluster
Browse files Browse the repository at this point in the history 
Summary:
- For nodes in Read Replica cluster limited next functions:
  1) Start node action (when the universe is in masters under replicated state, doesn't
     start the master process);
  2) Add node action (when the unierse is in masters under replicated state, creates a node
     with t-server running only);
  3) Start Master action (for nodes in Read Replica cluster it is strictly not available).

Test Plan:
Preparation (common steps for scenarios below):
  Create a universe with a primary cluster (RF=3, nodes N1-3) and a read replica cluster (nodes N4-6).

Scenario 1 (from the issue description):
  1. Stop and release a master+tserver node N1 in the primary cluster through the node dropdown.
  2. Stop and release a tserver node N4 in the read replica cluster through the node dropdown.
  3. "Add node" N4 in the read replica cluster through the node dropdown.
Expected result:
  Master is not started in the read replica cluster (on N4).

Scenario 2:
  1. Stop node N4 through the node dropdown.
  2. Stop node N1 through the node dropdown.
  3. Start node N4 through the node dropdown.
Expected result:
  Master is not started in the read replica cluster (on N4).

Scenario 3.
  1. Stop node N1 through the node dropdown.
  2. Check the dropdown menu of node N4.
Expected result:
  There is no action 'Start Master'.

Reviewers: sanketh

Reviewed By: sanketh

Subscribers: jenkins-bot, yugaware, wesley, daniel

Differential Revision: https://phabricator.dev.yugabyte.com/D9688
  • Loading branch information
@SergeyPotachev
SergeyPotachev committed Oct 22, 2020
1 parent 6b5ba0e commit dcb9577
Show file tree
Hide file tree
Showing 8 changed files with 386 additions and 73 deletions.
7 changes: 7 additions & 0 deletions managed/src/main/java/com/yugabyte/yw/common/Util.java
View file
Expand Up @@ -9,6 +9,8 @@
import com.google.common.collect.Lists;
import com.google.common.net.HostAndPort;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.ClusterType;
import com.yugabyte.yw.models.Customer;
import com.yugabyte.yw.models.Universe;
import com.yugabyte.yw.models.helpers.NodeDetails;
Expand Down Expand Up @@ -245,6 +247,11 @@ private static Map<UUID, Integer> getAZToStoppedNodesCountMap(Set<NodeDetails> n
*/
public static boolean areMastersUnderReplicated(NodeDetails currentNode,
Universe universe) {
Cluster cluster = universe.getCluster(currentNode.placementUuid);
if ((cluster == null) || (cluster.clusterType != ClusterType.PRIMARY)) {
return false;
}

UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails();
Set<NodeDetails> nodes = universeDetails.nodeDetailsSet;
long numMasters = getNumMasters(nodes);
Expand Down
13 changes: 6 additions & 7 deletions managed/src/main/java/com/yugabyte/yw/models/Universe.java
View file
Expand Up @@ -42,6 +42,7 @@
import com.yugabyte.yw.commissioner.tasks.UniverseDefinitionTaskBase.ServerType;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.ClusterType;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent;
import com.yugabyte.yw.models.helpers.NodeDetails;

Expand Down Expand Up @@ -716,20 +717,18 @@ public boolean isNodeActionAllowed(String nodeName, NodeActionType action) {
NodeDetails node = getNode(nodeName);
Cluster curCluster = getCluster(node.placementUuid);

if (node.isMaster && (action == NodeActionType.STOP || action == NodeActionType.REMOVE)) {
if (node.isMaster && (action == NodeActionType.STOP || action == NodeActionType.REMOVE)
&& (curCluster.clusterType == ClusterType.PRIMARY)) {
long numMasterNodesUp = universeDetails.getNodesInCluster(curCluster.uuid).stream()
.filter((n) -> n.isMaster && n.state == NodeDetails.NodeState.Live)
.count();
.filter((n) -> n.isMaster && n.state == NodeDetails.NodeState.Live).count();
if (numMasterNodesUp <= (curCluster.userIntent.replicationFactor + 1) / 2) {
return false;
}
}

if (action == NodeActionType.START_MASTER) {
if (node.isMaster || (node.state != NodeDetails.NodeState.Live)
|| !Util.areMastersUnderReplicated(node, this)) {
return false;
}
return (!node.isMaster && (node.state == NodeDetails.NodeState.Live)
&& Util.areMastersUnderReplicated(node, this));
}

return node.getAllowedActions().contains(action);
Expand Down
122 changes: 92 additions & 30 deletions managed/src/test/java/com/yugabyte/yw/commissioner/tasks/AddNodeToUniverseTest.java
View file
Expand Up @@ -5,12 +5,12 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.net.HostAndPort;
import com.yugabyte.yw.commissioner.Commissioner;
import com.yugabyte.yw.commissioner.tasks.params.NodeTaskParams;
import com.yugabyte.yw.common.ApiUtils;
import com.yugabyte.yw.common.ShellProcessHandler;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent;
import com.yugabyte.yw.models.AvailabilityZone;
import com.yugabyte.yw.models.Region;
Expand Down Expand Up @@ -80,7 +80,7 @@ public void setUp() {
ApiUtils.mockUniverseUpdater(userIntent, true /* setMasters */));

// Change one of the nodes' state to removed.
setDefaultNodeState(NodeState.Removed);
setDefaultNodeState(defaultUniverse, NodeState.Removed, DEFAULT_NODE_NAME);

mockClient = mock(YBClient.class);
when(mockClient.waitForServer(any(), anyLong())).thenReturn(true);
Expand All @@ -97,31 +97,33 @@ public void setUp() {
modifyBL = mock(ModifyMasterClusterConfigBlacklist.class);
}

private void setDefaultNodeState(final NodeState desiredState) {
private void setDefaultNodeState(Universe universe, final NodeState desiredState,
String nodeName) {
Universe.UniverseUpdater updater = new Universe.UniverseUpdater() {
@Override
public void run(Universe universe) {
UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails();
Set<NodeDetails> nodes = universeDetails.nodeDetailsSet;
for (NodeDetails node : nodes) {
if (node.nodeName.equals(DEFAULT_NODE_NAME)) {
if (node.nodeName.equals(nodeName)) {
node.state = desiredState;
break;
}
}
universe.setUniverseDetails(universeDetails);
}
};
Universe.saveDetails(defaultUniverse.universeUUID, updater);
Universe.saveDetails(universe.universeUUID, updater);
}

private TaskInfo submitTask(String nodeName, int version) {
private TaskInfo submitTask(UUID universeUUID, String nodeName, int version) {
Universe universe = Universe.get(universeUUID);
NodeTaskParams taskParams = new NodeTaskParams();
taskParams.clusters.add(defaultUniverse.getUniverseDetails().getPrimaryCluster());
taskParams.clusters.addAll(universe.getUniverseDetails().clusters);

taskParams.expectedUniverseVersion = version;
taskParams.nodeName = nodeName;
taskParams.universeUUID = defaultUniverse.universeUUID;
taskParams.universeUUID = universe.universeUUID;
try {
UUID taskUUID = commissioner.submit(TaskType.AddNodeToUniverse, taskParams);
return waitForTask(taskUUID);
Expand Down Expand Up @@ -211,7 +213,7 @@ private void assertAddNodeSequence(Map<Integer, List<TaskInfo>> subTasksByPositi
if (masterUnderReplicated) {
for (TaskType taskType: WITH_MASTER_UNDER_REPLICATED) {
List<TaskInfo> tasks = subTasksByPosition.get(position);
assertEquals(taskType, tasks.get(0).getTaskType());
assertEquals("At position: " + position, taskType, tasks.get(0).getTaskType());
JsonNode expectedResults =
WITH_MASTER_UNDER_REPLICATED_RESULTS.get(position);
List<JsonNode> taskDetails = tasks.stream()
Expand All @@ -224,7 +226,7 @@ private void assertAddNodeSequence(Map<Integer, List<TaskInfo>> subTasksByPositi
for (TaskType taskType: ADD_NODE_TASK_SEQUENCE) {
List<TaskInfo> tasks = subTasksByPosition.get(position);
assertEquals(1, tasks.size());
assertEquals(taskType, tasks.get(0).getTaskType());
assertEquals("At position: " + position, taskType, tasks.get(0).getTaskType());
JsonNode expectedResults =
ADD_NODE_TASK_EXPECTED_RESULTS.get(position);
List<JsonNode> taskDetails = tasks.stream()
Expand All @@ -239,34 +241,37 @@ private void assertAddNodeSequence(Map<Integer, List<TaskInfo>> subTasksByPositi

@Test
public void testAddNodeSuccess() {
TaskInfo taskInfo = submitTask(DEFAULT_NODE_NAME, 3);
TaskInfo taskInfo = submitTask(defaultUniverse.universeUUID, DEFAULT_NODE_NAME, 3);
verify(mockNodeManager, times(4)).nodeCommand(any(), any());
List<TaskInfo> subTasks = taskInfo.getSubTasks();
Map<Integer, List<TaskInfo>> subTasksByPosition =
subTasks.stream().collect(Collectors.groupingBy(w -> w.getPosition()));
assertAddNodeSequence(subTasksByPosition, false);
}

@Test
public void testAddNodeWithUnderReplicatedMaster() {
// Change one of the nodes' state to removed.
Universe.UniverseUpdater updater = new Universe.UniverseUpdater() {
public void run(Universe universe) {
UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails();
Set<NodeDetails> nodes = universeDetails.nodeDetailsSet;
for (NodeDetails node : nodes) {
if (node.nodeName.equals(DEFAULT_NODE_NAME)) {
node.isMaster = false;
break;
// Change one of the nodes' state to removed.
private final Universe.UniverseUpdater underReplicatedMasterUpdater =
new Universe.UniverseUpdater() {
public void run(Universe universe) {
UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails();
Set<NodeDetails> nodes = universeDetails.nodeDetailsSet;
for (NodeDetails node : nodes) {
if (node.nodeName.equals(DEFAULT_NODE_NAME)) {
node.isMaster = false;
break;
}
}
universe.setUniverseDetails(universeDetails);
}
universe.setUniverseDetails(universeDetails);
}
};
Universe.saveDetails(defaultUniverse.universeUUID, updater);
};

@Test
public void testAddNodeWithUnderReplicatedMaster() {
verify(mockNodeManager, never()).nodeCommand(any(), any());
Universe.saveDetails(defaultUniverse.universeUUID, underReplicatedMasterUpdater);

TaskInfo taskInfo = submitTask(DEFAULT_NODE_NAME, 4);
// 5 calls for setting up the server, 6 calls for setting the conf files, 1 to check cronjobs.
TaskInfo taskInfo = submitTask(defaultUniverse.universeUUID, DEFAULT_NODE_NAME, 4);
// 5 calls for setting up the server and then 6 calls for setting the conf files.
verify(mockNodeManager, times(12)).nodeCommand(any(), any());
List<TaskInfo> subTasks = taskInfo.getSubTasks();
Map<Integer, List<TaskInfo>> subTasksByPosition =
Expand All @@ -276,9 +281,66 @@ public void run(Universe universe) {

@Test
public void testAddUnknownNode() {
TaskInfo taskInfo = submitTask("host-n9", 3);
TaskInfo taskInfo = submitTask(defaultUniverse.universeUUID, "host-n9", 3);
verify(mockNodeManager, times(0)).nodeCommand(any(), any());
assertEquals(TaskInfo.State.Failure, taskInfo.getTaskState());
}
}

@Test
public void testAddNodeWithUnderReplicatedMaster_WithReadOnlyCluster_NodeFromPrimary() {
Universe universe = createUniverse("Demo");
universe = Universe.saveDetails(universe.universeUUID,
ApiUtils.mockUniverseUpdaterWithInactiveAndReadReplicaNodes(false, 1));
setDefaultGFlags(universe);

// Change one of the nodes' state to removed.
setDefaultNodeState(universe, NodeState.Removed, DEFAULT_NODE_NAME);

TaskInfo taskInfo = submitTask(universe.universeUUID, DEFAULT_NODE_NAME, 4);
verify(mockNodeManager, times(12)).nodeCommand(any(), any());
List<TaskInfo> subTasks = taskInfo.getSubTasks();
Map<Integer, List<TaskInfo>> subTasksByPosition = subTasks.stream()
.collect(Collectors.groupingBy(w -> w.getPosition()));
assertAddNodeSequence(subTasksByPosition, true /* Master start is expected */);
}

@Test
public void testAddNodeWithUnderReplicatedMaster_WithReadOnlyCluster_NodeFromReadReplica() {
Universe universe = createUniverse("Demo");
universe = Universe.saveDetails(universe.universeUUID,
ApiUtils.mockUniverseUpdaterWithInactiveAndReadReplicaNodes(false, 1));
setDefaultGFlags(universe);

// Change one of the nodes' state to removed.
setDefaultNodeState(universe, NodeState.Removed, "yb-tserver-0");

TaskInfo taskInfo = submitTask(universe.universeUUID, "yb-tserver-0", 4);
verify(mockNodeManager, times(4)).nodeCommand(any(), any());
List<TaskInfo> subTasks = taskInfo.getSubTasks();
Map<Integer, List<TaskInfo>> subTasksByPosition = subTasks.stream()
.collect(Collectors.groupingBy(w -> w.getPosition()));
assertAddNodeSequence(subTasksByPosition, false /* Master start is unexpected */);
}

private void setDefaultGFlags(Universe universe) {
Universe.UniverseUpdater updater = new Universe.UniverseUpdater() {
@Override
public void run(Universe universe) {
UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails();
Map<String, String> gflags = new HashMap<>();
gflags.put("foo", "bar");

Cluster primaryCluster = universeDetails.getPrimaryCluster();
primaryCluster.userIntent.masterGFlags = gflags;
primaryCluster.userIntent.tserverGFlags = gflags;

List<Cluster> readOnlyClusters = universeDetails.getReadOnlyClusters();
if (readOnlyClusters.size() > 0) {
readOnlyClusters.get(0).userIntent.masterGFlags = gflags;
readOnlyClusters.get(0).userIntent.tserverGFlags = gflags;
}
}
};
Universe.saveDetails(universe.universeUUID, updater);
}
}
19 changes: 17 additions & 2 deletions managed/src/test/java/com/yugabyte/yw/commissioner/tasks/StartMasterOnNodeTest.java
View file
Expand Up @@ -72,8 +72,7 @@ public void setUp() {
}

private TaskInfo submitTask(NodeTaskParams taskParams, String nodeName) {
taskParams.clusters
.add(Universe.get(taskParams.universeUUID).getUniverseDetails().getPrimaryCluster());
taskParams.clusters.addAll(Universe.get(taskParams.universeUUID).getUniverseDetails().clusters);
taskParams.expectedUniverseVersion = 2;
taskParams.nodeName = nodeName;
try {
Expand Down Expand Up @@ -180,4 +179,20 @@ public void testStartMasterOnNodeIfUnderReplicatedMasterAndNodeIsRemoved() {
verify(mockNodeManager, times(1)).nodeCommand(any(), any());
assertEquals(TaskInfo.State.Failure, taskInfo.getTaskState());
}

@Test
public void testStartMasterOnNodeIfNodeInReadOnlyCluster() {
Universe universe = createUniverse("DemoX");
universe = Universe.saveDetails(universe.universeUUID,
ApiUtils.mockUniverseUpdaterWithInactiveAndReadReplicaNodes(false, 3));

NodeTaskParams taskParams = new NodeTaskParams();
taskParams.universeUUID = universe.universeUUID;

// Node "yb-tserver-0" is in Read Only cluster.
TaskInfo taskInfo = submitTask(taskParams, "yb-tserver-0");
// one nodeCommand invocation is made from instanceExists()
verify(mockNodeManager, times(1)).nodeCommand(any(), any());
assertEquals(TaskInfo.State.Failure, taskInfo.getTaskState());
}
}

0 comments on commit dcb9577

Please sign in to comment.