Skip to content
Permalink
Browse files
Add instance disable reason (#1993) (#2004)
Add instance disable reason
  • Loading branch information
xyuanlu committed Mar 31, 2022
1 parent 1e5aa3c commit e39255c47236f5c5f2bffcf15153756cec892ec5
Showing 9 changed files with 209 additions and 31 deletions.
@@ -0,0 +1,10 @@
package org.apache.helix.constants;

public class InstanceConstants {
public enum InstanceDisabledType {
CLOUD_EVENT,
USER_OPERATION,
DEFAULT_INSTANCE_DISABLE_TYPE,
INSTANCE_NOT_DISABLED
}
}
@@ -26,6 +26,7 @@
import org.apache.helix.api.status.ClusterManagementMode;
import org.apache.helix.api.status.ClusterManagementModeRequest;
import org.apache.helix.api.topology.ClusterTopology;
import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -280,6 +281,16 @@ void addResource(String clusterName, String resourceName, int numPartitions, Str
*/
void enableInstance(String clusterName, String instanceName, boolean enabled);

/**
* @param clusterName
* @param instanceName
* @param enabled
* @param reason set additional string description on why the instance is disabled when
* <code>enabled</code> is false.
*/
void enableInstance(String clusterName, String instanceName, boolean enabled,
InstanceConstants.InstanceDisabledType disabledType, String reason);

/**
* Batch enable/disable instances in a cluster
* By default, all the instances are enabled
@@ -56,6 +56,7 @@
import org.apache.helix.api.status.ClusterManagementMode;
import org.apache.helix.api.status.ClusterManagementModeRequest;
import org.apache.helix.api.topology.ClusterTopology;
import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy;
import org.apache.helix.controller.rebalancer.util.WagedValidationUtil;
import org.apache.helix.controller.rebalancer.waged.WagedRebalancer;
@@ -341,17 +342,24 @@ public boolean setInstanceConfig(String clusterName, String instanceName,
@Override
public void enableInstance(final String clusterName, final String instanceName,
final boolean enabled) {
enableInstance(clusterName, instanceName, enabled, null, null);
}

@Override
public void enableInstance(final String clusterName, final String instanceName,
final boolean enabled, InstanceConstants.InstanceDisabledType disabledType, String reason) {
logger.info("{} instance {} in cluster {}.", enabled ? "Enable" : "Disable", instanceName,
clusterName);
BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<>(_zkClient);
enableSingleInstance(clusterName, instanceName, enabled, baseAccessor);
enableSingleInstance(clusterName, instanceName, enabled, baseAccessor, disabledType, reason);
// TODO: Reenable this after storage node bug fixed.
// enableBatchInstances(clusterName, Collections.singletonList(instanceName), enabled, baseAccessor);

}

@Override
public void enableInstance(String clusterName, List<String> instances, boolean enabled) {
// TODO: Considering adding another batched API with disabled type and reason.
// TODO: Reenable this after storage node bug fixed.
if (true) {
throw new HelixException("Current batch enable/disable instances are temporarily disabled!");
@@ -361,7 +369,7 @@ public void enableInstance(String clusterName, List<String> instances, boolean e
BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<>(_zkClient);
if (enabled) {
for (String instance : instances) {
enableSingleInstance(clusterName, instance, enabled, baseAccessor);
enableSingleInstance(clusterName, instance, enabled, baseAccessor, null, null);
}
}
enableBatchInstances(clusterName, instances, enabled, baseAccessor);
@@ -1861,7 +1869,8 @@ public void enableBatchMessageMode(String clusterName, String resourceName, bool
}

private void enableSingleInstance(final String clusterName, final String instanceName,
final boolean enabled, BaseDataAccessor<ZNRecord> baseAccessor) {
final boolean enabled, BaseDataAccessor<ZNRecord> baseAccessor,
InstanceConstants.InstanceDisabledType disabledType, String reason) {
String path = PropertyPathBuilder.instanceConfig(clusterName, instanceName);

if (!baseAccessor.exists(path, 0)) {
@@ -1879,6 +1888,14 @@ public ZNRecord update(ZNRecord currentData) {

InstanceConfig config = new InstanceConfig(currentData);
config.setInstanceEnabled(enabled);
if (!enabled) {
if (reason != null) {
config.setInstanceDisabledReason(reason);
}
if (disabledType != null) {
config.setInstanceDisabledType(disabledType);
}
}
return config.getRecord();
}
}, AccessOption.PERSISTENT);
@@ -31,6 +31,7 @@

import org.apache.helix.HelixException;
import org.apache.helix.HelixProperty;
import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.controller.rebalancer.topology.Topology;
import org.apache.helix.util.HelixUtil;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
@@ -50,6 +51,8 @@ public enum InstanceConfigProperty {
HELIX_ZONE_ID,
HELIX_ENABLED,
HELIX_ENABLED_TIMESTAMP,
HELIX_DISABLED_REASON,
HELIX_DISABLED_TYPE,
HELIX_DISABLED_PARTITION,
TAG_LIST,
INSTANCE_WEIGHT,
@@ -275,13 +278,59 @@ public boolean getInstanceEnabled() {

/**
* Set the enabled state of the instance
* If user enables the instance, HELIX_DISABLED_REASON filed will be removed.
*
* @param enabled true to enable, false to disable
*/
public void setInstanceEnabled(boolean enabled) {
_record.setBooleanField(InstanceConfigProperty.HELIX_ENABLED.toString(), enabled);
_record.setLongField(InstanceConfigProperty.HELIX_ENABLED_TIMESTAMP.name(),
System.currentTimeMillis());
if (enabled) {
_record.getSimpleFields().remove(InstanceConfigProperty.HELIX_DISABLED_REASON.toString());
_record.getSimpleFields().remove(InstanceConfigProperty.HELIX_DISABLED_TYPE.toString());
}
}

/**
* Set the instance disabled reason when instance is disabled.
* It will be a no-op when instance is enabled.
*/
public void setInstanceDisabledReason(String disabledReason) {
if (!getInstanceEnabled()) {
_record.setSimpleField(InstanceConfigProperty.HELIX_DISABLED_REASON.toString(), disabledReason);
}
}

/**
* Set the instance disabled type when instance is disabled.
* It will be a no-op when instance is enabled.
*/
public void setInstanceDisabledType(InstanceConstants.InstanceDisabledType disabledType) {
if (!getInstanceEnabled()) {
_record.setSimpleField(InstanceConfigProperty.HELIX_DISABLED_TYPE.toString(),
disabledType.toString());
}
}

/**
* @return Return instance disabled reason. Default is am empty string.
*/
public String getInstanceDisabledReason() {
return _record.getStringField(InstanceConfigProperty.HELIX_DISABLED_REASON.toString(), "");
}

/**
*
* @return Return instance disabled type (org.apache.helix.constants.InstanceConstants.InstanceDisabledType)
* Default is am empty string.
*/
public String getInstanceDisabledType() {
if (getInstanceEnabled()) {
return InstanceConstants.InstanceDisabledType.INSTANCE_NOT_DISABLED.toString();
}
return _record.getStringField(InstanceConfigProperty.HELIX_DISABLED_TYPE.toString(),
InstanceConstants.InstanceDisabledType.DEFAULT_INSTANCE_DISABLE_TYPE.toString());
}

/**
@@ -47,8 +47,10 @@
import org.apache.helix.ZkUnitTestBase;
import org.apache.helix.api.exceptions.HelixConflictException;
import org.apache.helix.api.status.ClusterManagementMode;
import org.apache.helix.api.status.ClusterManagementModeRequest;
import org.apache.helix.api.topology.ClusterTopology;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.controller.rebalancer.waged.WagedRebalancer;
import org.apache.helix.examples.MasterSlaveStateModelFactory;
import org.apache.helix.integration.manager.ClusterControllerManager;
@@ -73,7 +75,6 @@
import org.apache.helix.model.StateModelDefinition;
import org.apache.helix.model.builder.ConstraintItemBuilder;
import org.apache.helix.model.builder.HelixConfigScopeBuilder;
import org.apache.helix.api.status.ClusterManagementModeRequest;
import org.apache.helix.participant.StateMachineEngine;
import org.apache.helix.tools.StateModelConfigGenerator;
import org.apache.helix.zookeeper.api.client.RealmAwareZkClient;
@@ -176,6 +177,20 @@ public void testZkHelixAdmin() {
// OK
}

Assert.assertTrue(
tool.getInstanceConfig(clusterName, instanceName).getInstanceDisabledReason().isEmpty());
String disableReason = "Reason";
tool.enableInstance(clusterName, instanceName, false,
InstanceConstants.InstanceDisabledType.CLOUD_EVENT, disableReason);
Assert.assertTrue(tool.getInstanceConfig(clusterName, instanceName).getInstanceDisabledReason()
.equals(disableReason));
tool.enableInstance(clusterName, instanceName, true,
InstanceConstants.InstanceDisabledType.CLOUD_EVENT, disableReason);
Assert.assertTrue(
tool.getInstanceConfig(clusterName, instanceName).getInstanceDisabledReason().isEmpty());
Assert.assertEquals(tool.getInstanceConfig(clusterName, instanceName).getInstanceDisabledType(),
InstanceConstants.InstanceDisabledType.INSTANCE_NOT_DISABLED.toString());

dummyList.remove("bar");
dummyList.add("baz");
config = tool.getInstanceConfig(clusterName, instanceName);
@@ -31,7 +31,9 @@
import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.PropertyType;
import org.apache.helix.api.status.ClusterManagementMode;
import org.apache.helix.api.status.ClusterManagementModeRequest;
import org.apache.helix.api.topology.ClusterTopology;
import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
@@ -45,7 +47,6 @@
import org.apache.helix.model.MaintenanceSignal;
import org.apache.helix.model.ResourceConfig;
import org.apache.helix.model.StateModelDefinition;
import org.apache.helix.api.status.ClusterManagementModeRequest;
import org.apache.helix.zookeeper.datamodel.ZNRecord;

public class MockHelixAdmin implements HelixAdmin {
@@ -267,16 +268,30 @@ public void removeFromIdealState(String clusterName, String resourceName, IdealS

@Override
public void enableInstance(String clusterName, String instanceName, boolean enabled) {
enableInstance(clusterName, instanceName, enabled, null, null);
}

@Override
public void enableInstance(String clusterName, String instanceName, boolean enabled,
InstanceConstants.InstanceDisabledType disabledType, String reason) {
String instanceConfigsPath = PropertyPathBuilder.instanceConfig(clusterName);
if (!_baseDataAccessor.exists(instanceConfigsPath, 0)) {
_baseDataAccessor.create(instanceConfigsPath, new ZNRecord(instanceName), 0);
}

String instanceConfigPath = instanceConfigsPath + "/" + instanceName;

ZNRecord record = (ZNRecord) _baseDataAccessor.get(instanceConfigPath, null, 0);
ZNRecord record = (ZNRecord) _baseDataAccessor.get(instanceConfigPath, null, 0);
InstanceConfig instanceConfig = new InstanceConfig(record);
instanceConfig.setInstanceEnabled(enabled);
if (!enabled) {
if (reason != null) {
instanceConfig.setInstanceDisabledReason(reason);
}
if (disabledType != null) {
instanceConfig.setInstanceDisabledType(disabledType);
}
}
_baseDataAccessor.set(instanceConfigPath, instanceConfig.getRecord(), 0);
}

@@ -23,17 +23,11 @@
import java.util.Map;

import com.google.common.collect.ImmutableMap;
import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.testng.Assert;
import org.testng.annotations.Test;

/**
* Created with IntelliJ IDEA.
* User: zzhang
* Date: 3/19/13
* Time: 5:28 PM
* To change this template use File | Settings | File Templates.
*/
public class TestInstanceConfig {
@Test
public void testNotCheckingHostPortExistence() {
@@ -45,13 +39,42 @@ public void testNotCheckingHostPortExistence() {
@Test
public void testGetParsedDomain() {
InstanceConfig instanceConfig = new InstanceConfig(new ZNRecord("id"));
instanceConfig.setDomain("cluster=myCluster,zone=myZone1,rack=myRack,host=hostname,instance=instance001");
instanceConfig
.setDomain("cluster=myCluster,zone=myZone1,rack=myRack,host=hostname,instance=instance001");

Map<String, String> parsedDomain = instanceConfig.getDomainAsMap();
Assert.assertEquals(parsedDomain.size(), 5);
Assert.assertEquals(parsedDomain.get("zone"), "myZone1");
}

@Test
public void testSetInstanceEnableWithReason() {
InstanceConfig instanceConfig = new InstanceConfig(new ZNRecord("id"));
instanceConfig.setInstanceEnabled(true);
instanceConfig.setInstanceDisabledReason("NoShowReason");
instanceConfig.setInstanceDisabledType(InstanceConstants.InstanceDisabledType.USER_OPERATION);

Assert.assertEquals(instanceConfig.getRecord().getSimpleFields()
.get(InstanceConfig.InstanceConfigProperty.HELIX_ENABLED.toString()), "true");
Assert.assertEquals(instanceConfig.getRecord().getSimpleFields()
.get(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_REASON.toString()), null);
Assert.assertEquals(instanceConfig.getRecord().getSimpleFields()
.get(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_TYPE.toString()), null);


instanceConfig.setInstanceEnabled(false);
String reasonCode = "ReasonCode";
instanceConfig.setInstanceDisabledReason(reasonCode);
instanceConfig.setInstanceDisabledType(InstanceConstants.InstanceDisabledType.USER_OPERATION);
Assert.assertEquals(instanceConfig.getRecord().getSimpleFields()
.get(InstanceConfig.InstanceConfigProperty.HELIX_ENABLED.toString()), "false");
Assert.assertEquals(instanceConfig.getRecord().getSimpleFields()
.get(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_REASON.toString()), reasonCode);
Assert.assertEquals(instanceConfig.getInstanceDisabledReason(), reasonCode);
Assert.assertEquals(instanceConfig.getInstanceDisabledType(),
InstanceConstants.InstanceDisabledType.USER_OPERATION.toString());
}

@Test
public void testGetParsedDomainEmptyDomain() {
InstanceConfig instanceConfig = new InstanceConfig(new ZNRecord("id"));
@@ -48,6 +48,7 @@
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.model.CurrentState;
import org.apache.helix.model.Error;
@@ -369,7 +370,8 @@ public Response addInstance(@PathParam("clusterId") String clusterId,
@POST
public Response updateInstance(@PathParam("clusterId") String clusterId,
@PathParam("instanceName") String instanceName, @QueryParam("command") String command,
String content) {
@QueryParam("instanceDisabledType") String disabledType,
@QueryParam("instanceDisabledReason") String disabledReason, String content) {
Command cmd;
try {
cmd = Command.valueOf(command);
@@ -385,21 +387,30 @@ public Response updateInstance(@PathParam("clusterId") String clusterId,
}

switch (cmd) {
case enable:
admin.enableInstance(clusterId, instanceName, true);
break;
case disable:
admin.enableInstance(clusterId, instanceName, false);
break;
case enable:
admin.enableInstance(clusterId, instanceName, true);
break;
case disable:
InstanceConstants.InstanceDisabledType disabledTypeEnum = null;
if (disabledType != null) {
try {
disabledTypeEnum = InstanceConstants.InstanceDisabledType.valueOf(disabledType);
} catch (IllegalArgumentException ex) {
return badRequest("Invalid instanceDisabledType!");
}
}
admin.enableInstance(clusterId, instanceName, false, disabledTypeEnum, disabledReason);
break;

case reset:
case resetPartitions:
if (!validInstance(node, instanceName)) {
return badRequest("Instance names are not match!");
}
admin.resetPartition(clusterId, instanceName,
node.get(PerInstanceProperties.resource.name()).textValue(), (List<String>) OBJECT_MAPPER
.readValue(node.get(PerInstanceProperties.partitions.name()).toString(),
case reset:
case resetPartitions:
if (!validInstance(node, instanceName)) {
return badRequest("Instance names are not match!");
}
admin.resetPartition(clusterId, instanceName,
node.get(PerInstanceProperties.resource.name()).textValue(),
(List<String>) OBJECT_MAPPER
.readValue(node.get(PerInstanceProperties.partitions.name()).toString(),
OBJECT_MAPPER.getTypeFactory()
.constructCollectionType(List.class, String.class)));
break;

0 comments on commit e39255c

Please sign in to comment.