From 7bf99c1a988bcdfcb639f8a1355cc2a027fa9ef9 Mon Sep 17 00:00:00 2001 From: Ali Reza Zamani Zadeh Najari Date: Tue, 26 Nov 2019 15:26:10 -0800 Subject: [PATCH] Add java API to create cluster with CloudConfig In this commit the below APIs have been added. 1- AddCluster with CloudCOnfig API. 2- addCloudConfig to existing cluster. 3- Update CloudConfig to update existing Cloud Config. Several tests have been added to test these APIs. --- .../java/org/apache/helix/ConfigAccessor.java | 4 +- .../java/org/apache/helix/HelixAdmin.java | 15 +- .../apache/helix/manager/zk/ZKHelixAdmin.java | 28 ++++ .../org/apache/helix/model/CloudConfig.java | 149 ++++++------------ .../org/apache/helix/tools/ClusterSetup.java | 17 +- .../org/apache/helix/TestConfigAccessor.java | 66 ++++++-- .../helix/manager/zk/TestZkHelixAdmin.java | 69 ++++++++ .../org/apache/helix/mock/MockHelixAdmin.java | 11 ++ .../helix/model/cloud/TestCloudConfig.java | 60 ++++--- .../apache/helix/tools/TestClusterSetup.java | 88 ++++++++++- 10 files changed, 368 insertions(+), 139 deletions(-) diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java index 264fd3f126..97bfb34058 100644 --- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java +++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java @@ -591,7 +591,7 @@ public ClusterConfig getClusterConfig(String clusterName) { * @return The instance of {@link CloudConfig} */ public CloudConfig getCloudConfig(String clusterName) { - if (!ZKUtil.isClusterSetup(clusterName, zkClient)) { + if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException( String.format("Failed to get config. cluster: %s is not setup.", clusterName)); } @@ -604,7 +604,7 @@ public CloudConfig getCloudConfig(String clusterName) { return null; } - return new CloudConfig(record); + return new CloudConfig.Builder(record).build(); } /** diff --git a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java index a11b235fb0..c3dc8b565d 100644 --- a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java +++ b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java @@ -22,7 +22,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; - +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ClusterConstraints; import org.apache.helix.model.ClusterConstraints.ConstraintType; import org.apache.helix.model.ConstraintItem; @@ -380,6 +380,19 @@ void addStateModelDef(String clusterName, String stateModelDef, StateModelDefini */ void dropResource(String clusterName, String resourceName); + /** + * Add cloud config to the cluster. + * @param clusterName + * @param cloudConfig + */ + void addCloudConfig(String clusterName, CloudConfig cloudConfig); + + /** + * Remove the Cloud Config for specific cluster + * @param clusterName + */ + void removeCloudConfig(String clusterName); + /** * Get a list of state model definitions in a cluster * @param clusterName diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java index 0a978e542c..9d8d406d71 100644 --- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java +++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java @@ -59,6 +59,7 @@ import org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy; import org.apache.helix.manager.zk.client.HelixZkClient; import org.apache.helix.manager.zk.client.SharedZkClientFactory; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.ClusterConstraints; import org.apache.helix.model.ClusterConstraints.ConstraintType; @@ -1025,6 +1026,33 @@ public void dropResource(String clusterName, String resourceName) { accessor.removeProperty(keyBuilder.resourceConfig(resourceName)); } + @Override + public void addCloudConfig(String clusterName, CloudConfig cloudConfig) { + logger.info("Add CloudConfig to cluster {}, CloudConfig is {}.", clusterName, + cloudConfig.toString()); + + if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { + throw new HelixException("cluster " + clusterName + " is not setup yet"); + } + + CloudConfig.Builder builder = new CloudConfig.Builder(cloudConfig); + CloudConfig cloudConfigBuilder = builder.build(); + + ZKHelixDataAccessor accessor = + new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_zkClient)); + Builder keyBuilder = accessor.keyBuilder(); + accessor.setProperty(keyBuilder.cloudConfig(), cloudConfigBuilder); + } + + @Override + public void removeCloudConfig(String clusterName) { + logger.info("Remove Cloud Config for cluster {}.", clusterName); + HelixDataAccessor accessor = + new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_zkClient)); + Builder keyBuilder = accessor.keyBuilder(); + accessor.removeProperty(keyBuilder.cloudConfig()); + } + @Override public List getStateModelDefs(String clusterName) { return _zkClient.getChildren(PropertyPathBuilder.stateModelDef(clusterName)); diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java index c8ab6eb890..f6c279c9c6 100644 --- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java +++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java @@ -30,6 +30,9 @@ * Cloud configurations */ public class CloudConfig extends HelixProperty { + + public static final String CLOUD_CONFIG_KW = "CloudConfig"; + /** * Configurable characteristics of a cloud. * NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the @@ -52,39 +55,22 @@ public enum CloudConfigProperty { /** * Instantiate the CloudConfig for the cloud - * @param cluster */ - public CloudConfig(String cluster) { - super(cluster); + private CloudConfig() { + super(CLOUD_CONFIG_KW); } /** - * Instantiate with a pre-populated record - * @param record a ZNRecord corresponding to a cloud configuration + * The constructor from the ZNRecord. + * @param record */ - public CloudConfig(ZNRecord record) { - super(record); + private CloudConfig(ZNRecord record) { + super(CLOUD_CONFIG_KW); + _record.setSimpleFields(record.getSimpleFields()); + _record.setListFields(record.getListFields()); + _record.setMapFields(record.getMapFields()); } - /** - * Instantiate the config using each field individually. - * Users should use CloudConfig.Builder to create CloudConfig. - * @param cluster - * @param enabled - * @param cloudID - */ - public CloudConfig(String cluster, boolean enabled, CloudProvider cloudProvider, String cloudID, - List cloudInfoSource, String cloudProcessorName) { - super(cluster); - _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled); - _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name()); - _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID); - if (cloudProvider.equals(CloudProvider.CUSTOMIZED)) { - _record - .setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), cloudProcessorName); - _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSource); - } - } /** * Enable/Disable the CLOUD_ENABLED field. @@ -134,15 +120,6 @@ public List getCloudInfoSources() { return _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name()); } - /** - * Set the CLOUD_INFO_PROCESSOR_NAME field. - * @param cloudInfoProcessorName - */ - public void setCloudInfoFProcessorName(String cloudInfoProcessorName) { - _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), - cloudInfoProcessorName); - } - /** * Get the CLOUD_INFO_PROCESSOR_NAME field. * @return CLOUD_INFO_PROCESSOR_NAME field. @@ -151,14 +128,6 @@ public String getCloudInfoProcessorName() { return _record.getSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name()); } - /** - * Set the CLOUD_PROVIDER field. - * @param cloudProvider - */ - public void setCloudProvider(CloudProvider cloudProvider) { - _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name()); - } - /** * Get the CLOUD_PROVIDER field. * @return CLOUD_PROVIDER field. @@ -167,32 +136,28 @@ public String getCloudProvider() { return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name()); } + public static class Builder { - private String _clusterName = null; - private CloudProvider _cloudProvider; - private boolean _cloudEnabled = DEFAULT_CLOUD_ENABLED; - private String _cloudID; - private List _cloudInfoSources; - private String _cloudInfoProcessorName; + private ZNRecord _record; public CloudConfig build() { validate(); - return new CloudConfig(_clusterName, _cloudEnabled, _cloudProvider, _cloudID, - _cloudInfoSources, _cloudInfoProcessorName); + return new CloudConfig(_record); } /** * Default constructor */ public Builder() { + _record = new ZNRecord(CLOUD_CONFIG_KW); } /** - * Constructor with Cluster Name as input - * @param clusterName + * Instantiate with a pre-populated record + * @param record a ZNRecord corresponding to a cloud configuration */ - public Builder(String clusterName) { - _clusterName = clusterName; + public Builder(ZNRecord record) { + _record = record; } /** @@ -200,89 +165,75 @@ public Builder(String clusterName) { * @param cloudConfig */ public Builder(CloudConfig cloudConfig) { - _cloudEnabled = cloudConfig.isCloudEnabled(); - _cloudProvider = CloudProvider.valueOf(cloudConfig.getCloudProvider()); - _cloudID = cloudConfig.getCloudID(); - _cloudInfoSources = cloudConfig.getCloudInfoSources(); - _cloudInfoProcessorName = cloudConfig.getCloudInfoProcessorName(); - } - - public Builder setClusterName(String v) { - _clusterName = v; - return this; + _record = cloudConfig.getRecord(); } public Builder setCloudEnabled(boolean isEnabled) { - _cloudEnabled = isEnabled; + _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), isEnabled); return this; } public Builder setCloudProvider(CloudProvider cloudProvider) { - _cloudProvider = cloudProvider; + _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name()); return this; } - public Builder setCloudID(String v) { - _cloudID = v; + public Builder setCloudID(String cloudID) { + _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID); return this; } - public Builder setCloudInfoSources(List v) { - _cloudInfoSources = v; + public Builder setCloudInfoSources(List cloudInfoSources) { + _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSources); return this; } - public Builder addCloudInfoSource(String v) { - if (_cloudInfoSources == null) { - _cloudInfoSources = new ArrayList(); + public Builder addCloudInfoSource(String cloudInfoSource) { + if (_record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name()) == null) { + _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), new ArrayList()); } - _cloudInfoSources.add(v); + List cloudInfoSourcesList = _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name()); + cloudInfoSourcesList.add(cloudInfoSource); + _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSourcesList); return this; } - public Builder setCloudInfoProcessorName(String v) { - _cloudInfoProcessorName = v; + public Builder setCloudInfoProcessorName(String cloudInfoProcessorName) { + _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), + cloudInfoProcessorName); return this; } - public String getClusterName() { - return _clusterName; - } - - public CloudProvider getCloudProvider() { - return _cloudProvider; + public String getCloudProvider() { + return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name()); } public boolean getCloudEnabled() { - return _cloudEnabled; + return _record.getBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), + DEFAULT_CLOUD_ENABLED); } public String getCloudID() { - return _cloudID; + return _record.getSimpleField(CloudConfigProperty.CLOUD_ID.name()); } public List getCloudInfoSources() { - return _cloudInfoSources; + return _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name()); } public String getCloudInfoProcessorName() { - return _cloudInfoProcessorName; + return _record.getSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name()); } private void validate() { - if (_cloudEnabled) { - if (_cloudID == null) { - throw new HelixException( - "This Cloud Configuration is Invalid. The CloudID is missing from the config."); - } - if (_cloudProvider == null) { + if (this.getCloudProvider() == null) { + throw new HelixException( + "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config."); + } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) { + if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null + || this.getCloudInfoSources().size() == 0) { throw new HelixException( - "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config."); - } else if (_cloudProvider == CloudProvider.CUSTOMIZED) { - if (_cloudInfoProcessorName == null || _cloudInfoSources == null || _cloudInfoSources.size() == 0) { - throw new HelixException( - "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources"); - } + "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources"); } } } diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java index 5d5f86429d..d1dc6d36ba 100644 --- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java +++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java @@ -41,7 +41,9 @@ import org.apache.helix.HelixConstants; import org.apache.helix.HelixException; import org.apache.helix.PropertyKey.Builder; +import org.apache.helix.SystemPropertyKeys; import org.apache.helix.ZNRecord; +import org.apache.helix.cloud.constants.CloudProvider; import org.apache.helix.manager.zk.ZKHelixAdmin; import org.apache.helix.manager.zk.ZKHelixDataAccessor; import org.apache.helix.manager.zk.ZNRecordSerializer; @@ -49,6 +51,7 @@ import org.apache.helix.manager.zk.client.HelixZkClient; import org.apache.helix.manager.zk.client.SharedZkClientFactory; import org.apache.helix.model.BuiltInStateModelDefinitions; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.ClusterConstraints; import org.apache.helix.model.ClusterConstraints.ConstraintType; @@ -176,15 +179,23 @@ public void close() { } } - public void addCluster(String clusterName, boolean overwritePrevious) { + public void addCluster(String clusterName, boolean overwritePrevious, CloudConfig cloudConfig) + throws HelixException { _admin.addCluster(clusterName, overwritePrevious); - for (BuiltInStateModelDefinitions def : BuiltInStateModelDefinitions.values()) { addStateModelDef(clusterName, def.getStateModelDefinition().getId(), - def.getStateModelDefinition(), overwritePrevious); + def.getStateModelDefinition(), overwritePrevious); + } + + if (cloudConfig != null) { + _admin.addCloudConfig(clusterName, cloudConfig); } } + public void addCluster(String clusterName, boolean overwritePrevious) { + addCluster(clusterName, overwritePrevious, null); + } + public void activateCluster(String clusterName, String grandCluster, boolean enable) { if (enable) { _admin.addClusterToGrandCluster(clusterName, grandCluster); diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java index 0fa05c367d..e62bf81e56 100644 --- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java +++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java @@ -19,14 +19,18 @@ * under the License. */ +import java.util.ArrayList; import java.util.Date; import java.util.List; +import org.apache.helix.cloud.constants.CloudProvider; import org.apache.helix.manager.zk.ZKHelixAdmin; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ConfigScope; import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty; import org.apache.helix.model.InstanceConfig; import org.apache.helix.model.builder.ConfigScopeBuilder; +import org.apache.helix.tools.ClusterSetup; import org.testng.Assert; import org.testng.annotations.Test; @@ -68,9 +72,8 @@ public void testBasic() throws Exception { Assert.assertEquals(resourceConfigValue, "resourceConfigValue"); // partition scope config - ConfigScope partitionScope = - new ConfigScopeBuilder().forCluster(clusterName).forResource("testResource") - .forPartition("testPartition").build(); + ConfigScope partitionScope = new ConfigScopeBuilder().forCluster(clusterName) + .forResource("testResource").forPartition("testPartition").build(); configAccessor.set(partitionScope, "partitionConfigKey", "partitionConfigValue"); String partitionConfigValue = configAccessor.get(partitionScope, "partitionConfigKey"); Assert.assertEquals(partitionConfigValue, "partitionConfigValue"); @@ -113,9 +116,8 @@ public void testBasic() throws Exception { "should be [HELIX_ENABLED, HELIX_ENABLED_TIMESTAMP, HELIX_HOST, HELIX_PORT, participantConfigKey]"); Assert.assertEquals(keys.get(4), "participantConfigKey"); - keys = - configAccessor.getKeys(ConfigScopeProperty.PARTITION, clusterName, "testResource", - "testPartition"); + keys = configAccessor.getKeys(ConfigScopeProperty.PARTITION, clusterName, "testResource", + "testPartition"); Assert.assertEquals(keys.size(), 1, "should be [partitionConfigKey]"); Assert.assertEquals(keys.get(0), "partitionConfigKey"); @@ -183,8 +185,8 @@ public void testSetNonexistentParticipantConfig() throws Exception { try { configAccessor.set(participantScope, "participantConfigKey", "participantConfigValue"); - Assert - .fail("Except fail to set participant-config because participant: localhost_12918 is not added to cluster yet"); + Assert.fail( + "Except fail to set participant-config because participant: localhost_12918 is not added to cluster yet"); } catch (HelixException e) { // OK } @@ -193,8 +195,8 @@ public void testSetNonexistentParticipantConfig() throws Exception { try { configAccessor.set(participantScope, "participantConfigKey", "participantConfigValue"); } catch (Exception e) { - Assert - .fail("Except succeed to set participant-config because participant: localhost_12918 has been added to cluster"); + Assert.fail( + "Except succeed to set participant-config because participant: localhost_12918 has been added to cluster"); } String participantConfigValue = configAccessor.get(participantScope, "participantConfigKey"); @@ -204,4 +206,48 @@ public void testSetNonexistentParticipantConfig() throws Exception { configAccessor.close(); System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis())); } + + @Test + public void testUpdateCloudConfig() throws Exception { + ClusterSetup _clusterSetup = new ClusterSetup(ZK_ADDR); + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder(); + cloudConfigInitBuilder.setCloudEnabled(true); + cloudConfigInitBuilder.setCloudID("TestCloudID"); + List sourceList = new ArrayList(); + sourceList.add("TestURL"); + cloudConfigInitBuilder.setCloudInfoSources(sourceList); + cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor"); + cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED); + CloudConfig cloudConfigInit = cloudConfigInitBuilder.build(); + + _clusterSetup.addCluster(clusterName, false, cloudConfigInit); + + // Read CloudConfig from Zookeeper and check the content + ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient); + CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID"); + List listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor"); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); + + // Change the processor name and check if processor name has been changed in Zookeeper. + cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor2"); + cloudConfigInit = cloudConfigInitBuilder.build(); + ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient); + admin.addCloudConfig(clusterName, cloudConfigInit); + + cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID"); + listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor2"); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); + } } diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java index 20acef4e2e..69fa558578 100644 --- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java +++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.helix.BaseDataAccessor; +import org.apache.helix.ConfigAccessor; import org.apache.helix.HelixAdmin; import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixException; @@ -39,7 +40,9 @@ import org.apache.helix.TestHelper; import org.apache.helix.ZNRecord; import org.apache.helix.ZkUnitTestBase; +import org.apache.helix.cloud.constants.CloudProvider; import org.apache.helix.examples.MasterSlaveStateModelFactory; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ClusterConstraints; import org.apache.helix.model.ClusterConstraints.ConstraintAttribute; import org.apache.helix.model.ClusterConstraints.ConstraintType; @@ -506,4 +509,70 @@ public void testLegacyEnableDisablePartition() { .getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(), 2); } + + @Test + public void testAddCloudConfig() { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + HelixAdmin admin = new ZKHelixAdmin(_gZkClient); + admin.addCluster(clusterName, true); + + CloudConfig.Builder builder = new CloudConfig.Builder(); + builder.setCloudEnabled(true); + builder.setCloudID("TestID"); + builder.addCloudInfoSource("TestURL"); + builder.setCloudProvider(CloudProvider.CUSTOMIZED); + builder.setCloudInfoProcessorName("TestProcessor"); + CloudConfig cloudConfig = builder.build(); + + admin.addCloudConfig(clusterName, cloudConfig); + + // Read CloudConfig from Zookeeper and check the content + ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient); + CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID"); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); + List listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor"); + } + + + @Test + public void testRemoveCloudConfig() throws Exception { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + HelixAdmin admin = new ZKHelixAdmin(_gZkClient); + admin.addCluster(clusterName, true); + + CloudConfig.Builder builder = new CloudConfig.Builder(); + builder.setCloudEnabled(true); + builder.setCloudID("TestID"); + builder.addCloudInfoSource("TestURL"); + builder.setCloudProvider(CloudProvider.CUSTOMIZED); + builder.setCloudInfoProcessorName("TestProcessor"); + CloudConfig cloudConfig = builder.build(); + + admin.addCloudConfig(clusterName, cloudConfig); + + // Read CloudConfig from Zookeeper and check the content + ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient); + CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID"); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); + List listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor"); + + // Remove Cloud Config and make sure it has been removed from Zookeeper + admin.removeCloudConfig(clusterName); + cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertNull(cloudConfigFromZk); + } } diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java index b299ab1908..76e219b488 100644 --- a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java +++ b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java @@ -31,6 +31,7 @@ import org.apache.helix.PropertyPathBuilder; import org.apache.helix.PropertyType; import org.apache.helix.ZNRecord; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.ClusterConstraints; import org.apache.helix.model.ConstraintItem; @@ -313,6 +314,16 @@ public boolean isInMaintenanceMode(String clusterName) { } + @Override + public void addCloudConfig(String clusterName, CloudConfig cloudConfig) { + + } + + @Override + public void removeCloudConfig(String clusterName) { + + } + @Override public List getStateModelDefs(String clusterName) { return null; } diff --git a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java index ee83011dc6..01945de357 100644 --- a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java +++ b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java @@ -57,20 +57,22 @@ public void testCloudConfigNull() { } @Test(dependsOnMethods = "testCloudConfigNull") - public void testCloudConfig() { + public void testCloudConfig() throws Exception { String className = getShortClassName(); String clusterName = "CLUSTER_" + className; TestHelper.setupEmptyCluster(_gZkClient, clusterName); // Create dummy CloudConfig object - CloudConfig cloudConfig = new CloudConfig(clusterName); - cloudConfig.setCloudEnabled(true); - cloudConfig.setCloudProvider(CloudProvider.AZURE); - cloudConfig.setCloudID("TestID"); + CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder(); + cloudConfigBuilder.setCloudEnabled(true); + cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE); + cloudConfigBuilder.setCloudID("TestID"); List infoURL = new ArrayList(); infoURL.add("TestURL"); - cloudConfig.setCloudInfoSource(infoURL); - cloudConfig.setCloudInfoFProcessorName("TestProcessor"); + cloudConfigBuilder.setCloudInfoSources(infoURL); + cloudConfigBuilder.setCloudInfoProcessorName("TestProcessor"); + + CloudConfig cloudConfig = cloudConfigBuilder.build(); // Write the CloudConfig to Zookeeper ZKHelixDataAccessor accessor = @@ -84,8 +86,8 @@ public void testCloudConfig() { Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name()); Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().size(), 1); - Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().get(0), "TestURL"); Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor"); } @@ -93,7 +95,7 @@ public void testCloudConfig() { public void testUnverifiedCloudConfigBuilder() { String className = getShortClassName(); String clusterName = "CLUSTER_" + className; - CloudConfig.Builder builder = new CloudConfig.Builder(clusterName); + CloudConfig.Builder builder = new CloudConfig.Builder(); builder.setCloudEnabled(true); // Verify will fail because cloudID has net been defined. CloudConfig cloudConfig = builder.build(); @@ -103,7 +105,7 @@ public void testUnverifiedCloudConfigBuilder() { public void testUnverifiedCloudConfigBuilderEmptySources() { String className = getShortClassName(); String clusterName = "CLUSTER_" + className; - CloudConfig.Builder builder = new CloudConfig.Builder(clusterName); + CloudConfig.Builder builder = new CloudConfig.Builder(); builder.setCloudEnabled(true); builder.setCloudProvider(CloudProvider.CUSTOMIZED); builder.setCloudID("TestID"); @@ -115,9 +117,7 @@ public void testUnverifiedCloudConfigBuilderEmptySources() { @Test(expectedExceptions = HelixException.class) public void testUnverifiedCloudConfigBuilderWithoutProcessor() { - String className = getShortClassName(); - String clusterName = "CLUSTER_" + className; - CloudConfig.Builder builder = new CloudConfig.Builder(clusterName); + CloudConfig.Builder builder = new CloudConfig.Builder(); builder.setCloudEnabled(true); builder.setCloudProvider(CloudProvider.CUSTOMIZED); builder.setCloudID("TestID"); @@ -132,7 +132,7 @@ public void testCloudConfigBuilder() { String className = getShortClassName(); String clusterName = "CLUSTER_" + className; TestHelper.setupEmptyCluster(_gZkClient, clusterName); - CloudConfig.Builder builder = new CloudConfig.Builder(clusterName); + CloudConfig.Builder builder = new CloudConfig.Builder(); builder.setCloudEnabled(true); builder.setCloudProvider(CloudProvider.CUSTOMIZED); builder.setCloudID("TestID"); @@ -142,8 +142,7 @@ public void testCloudConfigBuilder() { // Check builder getter methods Assert.assertTrue(builder.getCloudEnabled()); - Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED); - Assert.assertEquals(builder.getClusterName(), clusterName); + Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); Assert.assertEquals(builder.getCloudID(), "TestID"); List listUrlFromBuilder = builder.getCloudInfoSources(); Assert.assertEquals(listUrlFromBuilder.size(), 2); @@ -175,15 +174,14 @@ public void testCloudConfigBuilderAzureProvider() { String className = getShortClassName(); String clusterName = "CLUSTER_" + className; TestHelper.setupEmptyCluster(_gZkClient, clusterName); - CloudConfig.Builder builder = new CloudConfig.Builder(clusterName); + CloudConfig.Builder builder = new CloudConfig.Builder(); builder.setCloudEnabled(true); builder.setCloudProvider(CloudProvider.AZURE); builder.setCloudID("TestID"); - builder.setCloudInfoProcessorName("TestProcessor"); // Check builder getter methods Assert.assertTrue(builder.getCloudEnabled()); - Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE); + Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE.name()); CloudConfig cloudConfig = builder.build(); @@ -198,7 +196,29 @@ public void testCloudConfigBuilderAzureProvider() { Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name()); - // Since CloudProvider is not CUSTOMIZED, CloudInfoProcessor will be null. + // Since user does not set the CloudInfoProcessorName, this field will be null. Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName()); + + // Checking the set method in CloudConfig + cloudConfig.setCloudEnabled(false); + accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig); + cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertFalse(cloudConfigFromZk.isCloudEnabled()); + + cloudConfig.setCloudEnabled(true); + cloudConfig.setCloudID("TestID2"); + List sourceList = new ArrayList(); + sourceList.add("TestURL0"); + sourceList.add("TestURL1"); + cloudConfig.setCloudInfoSource(sourceList); + accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig); + + cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID2"); + List listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL0"); + Assert.assertEquals(listUrlFromZk.get(1), "TestURL1"); } } diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java index df3f341472..68195bd32c 100644 --- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java +++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java @@ -19,10 +19,13 @@ * under the License. */ +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.List; import org.apache.helix.BaseDataAccessor; +import org.apache.helix.ConfigAccessor; import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixException; import org.apache.helix.PropertyKey; @@ -31,15 +34,15 @@ import org.apache.helix.TestHelper; import org.apache.helix.ZNRecord; import org.apache.helix.ZkUnitTestBase; +import org.apache.helix.cloud.constants.CloudProvider; import org.apache.helix.manager.zk.ZKHelixAdmin; import org.apache.helix.manager.zk.ZKHelixDataAccessor; import org.apache.helix.manager.zk.ZNRecordSerializer; import org.apache.helix.manager.zk.ZkBaseDataAccessor; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty; import org.apache.helix.model.IdealState; import org.apache.helix.model.LiveInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.AssertJUnit; import org.testng.annotations.AfterClass; @@ -48,8 +51,6 @@ import org.testng.annotations.Test; public class TestClusterSetup extends ZkUnitTestBase { - private static Logger LOG = LoggerFactory.getLogger(TestClusterSetup.class); - protected static final String CLUSTER_NAME = "TestClusterSetup"; protected static final String TEST_DB = "TestDB"; protected static final String INSTANCE_PREFIX = "instance_"; @@ -436,4 +437,83 @@ public void testDisableResource() throws Exception { System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis())); } + + @Test(expectedExceptions = HelixException.class) + public void testAddClusterWithInvalidCloudConfig() throws Exception { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder(); + cloudConfigInitBuilder.setCloudEnabled(true); + List sourceList = new ArrayList(); + sourceList.add("TestURL"); + cloudConfigInitBuilder.setCloudInfoSources(sourceList); + cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED); + + CloudConfig cloudConfigInit = cloudConfigInitBuilder.build(); + + + // Since setCloudInfoProcessorName is missing, this add cluster call will throw an exception + _clusterSetup.addCluster(clusterName, false, cloudConfigInit); + } + + @Test(dependsOnMethods = "testAddClusterWithInvalidCloudConfig") + public void testAddClusterWithValidCloudConfig() throws Exception { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder(); + cloudConfigInitBuilder.setCloudEnabled(true); + cloudConfigInitBuilder.setCloudID("TestID"); + List sourceList = new ArrayList(); + sourceList.add("TestURL"); + cloudConfigInitBuilder.setCloudInfoSources(sourceList); + cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessorName"); + cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED); + + CloudConfig cloudConfigInit = cloudConfigInitBuilder.build(); + + _clusterSetup.addCluster(clusterName, false, cloudConfigInit); + + // Read CloudConfig from Zookeeper and check the content + ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient); + CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID"); + List listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName"); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); + } + + + @Test(dependsOnMethods = "testAddClusterWithValidCloudConfig") + public void testAddClusterAzureProvider() throws Exception { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder(); + cloudConfigInitBuilder.setCloudEnabled(true); + cloudConfigInitBuilder.setCloudID("TestID"); + cloudConfigInitBuilder.setCloudProvider(CloudProvider.AZURE); + + CloudConfig cloudConfigInit = cloudConfigInitBuilder.build(); + + _clusterSetup.addCluster(clusterName, false, cloudConfigInit); + + // Read CloudConfig from Zookeeper and check the content + ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient); + CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID"); + List listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + + // Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null. + Assert.assertNull(listUrlFromZk); + Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName()); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name()); + } }