Skip to content

[Ambari 23549] Blueprint configuration processor updates for HDFS NameNode Federation #985

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rnettleton merged 11 commits into from
Apr 16, 2018
Merged

[Ambari 23549] Blueprint configuration processor updates for HDFS NameNode Federation #985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
2dc2966
Adding property substitution support for various Federation-related p…
Apr 6, 2018
5ad9191
Merge remote-tracking branch 'upstream/trunk' into BP-FEDERATION-PROP…
Apr 6, 2018
77a78d0
Minor changes to Blueprint Configuration Processor for NameNode Feder…
Apr 11, 2018
ff7de04
Merge remote-tracking branch 'upstream/trunk' into BP-FEDERATION-PROP…
Apr 11, 2018
8d15169
Updates to unit tests and minor refactorings
Apr 11, 2018
bfecbb8
Updates to unit tests and minor changes.
Apr 11, 2018
8b2a677
Merge remote-tracking branch 'upstream/trunk' into BP-FEDERATION-PROP…
Apr 11, 2018
781687a
Merge remote-tracking branch 'upstream/trunk' into AMBARI-23549-trunk
Apr 12, 2018
6e5d267
Merge remote-tracking branch 'upstream/trunk' into AMBARI-23549-trunk
Apr 13, 2018
0129290
Updated error handling for cluster name and new unit test assertions.
Apr 13, 2018
b5330f5
Merge remote-tracking branch 'upstream/trunk' into AMBARI-23549-trunk
Apr 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 46 additions & 15 deletions ...in/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,10 @@ public class BlueprintConfigurationProcessor {
private final static String HDFS_ACTIVE_NAMENODE_SET_PROPERTY_NAME = "dfs_ha_initial_namenode_active_set";
private final static String HDFS_STANDBY_NAMENODE_SET_PROPERTY_NAME = "dfs_ha_initial_namenode_standby_set";

private final static String HDFS_HA_INITIAL_CLUSTER_ID_PROPERTY_NAME = "dfs_ha_initial_cluster_id";

private final static String HADOOP_ENV_CONFIG_TYPE_NAME = "hadoop-env";


/**
* Single host topology updaters
Expand Down Expand Up @@ -185,10 +189,10 @@ private PropertyFilter[] getExportPropertyFilters (Map<Long, Set<String>> authTo
new SimplePropertyNameExportFilter("ldap-url", "kerberos-env"),
new SimplePropertyNameExportFilter("container_dn", "kerberos-env"),
new SimplePropertyNameExportFilter("domains", "krb5-conf"),
new SimplePropertyNameExportFilter("dfs_ha_initial_namenode_active", "hadoop-env"),
new SimplePropertyNameExportFilter("dfs_ha_initial_namenode_standby", "hadoop-env"),
new SimplePropertyNameExportFilter(HDFS_ACTIVE_NAMENODE_SET_PROPERTY_NAME, "hadoop-env"),
new SimplePropertyNameExportFilter(HDFS_STANDBY_NAMENODE_SET_PROPERTY_NAME, "hadoop-env"),
new SimplePropertyNameExportFilter("dfs_ha_initial_namenode_active", HADOOP_ENV_CONFIG_TYPE_NAME),
new SimplePropertyNameExportFilter("dfs_ha_initial_namenode_standby", HADOOP_ENV_CONFIG_TYPE_NAME),
new SimplePropertyNameExportFilter(HDFS_ACTIVE_NAMENODE_SET_PROPERTY_NAME, HADOOP_ENV_CONFIG_TYPE_NAME),
new SimplePropertyNameExportFilter(HDFS_STANDBY_NAMENODE_SET_PROPERTY_NAME, HADOOP_ENV_CONFIG_TYPE_NAME),
new StackPropertyTypeFilter(),
new KerberosAuthToLocalRulesFilter(authToLocalPerClusterMap)};
}
Expand Down Expand Up @@ -429,13 +433,13 @@ public Set<String> doUpdateForClusterCreate() throws ConfigurationTopologyExcept
// set the properties that configure which namenode is active,
// and which is a standby node in this HA deployment
Iterator<String> nnHostIterator = nnHosts.iterator();
clusterConfig.setProperty("hadoop-env", "dfs_ha_initial_namenode_active", nnHostIterator.next());
clusterConfig.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", nnHostIterator.next());
clusterConfig.setProperty(HADOOP_ENV_CONFIG_TYPE_NAME, "dfs_ha_initial_namenode_active", nnHostIterator.next());
clusterConfig.setProperty(HADOOP_ENV_CONFIG_TYPE_NAME, "dfs_ha_initial_namenode_standby", nnHostIterator.next());

configTypesUpdated.add("hadoop-env");
configTypesUpdated.add(HADOOP_ENV_CONFIG_TYPE_NAME);
}
} else {
if (!isPropertySet(clusterProps, "hadoop-env", HDFS_ACTIVE_NAMENODE_SET_PROPERTY_NAME) && !isPropertySet(clusterProps, "hadoop-env", HDFS_STANDBY_NAMENODE_SET_PROPERTY_NAME)) {
if (!isPropertySet(clusterProps, HADOOP_ENV_CONFIG_TYPE_NAME, HDFS_ACTIVE_NAMENODE_SET_PROPERTY_NAME) && !isPropertySet(clusterProps, HADOOP_ENV_CONFIG_TYPE_NAME, HDFS_STANDBY_NAMENODE_SET_PROPERTY_NAME)) {
// multiple nameservices indicates an HDFS NameNode Federation install
// process each nameservice to determine the active/standby nodes
LOG.info("Processing multiple HDFS NameService instances, which indicates a NameNode Federation deployment");
Expand Down Expand Up @@ -473,9 +477,15 @@ public Set<String> doUpdateForClusterCreate() throws ConfigurationTopologyExcept

// set the properties what configure the NameNode Active/Standby status for each nameservice
if (!activeNameNodeHostnames.isEmpty() && !standbyNameNodeHostnames.isEmpty()) {
clusterConfig.setProperty("hadoop-env", HDFS_ACTIVE_NAMENODE_SET_PROPERTY_NAME, String.join(",", activeNameNodeHostnames));
clusterConfig.setProperty("hadoop-env", HDFS_STANDBY_NAMENODE_SET_PROPERTY_NAME, String.join(",", standbyNameNodeHostnames));
configTypesUpdated.add("hadoop-env");
clusterConfig.setProperty(HADOOP_ENV_CONFIG_TYPE_NAME, HDFS_ACTIVE_NAMENODE_SET_PROPERTY_NAME, String.join(",", activeNameNodeHostnames));
clusterConfig.setProperty(HADOOP_ENV_CONFIG_TYPE_NAME, HDFS_STANDBY_NAMENODE_SET_PROPERTY_NAME, String.join(",", standbyNameNodeHostnames));

// also set the clusterID property, required for Federation installs of HDFS
if (!isPropertySet(clusterProps, HADOOP_ENV_CONFIG_TYPE_NAME, HDFS_HA_INITIAL_CLUSTER_ID_PROPERTY_NAME)) {
clusterConfig.setProperty(HADOOP_ENV_CONFIG_TYPE_NAME, HDFS_HA_INITIAL_CLUSTER_ID_PROPERTY_NAME, getClusterName());
}

configTypesUpdated.add(HADOOP_ENV_CONFIG_TYPE_NAME);
} else {
LOG.warn("Error in processing the set of active/standby namenodes in this federated cluster, please check hdfs-site configuration");
}
Expand All @@ -495,6 +505,21 @@ public Set<String> doUpdateForClusterCreate() throws ConfigurationTopologyExcept
return configTypesUpdated;
}

private String getClusterName() throws ConfigurationTopologyException {
String clusterNameToReturn = null;
try {
clusterNameToReturn = clusterTopology.getAmbariContext().getClusterName(clusterTopology.getClusterId());
} catch (AmbariException e) {
throw new ConfigurationTopologyException("Cluster name could not obtained, this may indicate a deployment or configuration error.", e);
}

if (clusterNameToReturn == null) {
throw new ConfigurationTopologyException("Cluster name could not obtained, this may indicate a deployment or configuration error.");
}

return clusterNameToReturn;
}

private void trimProperties(Configuration clusterConfig, ClusterTopology clusterTopology) {
Blueprint blueprint = clusterTopology.getBlueprint();
Stack stack = blueprint.getStack();
Expand Down Expand Up @@ -948,13 +973,19 @@ private Map<String, Map<String, PropertyUpdater>> createMapOfNameNodeHAUpdaters(
Map<String, String> hdfsSiteConfig = clusterTopology.getConfiguration().getFullProperties().get("hdfs-site");
// generate the property names based on the current HA config for the NameNode deployments
for (String nameService : parseNameServices(hdfsSiteConfig)) {
final String journalEditsDirPropertyName = "dfs.namenode.shared.edits.dir." + nameService;
// register an updater for the nameservice-specific shared edits dir
hdfsSiteUpdatersForAvailability.put(journalEditsDirPropertyName, new MultipleHostTopologyUpdater("JOURNALNODE", ';', false, false, true));

for (String nameNode : parseNameNodes(nameService, hdfsSiteConfig)) {
final String httpsPropertyName = "dfs.namenode.https-address." + nameService + "." + nameNode;
hdfsSiteUpdatersForAvailability.put(httpsPropertyName, new SingleHostTopologyUpdater("NAMENODE"));
final String httpPropertyName = "dfs.namenode.http-address." + nameService + "." + nameNode;
hdfsSiteUpdatersForAvailability.put(httpPropertyName, new SingleHostTopologyUpdater("NAMENODE"));
final String rpcPropertyName = "dfs.namenode.rpc-address." + nameService + "." + nameNode;
hdfsSiteUpdatersForAvailability.put(rpcPropertyName, new SingleHostTopologyUpdater("NAMENODE"));
final String serviceRpcPropertyName = "dfs.namenode.servicerpc-address." + nameService + "." + nameNode;
hdfsSiteUpdatersForAvailability.put(serviceRpcPropertyName, new SingleHostTopologyUpdater("NAMENODE"));
}
}
return highAvailabilityUpdaters;
Expand Down Expand Up @@ -1045,7 +1076,7 @@ static boolean isHiveServerHAEnabled(Map<String, Map<String, String>> configProp
* false if the initial active namenode property has not been configured
*/
static boolean isNameNodeHAInitialActiveNodeSet(Map<String, Map<String, String>> configProperties) {
return configProperties.containsKey("hadoop-env") && configProperties.get("hadoop-env").containsKey("dfs_ha_initial_namenode_active");
return configProperties.containsKey(HADOOP_ENV_CONFIG_TYPE_NAME) && configProperties.get(HADOOP_ENV_CONFIG_TYPE_NAME).containsKey("dfs_ha_initial_namenode_active");
}


Expand All @@ -1059,7 +1090,7 @@ static boolean isNameNodeHAInitialActiveNodeSet(Map<String, Map<String, String>>
* false if the initial standby namenode property has not been configured
*/
static boolean isNameNodeHAInitialStandbyNodeSet(Map<String, Map<String, String>> configProperties) {
return configProperties.containsKey("hadoop-env") && configProperties.get("hadoop-env").containsKey("dfs_ha_initial_namenode_standby");
return configProperties.containsKey(HADOOP_ENV_CONFIG_TYPE_NAME) && configProperties.get(HADOOP_ENV_CONFIG_TYPE_NAME).containsKey("dfs_ha_initial_namenode_standby");
}

/**
Expand Down Expand Up @@ -2539,13 +2570,13 @@ public String updateForBlueprintExport(String propertyName,
singleHostTopologyUpdaters.put("ranger-kafka-audit", rangerKafkaAuditPropsMap);
singleHostTopologyUpdaters.put("ranger-storm-audit", rangerStormAuditPropsMap);
singleHostTopologyUpdaters.put("ranger-atlas-audit", rangerAtlasAuditPropsMap);
singleHostTopologyUpdaters.put("hadoop-env", shHadoopEnvMap);
singleHostTopologyUpdaters.put(HADOOP_ENV_CONFIG_TYPE_NAME, shHadoopEnvMap);

singleHostTopologyUpdaters.put("hawq-site", hawqSiteMap);
singleHostTopologyUpdaters.put("zookeeper-env", zookeeperEnvMap);


mPropertyUpdaters.put("hadoop-env", mHadoopEnvMap);
mPropertyUpdaters.put(HADOOP_ENV_CONFIG_TYPE_NAME, mHadoopEnvMap);
mPropertyUpdaters.put("hbase-env", hbaseEnvMap);
mPropertyUpdaters.put("mapred-env", mapredEnvMap);
mPropertyUpdaters.put("oozie-env", oozieEnvHeapSizeMap);
Expand Down
Loading