[WIP] AMBARI-25043. Sensitive Ambari configuration values are encrypted in Ambari Server DB

ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariServerConfiguration.java

@@ -17,6 +17,7 @@
  */
 package org.apache.ambari.server.configuration;
 
+import java.util.HashMap;
 import java.util.Map;
 
 import org.slf4j.Logger;
@@ -32,6 +33,14 @@
 public abstract class AmbariServerConfiguration {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(AmbariServerConfiguration.class);
+
+  protected final Map<String, String> configurationMap = new HashMap<>();
+
+  protected AmbariServerConfiguration(Map<String, String> configurationMap) {
+    if (configurationMap != null) {
+      this.configurationMap.putAll(configurationMap);
+    }
+  }
 
   /**
    * Gets the configuration value for given {@link AmbariServerConfigurationKey}.
@@ -64,5 +73,52 @@ protected String getValue(String propertyName, Map<String, String> configuration
       return defaultValue;
     }
   }
+
+  /**
+   * @return this configuration represented as a map
+   */
+  public Map<String, String> toMap() {
+    return new HashMap<>(configurationMap);
+  }
+
+  /**
+   * Sets the given value for the given configuration
+   * 
+   * @param configName
+   *          the name of the configuration to set the value for
+   * @param value
+   *          the new value
+   *
+   * @throws IllegalArgumentException
+   *           in case the supplied configuration does not belong to the
+   *           configuration category of this instance
+   */
+  public void setValueFor(String configName, String value) {
+    AmbariServerConfigurationKey ambariServerConfigurationKey = AmbariServerConfigurationKey.translate(getCategory(), configName);
+    if (ambariServerConfigurationKey != null) {
+      setValueFor(ambariServerConfigurationKey, value);
+    }
+  }
+
+  /**
+   * Sets the given value for the given configuration
+   * 
+   * @param ambariServerConfigurationKey
+   *          the configuration key to set the value for
+   * @param value
+   *          the new value
+   *
+   * @throws IllegalArgumentException
+   *           in case the supplied configuration does not belong to the
+   *           configuration category of this instance
+   */
+  public void setValueFor(AmbariServerConfigurationKey ambariServerConfigurationKey, String value) {
+    if (ambariServerConfigurationKey.getConfigurationCategory() != getCategory()) {
+      throw new IllegalArgumentException(ambariServerConfigurationKey.key() + " is not a valid " + getCategory().getCategoryName());
+    }
+    configurationMap.put(ambariServerConfigurationKey.key(), value);
+  }
+
+  protected abstract AmbariServerConfigurationCategory getCategory();
 
 }

ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariServerConfigurationKey.java

@@ -17,6 +17,10 @@
 import static org.apache.ambari.server.configuration.ConfigurationPropertyType.PASSWORD;
 import static org.apache.ambari.server.configuration.ConfigurationPropertyType.PLAINTEXT;
 
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -162,4 +166,8 @@ public static AmbariServerConfigurationKey translate(AmbariServerConfigurationCa
     LOG.warn("Invalid Ambari server configuration key: {}:{}", categoryName, keyName);
     return null;
   }
+
+  public static Set<String> findPasswordConfigurations() {
+      return Stream.of(AmbariServerConfigurationKey.values()).filter(k -> PASSWORD == k.getConfigurationPropertyType()).map(f -> f.propertyName).collect(Collectors.toSet());
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariServerConfigurationProvider.java

@@ -28,12 +28,14 @@
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
 import org.apache.ambari.server.orm.dao.AmbariConfigurationDAO;
 import org.apache.ambari.server.orm.entities.AmbariConfigurationEntity;
+import org.apache.ambari.server.security.encryption.Encryptor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.eventbus.Subscribe;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.google.inject.name.Named;
 import com.google.inject.persist.jpa.AmbariJpaPersistService;
 
 /**
@@ -54,6 +56,9 @@ public abstract class AmbariServerConfigurationProvider<T extends AmbariServerCo
   @Inject
   private Provider<AmbariConfigurationDAO> ambariConfigurationDAOProvider;
 
+  @Inject @Named("AmbariServerConfigurationEncryptor")
+  private Encryptor<AmbariServerConfiguration> encryptor;
+
   private final AtomicBoolean jpaStarted = new AtomicBoolean(false);
 
   private final AmbariServerConfigurationCategory configurationCategory;
@@ -84,6 +89,7 @@ public void ambariConfigurationChanged(AmbariConfigurationChangedEvent event) {
     if (configurationCategory.getCategoryName().equalsIgnoreCase(event.getCategoryName())) {
       LOGGER.info("Ambari configuration changed event received: {}", event);
       instance = loadInstance();
+
     }
   }
 
@@ -120,7 +126,9 @@ public T get() {
   private T loadInstance() {
     if (jpaStarted.get()) {
       LOGGER.info("Loading {} configuration data", configurationCategory.getCategoryName());
-      return loadInstance(ambariConfigurationDAOProvider.get().findByCategory(configurationCategory.getCategoryName()));
+      T instance = loadInstance(ambariConfigurationDAOProvider.get().findByCategory(configurationCategory.getCategoryName()));
+      encryptor.decryptSensitiveData(instance);
+      return instance;
     } else {
       LOGGER.info("Cannot load {} configuration data since JPA is not initialized", configurationCategory.getCategoryName());
       if (instance == null) {

ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java

@@ -59,6 +59,7 @@
 import org.apache.ambari.server.checks.UpgradeCheckRegistry;
 import org.apache.ambari.server.checks.UpgradeCheckRegistryProvider;
 import org.apache.ambari.server.cleanup.ClasspathScannerUtils;
+import org.apache.ambari.server.configuration.AmbariServerConfiguration;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.configuration.Configuration.ConnectionPoolType;
 import org.apache.ambari.server.configuration.Configuration.DatabaseType;
@@ -117,6 +118,7 @@
 import org.apache.ambari.server.security.authorization.internal.RunWithInternalSecurityContext;
 import org.apache.ambari.server.security.encryption.AESEncryptionService;
 import org.apache.ambari.server.security.encryption.AgentConfigUpdateEncryptor;
+import org.apache.ambari.server.security.encryption.AmbariServerConfigurationEncryptor;
 import org.apache.ambari.server.security.encryption.ConfigPropertiesEncryptor;
 import org.apache.ambari.server.security.encryption.CredentialStoreService;
 import org.apache.ambari.server.security.encryption.CredentialStoreServiceImpl;
@@ -341,9 +343,11 @@ protected void configure() {
     if (configuration.shouldEncryptSensitiveData()) {
       bind(new TypeLiteral<Encryptor<Config>>() {}).annotatedWith(Names.named("ConfigPropertiesEncryptor")).to(ConfigPropertiesEncryptor.class);
       bind(new TypeLiteral<Encryptor<AgentConfigsUpdateEvent>>() {}).annotatedWith(Names.named("AgentConfigEncryptor")).to(AgentConfigUpdateEncryptor.class);
+      bind(new TypeLiteral<Encryptor<AmbariServerConfiguration>>() {}).annotatedWith(Names.named("AmbariServerConfigurationEncryptor")).to(AmbariServerConfigurationEncryptor.class);
     } else {
       bind(new TypeLiteral<Encryptor<Config>>() {}).annotatedWith(Names.named("ConfigPropertiesEncryptor")).toInstance(Encryptor.NONE);
       bind(new TypeLiteral<Encryptor<AgentConfigsUpdateEvent>>() {}).annotatedWith(Names.named("AgentConfigEncryptor")).toInstance(Encryptor.NONE);
+      bind(new TypeLiteral<Encryptor<AmbariServerConfiguration>>() {}).annotatedWith(Names.named("AmbariServerConfigurationEncryptor")).toInstance(Encryptor.NONE);
     }
 
     bind(Configuration.class).toInstance(configuration);

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerConfigurationHandler.java

@@ -18,29 +18,21 @@
 
 package org.apache.ambari.server.controller.internal;
 
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.RootServiceComponentConfiguration;
-import org.apache.ambari.server.configuration.AmbariServerConfigurationKey;
-import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.events.AmbariConfigurationChangedEvent;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
 import org.apache.ambari.server.orm.dao.AmbariConfigurationDAO;
 import org.apache.ambari.server.orm.entities.AmbariConfigurationEntity;
-import org.apache.ambari.server.security.encryption.CredentialProvider;
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -58,15 +50,11 @@ public class AmbariServerConfigurationHandler extends RootServiceComponentConfig
 
   private final AmbariConfigurationDAO ambariConfigurationDAO;
   private final AmbariEventPublisher publisher;
-  private final Configuration ambariConfiguration;
-
-  private CredentialProvider credentialProvider;
 
   @Inject
-  AmbariServerConfigurationHandler(AmbariConfigurationDAO ambariConfigurationDAO, AmbariEventPublisher publisher, Configuration ambariConfiguration) {
+  AmbariServerConfigurationHandler(AmbariConfigurationDAO ambariConfigurationDAO, AmbariEventPublisher publisher) {
     this.ambariConfigurationDAO = ambariConfigurationDAO;
     this.publisher = publisher;
-    this.ambariConfiguration = ambariConfiguration;
   }
 
   @Override
@@ -111,38 +99,23 @@ public void removeComponentConfiguration(String categoryName) {
 
   @Override
   public void updateComponentCategory(String categoryName, Map<String, String> properties, boolean removePropertiesIfNotSpecified) throws AmbariException {
-    boolean toBePublished = false;
-    final Iterator<Map.Entry<String, String>> propertiesIterator = properties.entrySet().iterator();
-    while (propertiesIterator.hasNext()) {
-      Map.Entry<String, String> property = propertiesIterator.next();
-
-      // Ensure the incoming property is valid
-      AmbariServerConfigurationKey key = AmbariServerConfigurationUtils.getConfigurationKey(categoryName, property.getKey());
-      if(key == null) {
-        throw new IllegalArgumentException(String.format("Invalid Ambari server configuration key: %s:%s", categoryName, property.getKey()));
-      }
-
-      if (AmbariServerConfigurationUtils.isPassword(key)) {
-        final String passwordFileOrCredentialStoreAlias = fetchPasswordFileNameOrCredentialStoreAlias(categoryName, property.getKey());
-        if (StringUtils.isNotBlank(passwordFileOrCredentialStoreAlias)) { //if blank -> this is the first time setup; we simply need to store the alias/file name
-          if (updatePasswordIfNeeded(categoryName, property.getKey(), property.getValue())) {
-            toBePublished = true;
-          }
-          propertiesIterator.remove(); //we do not need to change the any PASSWORD type configuration going forward
-        }
-      }
-    }
-
-    if (!properties.isEmpty()) {
-      toBePublished = ambariConfigurationDAO.reconcileCategory(categoryName, properties, removePropertiesIfNotSpecified) || toBePublished;
-    }
+    validateProperties(categoryName, properties);
+    final boolean toBePublished = properties.isEmpty() ? false : ambariConfigurationDAO.reconcileCategory(categoryName, properties, removePropertiesIfNotSpecified);
 
     if (toBePublished) {
       // notify subscribers about the configuration changes
       publisher.publish(new AmbariConfigurationChangedEvent(categoryName));
     }
   }
 
+  private void validateProperties(String categoryName, Map<String, String> properties) {
+    for (String key : properties.keySet()) {
+      if (AmbariServerConfigurationUtils.getConfigurationKey(categoryName, key) == null) {
+        throw new IllegalArgumentException(String.format("Invalid Ambari server configuration key: %s:%s", categoryName, key));
+      }
+    }
+  }
+
   @Override
   public OperationResult performOperation(String categoryName, Map<String, String> properties,
                                           boolean mergeExistingProperties, String operation,
@@ -197,50 +170,4 @@ protected Set<String> getEnabledServices(String categoryName, String manageServi
       return Arrays.stream(enabledServices.split(",")).map(String::trim).map(String::toUpperCase).collect(Collectors.toSet());
     }
   }
-
-  private boolean updatePasswordIfNeeded(String categoryName, String propertyName, String newPassword) throws AmbariException {
-    if (newPassword != null) {
-      final String passwordFileOrCredentailStoreAlias = fetchPasswordFileNameOrCredentialStoreAlias(categoryName, propertyName);
-      if (!newPassword.equals(passwordFileOrCredentailStoreAlias)) { //we only need to do anything if the user-supplied password is a 'real' password
-        if (ambariConfiguration.isSecurityPasswordEncryptionEnabled()) {
-          getCredentialProvider().addAliasToCredentialStore(passwordFileOrCredentailStoreAlias, newPassword);
-        } else {
-          savePasswordInFile(passwordFileOrCredentailStoreAlias, newPassword);
-        }
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /*
-   * If the configuration element is actually a PASSWORD type element then we either have a password file name stored in the DB
-   * or - in case security password encryption is enabled - a Credential Store alias.
-   */
-  private String fetchPasswordFileNameOrCredentialStoreAlias(String categoryName, String propertyName) {
-    for (AmbariConfigurationEntity entity : ambariConfigurationDAO.findByCategory(categoryName)) {
-      if (entity.getPropertyName().equals(propertyName)) {
-        return entity.getPropertyValue();
-      }
-    }
-
-    return null;
-  }
-
-  private CredentialProvider getCredentialProvider() throws AmbariException {
-    if (credentialProvider == null) {
-      credentialProvider = new CredentialProvider(null, ambariConfiguration);
-    }
-    return credentialProvider;
-  }
-
-  private void savePasswordInFile(String passwordFileName, String newPassword) throws AmbariException {
-    try {
-      if (StringUtils.isNotBlank(passwordFileName)) {
-        FileUtils.writeStringToFile(new File(passwordFileName), newPassword, Charset.defaultCharset());
-      }
-    } catch (IOException e) {
-      throw new AmbariException("Error while updating password file [" + passwordFileName + "]", e);
-    }
-  }
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerLDAPConfigurationHandler.java

@@ -30,15 +30,16 @@
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
+import org.apache.ambari.server.configuration.AmbariServerConfiguration;
 import org.apache.ambari.server.configuration.AmbariServerConfigurationCategory;
-import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
 import org.apache.ambari.server.ldap.domain.AmbariLdapConfiguration;
 import org.apache.ambari.server.ldap.service.AmbariLdapException;
 import org.apache.ambari.server.ldap.service.LdapFacade;
 import org.apache.ambari.server.orm.dao.AmbariConfigurationDAO;
+import org.apache.ambari.server.security.encryption.Encryptor;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.ConfigHelper;
 import org.apache.commons.lang.StringUtils;
@@ -47,6 +48,7 @@
 
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+import com.google.inject.name.Named;
 
 /**
  * AmbariServerLDAPConfigurationHandler handles Ambari server LDAP-specific configuration properties.
@@ -56,19 +58,22 @@ public class AmbariServerLDAPConfigurationHandler extends AmbariServerStackAdvis
   private static final Logger LOGGER = LoggerFactory.getLogger(AmbariServerLDAPConfigurationHandler.class);
 
   private final LdapFacade ldapFacade;
+  private final Encryptor<AmbariServerConfiguration> encryptor;
 
   @Inject
   AmbariServerLDAPConfigurationHandler(Clusters clusters, ConfigHelper configHelper, AmbariManagementController managementController,
-      StackAdvisorHelper stackAdvisorHelper, AmbariConfigurationDAO ambariConfigurationDAO, AmbariEventPublisher publisher, Configuration ambariConfiguration,
-      LdapFacade ldapFacade) {
-    super(ambariConfigurationDAO, publisher, ambariConfiguration, clusters, configHelper, managementController, stackAdvisorHelper);
+      StackAdvisorHelper stackAdvisorHelper, AmbariConfigurationDAO ambariConfigurationDAO, AmbariEventPublisher publisher,
+      LdapFacade ldapFacade, @Named("AmbariServerConfigurationEncryptor") Encryptor<AmbariServerConfiguration> encryptor) {
+    super(ambariConfigurationDAO, publisher, clusters, configHelper, managementController, stackAdvisorHelper);
     this.ldapFacade = ldapFacade;
+    this.encryptor = encryptor;
   }
 
   @Override
   public void updateComponentCategory(String categoryName, Map<String, String> properties, boolean removePropertiesIfNotSpecified) throws AmbariException {
-    super.updateComponentCategory(categoryName, properties, removePropertiesIfNotSpecified);
-    final AmbariLdapConfiguration ldapConfiguration = new AmbariLdapConfiguration(getConfigurationProperties(AmbariServerConfigurationCategory.LDAP_CONFIGURATION.getCategoryName()));
+    final AmbariLdapConfiguration ldapConfiguration = new AmbariLdapConfiguration(properties);
+    encryptor.encryptSensitiveData(ldapConfiguration);
+    super.updateComponentCategory(categoryName, ldapConfiguration.toMap(), removePropertiesIfNotSpecified);
     if (ldapConfiguration.isAmbariManagesLdapConfiguration()) {
       processClusters(LDAP_CONFIGURATIONS);
     }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerSSOConfigurationHandler.java

@@ -24,7 +24,6 @@
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
-import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
 import org.apache.ambari.server.orm.dao.AmbariConfigurationDAO;
@@ -43,8 +42,8 @@ public class AmbariServerSSOConfigurationHandler extends AmbariServerStackAdviso
 
   @Inject
   public AmbariServerSSOConfigurationHandler(Clusters clusters, ConfigHelper configHelper, AmbariManagementController managementController,
-      StackAdvisorHelper stackAdvisorHelper, AmbariConfigurationDAO ambariConfigurationDAO, AmbariEventPublisher publisher, Configuration ambariConfiguration) {
-    super(ambariConfigurationDAO, publisher, ambariConfiguration, clusters, configHelper, managementController, stackAdvisorHelper);
+      StackAdvisorHelper stackAdvisorHelper, AmbariConfigurationDAO ambariConfigurationDAO, AmbariEventPublisher publisher) {
+    super(ambariConfigurationDAO, publisher, clusters, configHelper, managementController, stackAdvisorHelper);
   }
 
   @Override