Skip to content
Permalink
Browse files
AMBARI-24745. Enable encryption of sensitive data in Ambari DB using … (
#2755)

AMBARI-24745. Enable encryption of sensitive data in Ambari DB using Ambari CLI (dlysnichenko)
  • Loading branch information
dlysnichenko committed Jan 18, 2019
1 parent c50dc8d commit 97a6bc8b389fa5cfb73c87392af91e2d4d405ae3
Showing 5 changed files with 802 additions and 339 deletions.
@@ -54,7 +54,7 @@
from ambari_server.setupHttps import setup_https, setup_truststore
from ambari_server.setupMpacks import install_mpack, uninstall_mpack, upgrade_mpack, STACK_DEFINITIONS_RESOURCE_NAME, \
SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME
from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_master_key, setup_ambari_krb5_jaas, setup_pam, \
from ambari_server.setupSecurity import setup_ldap, sync_ldap, setup_sensitive_data_encryption, setup_ambari_krb5_jaas, setup_pam, \
migrate_ldap_pam, LDAP_TYPES
from ambari_server.setupSso import setup_sso
from ambari_server.setupTrustedProxy import setup_trusted_proxy
@@ -253,7 +253,7 @@ def refresh_stack_hash_action():
def create_setup_security_actions(args):
action_list = [
['setup-https', 'Enable HTTPS for Ambari server.', UserActionRestart(setup_https, args)],
['encrypt-passwords', 'Encrypt passwords stored in ambari.properties file.', UserAction(setup_master_key, args)],
['encrypt-passwords', 'Encrypt passwords managed by Ambari.', UserAction(setup_sensitive_data_encryption, args)],
['setup-kerberos-jaas', 'Setup Ambari kerberos JAAS configuration.', UserAction(setup_ambari_krb5_jaas, args)],
['setup-truststore', 'Setup truststore.', UserActionRestart(setup_truststore, args)],
['import-certificate', 'Import certificate to truststore.', UserActionRestart(setup_truststore, True, args)],
@@ -264,7 +264,7 @@ def create_setup_security_actions(args):
def create_setup_security_actions(args):
action_list = [
['setup-https', 'Enable HTTPS for Ambari server.', UserActionRestart(setup_https, args)],
['encrypt-passwords', 'Encrypt passwords stored in ambari.properties file.', UserAction(setup_master_key, args)],
['encrypt-passwords', 'Encrypt passwords managed by Ambari.', UserAction(setup_sensitive_data_encryption, args)],
['setup-kerberos-jaas', 'Setup Ambari kerberos JAAS configuration.', UserAction(setup_ambari_krb5_jaas, args)],
['setup-truststore', 'Setup truststore.', UserActionRestart(setup_truststore, args)],
['import-certificate', 'Import certificate to truststore.', UserActionRestart(setup_truststore, args, True)],
@@ -608,6 +608,7 @@ def __init__(self):
SECURITY_KEY_ENV_VAR_NAME = "AMBARI_SECURITY_MASTER_KEY"
SECURITY_MASTER_KEY_FILENAME = "master"
SECURITY_IS_ENCRYPTION_ENABLED = "security.passwords.encryption.enabled"
SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED = "security.server.encrypt_sensitive_data"
SECURITY_KERBEROS_JASS_FILENAME = "krb5JAASLogin.conf"

SECURITY_PROVIDER_GET_CMD = "{0} -cp {1} " + \
@@ -625,6 +626,11 @@ def __init__(self):
".MasterKeyServiceImpl {2} {3} {4} " + \
"> " + configDefaults.SERVER_OUT_FILE + " 2>&1"

SECURITY_SENSITIVE_DATA_ENCRYPTON_CMD = "{0} -cp {1} " + \
"org.apache.ambari.server.security.encryption.SensitiveDataEncryption" + \
" {2} " + \
"> " + configDefaults.SERVER_OUT_FILE + " 2>&1"



def read_ambari_user():
@@ -1045,7 +1051,7 @@ def get_original_master_key(properties, options = None):
masterKey = options.master_key
if not masterKey:
masterKey = get_validated_string_input('Enter current Master Key: ',
"", ".*", "", True, False)
"", ".*", "", True, True)
if options is not None:
options.master_key = masterKey
except KeyboardInterrupt:
@@ -47,8 +47,8 @@
BLIND_PASSWORD, BOOTSTRAP_DIR_PROPERTY, JDBC_PASSWORD_FILENAME, JDBC_PASSWORD_PROPERTY, \
JDBC_RCA_PASSWORD_ALIAS, JDBC_RCA_PASSWORD_FILE_PROPERTY, JDBC_USE_INTEGRATED_AUTH_PROPERTY, \
LDAP_MGR_PASSWORD_ALIAS, LDAP_MGR_PASSWORD_PROPERTY, CLIENT_SECURITY, \
SECURITY_IS_ENCRYPTION_ENABLED, SECURITY_KEY_ENV_VAR_NAME, SECURITY_KERBEROS_JASS_FILENAME, \
SECURITY_PROVIDER_KEY_CMD, SECURITY_MASTER_KEY_FILENAME, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
SECURITY_IS_ENCRYPTION_ENABLED, SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED, SECURITY_KEY_ENV_VAR_NAME, SECURITY_KERBEROS_JASS_FILENAME, \
SECURITY_PROVIDER_KEY_CMD, SECURITY_SENSITIVE_DATA_ENCRYPTON_CMD, SECURITY_MASTER_KEY_FILENAME, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
SSL_TRUSTSTORE_PASSWORD_PROPERTY, SSL_TRUSTSTORE_PATH_PROPERTY, SSL_TRUSTSTORE_TYPE_PROPERTY, \
JDK_NAME_PROPERTY, JCE_NAME_PROPERTY, JAVA_HOME_PROPERTY, \
get_resources_location, SECURITY_MASTER_KEY_LOCATION, SETUP_OR_UPGRADE_MSG, \
@@ -487,9 +487,24 @@ def sync_ldap(options):
sys.stdout.write('\n')
sys.stdout.flush()

def setup_master_key(options):
def sensitive_data_encryption(options, direction, masterKey=None):
environ = os.environ.copy()
if masterKey:
environ[SECURITY_KEY_ENV_VAR_NAME] = masterKey
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
"command to install a JDK automatically or install any "
"JDK manually to " + configDefaults.JDK_INSTALL_DIR)
return 1
serverClassPath = ServerClassPath(get_ambari_properties(), options)
command = SECURITY_SENSITIVE_DATA_ENCRYPTON_CMD.format(get_java_exe_path(), serverClassPath.get_full_ambari_classpath_escaped_for_shell(), direction)
(retcode, stdout, stderr) = run_os_command(command, environ)
pass

def setup_sensitive_data_encryption(options):
if not is_root():
warn = 'ambari-server setup-https is run as ' \
warn = 'ambari-server encrypt-passwords is run as ' \
'non-root user, some sudo privileges might be required'
print warn

@@ -516,76 +531,67 @@ def setup_master_key(options):

ts_password = properties.get_property(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
resetKey = False
decrypt = False
masterKey = None

if isSecure:
print "Password encryption is enabled."
resetKey = True if options.security_option is not None else get_YN_input("Do you want to reset Master Key? [y/n] (n): ", False)

# For encrypting of only unencrypted passwords without resetting the key ask
# for master key if not persisted.
if isSecure and not isPersisted and not resetKey:
print "Master Key not persisted."
masterKey = get_original_master_key(properties, options)
pass
decrypt = get_YN_input("Do you want to decrypt passwords managed by Ambari? [y/n] (n): ", False)
if not decrypt:
resetKey = get_YN_input("Do you want to reset Master Key? [y/n] (n): ", False)

# Make sure both passwords are clear-text if master key is lost
if resetKey:
if not isPersisted:
if resetKey or decrypt:
if isPersisted:
sensitive_data_encryption(options, "decryption")
else:
print "Master Key not persisted."
masterKey = get_original_master_key(properties, options)
# Unable get the right master key or skipped question <enter>
if not masterKey:
print "To disable encryption, do the following:"
print "- Edit " + find_properties_file() + \
" and set " + SECURITY_IS_ENCRYPTION_ENABLED + " = " + "false."
err = "{0} is already encrypted. Please call {1} to store unencrypted" \
" password and call 'encrypt-passwords' again."
if db_sql_auth and db_password and is_alias_string(db_password):
print err.format('- Database password', "'" + SETUP_ACTION + "'")
if ts_password and is_alias_string(ts_password):
print err.format('TrustStore password', "'" + LDAP_SETUP_ACTION + "'")

# todo fix unreachable code
printManualDecryptionWarning(db_password, db_sql_auth, ts_password)
return 1
pass
pass
pass
sensitive_data_encryption(options, "decryption", masterKey)

# Read back any encrypted passwords
if db_sql_auth and db_password and is_alias_string(db_password):
db_password = read_passwd_for_alias(JDBC_RCA_PASSWORD_ALIAS, masterKey)
if ts_password and is_alias_string(ts_password):
ts_password = read_passwd_for_alias(SSL_TRUSTSTORE_PASSWORD_ALIAS, masterKey)
# Read master key, if non-secure or reset is true
if resetKey or not isSecure:
masterKey = read_master_key(resetKey, options)
persist = get_YN_input("Do you want to persist master key. If you choose " \
"not to persist, you need to provide the Master " \
"Key while starting the ambari server as an env " \
"variable named " + SECURITY_KEY_ENV_VAR_NAME + \
" or the start will prompt for the master key."
" Persist [y/n] (y)? ", True, options.master_key_persist)
if persist:
save_master_key(options, masterKey, get_master_key_location(properties) + os.sep +
SECURITY_MASTER_KEY_FILENAME, persist)
elif not persist and masterKeyFile:
try:
os.remove(masterKeyFile)
print_info_msg("Deleting master key file at location: " + str(
masterKeyFile))
except Exception, e:
print 'ERROR: Could not remove master key file. %s' % e
# Blow up the credential store made with previous key, if any
store_file = get_credential_store_location(properties)
if os.path.exists(store_file):
try:
os.remove(store_file)
except:
print_warning_msg("Failed to remove credential store file.")
pass
pass
pass
db_password, ts_password = deryptPasswordsConfigs(db_password, db_sql_auth, masterKey, ts_password)
save_decrypted_ambari_properties(db_password, properties, ts_password)


if not decrypt:
if resetKey or not isSecure:
# Read master key and encrypt sensitive data, if non-secure or reset is true
masterKey, isPersisted = setup_master_key(masterKeyFile, options, properties, resetKey)
else:
if not isPersisted:
# For encrypting of only unencrypted passwords without resetting the key ask
# for master key if not persisted.
print "Master Key not persisted."
masterKey = get_original_master_key(properties, options)
encrypt_sensitive_data(db_password, masterKey, options, isPersisted, properties, ts_password)

# Since files for store and master are created we need to ensure correct
# permissions
ambari_user = read_ambari_user()
if ambari_user:
adjust_directory_permissions(ambari_user)
return 0


def save_decrypted_ambari_properties(db_password, properties, ts_password):
propertyMap = {SECURITY_IS_ENCRYPTION_ENABLED: 'false'}
propertyMap[SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED] = 'false'
if db_password:
propertyMap[JDBC_PASSWORD_PROPERTY] = db_password
if properties.get_property(JDBC_RCA_PASSWORD_FILE_PROPERTY):
propertyMap[JDBC_RCA_PASSWORD_FILE_PROPERTY] = db_password
if ts_password:
propertyMap[SSL_TRUSTSTORE_PASSWORD_PROPERTY] = ts_password
update_properties_2(properties, propertyMap)


def encrypt_sensitive_data(db_password, masterKey, options, persist, properties, ts_password):
propertyMap = {SECURITY_IS_ENCRYPTION_ENABLED: 'true'}
# Encrypt only un-encrypted passwords
if db_password and not is_alias_string(db_password):
@@ -597,25 +603,71 @@ def setup_master_key(options):
remove_password_file(JDBC_PASSWORD_FILENAME)
if properties.get_property(JDBC_RCA_PASSWORD_FILE_PROPERTY):
propertyMap[JDBC_RCA_PASSWORD_FILE_PROPERTY] = get_alias_string(JDBC_RCA_PASSWORD_ALIAS)
pass

if ts_password and not is_alias_string(ts_password):
retCode = save_passwd_for_alias(SSL_TRUSTSTORE_PASSWORD_ALIAS, ts_password, masterKey)
if retCode != 0:
print 'Failed to save secure TrustStore password.'
else:
propertyMap[SSL_TRUSTSTORE_PASSWORD_PROPERTY] = get_alias_string(SSL_TRUSTSTORE_PASSWORD_ALIAS)
pass

propertyMap[SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED] = 'true'
if persist:
sensitive_data_encryption(options, "encryption")
else:
sensitive_data_encryption(options, "encryption", masterKey)
update_properties_2(properties, propertyMap)

# Since files for store and master are created we need to ensure correct
# permissions
ambari_user = read_ambari_user()
if ambari_user:
adjust_directory_permissions(ambari_user)

return 0
def setup_master_key(masterKeyFile, options, properties, resetKey):
masterKey = read_master_key(resetKey, options)
persist = get_YN_input("Do you want to persist master key. If you choose " \
"not to persist, you need to provide the Master " \
"Key while starting the ambari server as an env " \
"variable named " + SECURITY_KEY_ENV_VAR_NAME + \
" or the start will prompt for the master key."
" Persist [y/n] (y)? ", True, options.master_key_persist)
if persist:
save_master_key(options, masterKey, get_master_key_location(properties) + os.sep +
SECURITY_MASTER_KEY_FILENAME, persist)
elif not persist and masterKeyFile:
try:
os.remove(masterKeyFile)
print_info_msg("Deleting master key file at location: " + str(
masterKeyFile))
except Exception, e:
print 'ERROR: Could not remove master key file. %s' % e
# Blow up the credential store made with previous key, if any
store_file = get_credential_store_location(properties)
if os.path.exists(store_file):
try:
os.remove(store_file)
except:
print_warning_msg("Failed to remove credential store file.")
return masterKey, persist


def deryptPasswordsConfigs(db_password, db_sql_auth, masterKey, ts_password):
if db_sql_auth and db_password and is_alias_string(db_password):
db_password = read_passwd_for_alias(JDBC_RCA_PASSWORD_ALIAS, masterKey)
if ts_password and is_alias_string(ts_password):
ts_password = read_passwd_for_alias(SSL_TRUSTSTORE_PASSWORD_ALIAS, masterKey)
return db_password, ts_password


def printManualDecryptionWarning(db_password, db_sql_auth, ts_password):
print "To disable encryption, do the following:"
print "- Edit " + find_properties_file() + \
" and set " + SECURITY_IS_ENCRYPTION_ENABLED + " = " + "false." + \
" and set " + SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED + " = " + "false." + \
" and set all passwords and sensitive data in service configs to right value."
err = "{0} is already encrypted. Please call {1} to store unencrypted" \
" password and call 'encrypt-passwords' again."
if db_sql_auth and db_password and is_alias_string(db_password):
print err.format('- Database password', "'" + SETUP_ACTION + "'")
if ts_password and is_alias_string(ts_password):
print err.format('TrustStore password', "'" + LDAP_SETUP_ACTION + "'")


def setup_ambari_krb5_jaas(options):
jaas_conf_file = search_file(SECURITY_KERBEROS_JASS_FILENAME, get_conf_dir())

0 comments on commit 97a6bc8

Please sign in to comment.