From 13e851b31cc643e7582abfc0e5e3f0c394e5cbca Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 15 Aug 2018 11:18:01 -0500 Subject: [PATCH 1/2] initial commit --- .../CURRENT/package/scripts/rest_commands.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index 463dca1543..30ad13d1fa 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -118,26 +118,29 @@ def init_kafka_acls(self): def init_pcap(self): Logger.info("Creating HDFS locations for Pcap") + # Non Kerberized Metron runs under 'storm', requiring write under the 'hadoop' group. + # Kerberized Metron runs under it's own user. + ownership = 0755 if self.__params.security_enabled else 0775 self.__params.HdfsResource(self.__params.pcap_base_path, type="directory", action="create_on_execute", owner=self.__params.metron_user, - group=self.__params.metron_group, - mode=0755, + group=self.__params.hadoop_group, + mode=ownership, ) self.__params.HdfsResource(self.__params.pcap_base_interim_result_path, type="directory", action="create_on_execute", owner=self.__params.metron_user, - group=self.__params.metron_group, - mode=0755, + group=self.__params.hadoop_group, + mode=ownership, ) self.__params.HdfsResource(self.__params.pcap_final_output_path, type="directory", action="create_on_execute", owner=self.__params.metron_user, - group=self.__params.metron_group, - mode=0755, + group=self.__params.hadoop_group, + mode=ownership, ) def create_metron_user_hdfs_dir(self): From cff9916697207348af22dfa9f938e1674cb24a29 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 15 Aug 2018 12:45:24 -0500 Subject: [PATCH 2/2] added check to update permissions when a cluster is Kerberized --- .../METRON/CURRENT/package/scripts/params/params_linux.py | 1 + .../CURRENT/package/scripts/params/status_params.py | 1 + .../METRON/CURRENT/package/scripts/rest_commands.py | 8 ++++++++ .../METRON/CURRENT/package/scripts/rest_master.py | 5 +++++ 4 files changed, 15 insertions(+) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 4f8a9a7535..362f8c4368 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -392,6 +392,7 @@ pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue'] pcap_configured_flag_file = status_params.pcap_configured_flag_file +pcap_perm_configured_flag_file = status_params.pcap_perm_configured_flag_file # MapReduce metron_user_hdfs_dir = '/user/' + metron_user diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 0a9fdd08ef..99f5ec037a 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -120,6 +120,7 @@ # Pcap pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured' +pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured' # MapReduce metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + '/../metron_user_hdfs_dir_configured' \ No newline at end of file diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index 30ad13d1fa..d44f478e95 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -38,6 +38,7 @@ class RestCommands: __hbase_configured = False __hbase_acl_configured = False __pcap_configured = False + __pcap_perm_configured = False __metron_user_hdfs_dir_configured = False def __init__(self, params): @@ -49,6 +50,7 @@ def __init__(self, params): self.__hbase_configured = os.path.isfile(self.__params.rest_hbase_configured_flag_file) self.__hbase_acl_configured = os.path.isfile(self.__params.rest_hbase_acl_configured_flag_file) self.__pcap_configured = os.path.isfile(self.__params.pcap_configured_flag_file) + self.__pcap_perm_configured = os.path.isfile(self.__params.pcap_perm_configured_flag_file) self.__metron_user_hdfs_dir_configured = os.path.isfile(self.__params.metron_user_hdfs_dir_configured_flag_file) Directory(params.metron_rest_pid_dir, mode=0755, @@ -81,6 +83,9 @@ def is_hbase_acl_configured(self): def is_pcap_configured(self): return self.__pcap_configured + def is_pcap_perm_configured(self): + return self.__pcap_perm_configured + def is_metron_user_hdfs_dir_configured(self): return self.__metron_user_hdfs_dir_configured @@ -99,6 +104,9 @@ def set_hbase_acl_configured(self): def set_pcap_configured(self): metron_service.set_configured(self.__params.metron_user, self.__params.pcap_configured_flag_file, "Setting Pcap configured to True") + def set_pcap_perm_configured(self): + metron_service.set_configured(self.__params.metron_user, self.__params.pcap_perm_configured_flag_file, "Setting Pcap perm configured to True") + def set_metron_user_hdfs_dir_configured(self): metron_service.set_configured(self.__params.metron_user, self.__params.metron_user_hdfs_dir_configured_flag_file, "Setting Metron user HDFS directory configured to True") diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py index c842214b16..791ca77c5c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py @@ -60,6 +60,11 @@ def configure(self, env, upgrade_type=None, config_dir=None): if params.security_enabled and not commands.is_kafka_acl_configured(): commands.init_kafka_acls() commands.set_kafka_acl_configured() + if params.security_enabled and not commands.is_pcap_perm_configured(): + # If we Kerberize the cluster, we need to call this again, to remove write perms from hadoop group + # If we start off Kerberized, it just does the same thing twice. + commands.init_pcap() + commands.set_pcap_perm_configured() def start(self, env, upgrade_type=None): from params import params