AMBARI-17462. Install Log search client package with Ranger Service(M…

…ugdha Varadkar via gautam)
apache · Jun 28, 2016 · 82d5a0b · 82d5a0b
1 parent 614b4c0
commit 82d5a0b
Show file tree

Hide file tree

Showing 14 changed files with 457 additions and 40 deletions.
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -86,7 +86,7 @@ class StackFeature:
   RANGER_KERBEROS_SUPPORT = "ranger_kerberos_support"
   HIVE_METASTORE_SITE_SUPPORT = "hive_metastore_site_support"
   RANGER_USERSYNC_PASSWORD_JCEKS = "ranger_usersync_password_jceks"
-  LOGSEARCH_SUPPORT = "logsearch_support"
+  RANGER_INSTALL_LOGSEARCH_CLIENT = "ranger_install_logsearch_client"
   HBASE_HOME_DIRECTORY = "hbase_home_directory"
   ATLAS_RANGER_PLUGIN_SUPPORT = "atlas_ranger_plugin_support"
   ATLAS_UPGRADE_SUPPORT = "atlas_upgrade_support"

diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
@@ -52,7 +52,7 @@ def should_install_logsearch_solr():
 
 def should_install_logsearch_solr_client():
   config = Script.get_config()
-  return 'role' in config and (config['role'] == "LOGSEARCH_SOLR_CLIENT" or config['role'] == 'ATLAS_SERVER')
+  return 'role' in config and (config['role'] == "LOGSEARCH_SOLR_CLIENT" or config['role'] == 'ATLAS_SERVER' or config['role'] == 'RANGER_ADMIN')
 
 def should_install_logsearch_portal():
   config = Script.get_config()

diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
@@ -46,7 +46,7 @@ def upload_configuration_to_zk(zookeeper_quorum, solr_znode, config_set, config_
   """
   solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home)
   Execute(format('{solr_cli_prefix} --download-config --config-dir {tmp_config_set_dir} --config-set {config_set} --retry {retry} --interval {interval}'),
-          only_if=as_user(format("{solr_cli_prefix} --check-config --config-set{config_set} --retry {retry} --interval {interval}"), user),
+          only_if=as_user(format("{solr_cli_prefix} --check-config --config-set {config_set} --retry {retry} --interval {interval}"), user),
           user=user
           )
 

diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/metainfo.xml b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/metainfo.xml
@@ -96,10 +96,6 @@
             <package>
               <name>ranger_${stack_version}-usersync</name>
             </package>
-            <package>
-              <name>ranger_${stack_version}-tagsync</name>
-              <condition>should_install_ranger_tagsync</condition>
-            </package>
           </packages>
         </osSpecific>
         <osSpecific>
@@ -111,10 +107,6 @@
             <package>
               <name>ranger-${stack_version}-usersync</name>
             </package>
-            <package>
-              <name>ranger-${stack_version}-tagsync</name>
-              <condition>should_install_ranger_tagsync</condition>
-            </package>
           </packages>
         </osSpecific>
       </osSpecifics>

diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
@@ -64,7 +64,7 @@
 stack_supports_ranger_log4j =  stack_version_formatted and check_stack_feature(StackFeature.RANGER_LOG4J_SUPPORT, stack_version_formatted)
 stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted)
 stack_supports_usersync_passwd = stack_version_formatted and check_stack_feature(StackFeature.RANGER_USERSYNC_PASSWORD_JCEKS, stack_version_formatted)
-stack_supports_logsearch = stack_version_formatted and check_stack_feature(StackFeature.LOGSEARCH_SUPPORT, stack_version_formatted)
+stack_supports_logsearch_client = stack_version_formatted and check_stack_feature(StackFeature.RANGER_INSTALL_LOGSEARCH_CLIENT, stack_version_formatted)
 stack_supports_pid = stack_version_formatted and check_stack_feature(StackFeature.RANGER_PID_SUPPORT, stack_version_formatted)
 
 downgrade_from_version = default("/commandParams/downgrade_from_version", None)
@@ -263,31 +263,53 @@
 ranger_solr_config_set = config['configurations']['ranger-env']['ranger_solr_config_set']
 ranger_solr_collection_name = config['configurations']['ranger-env']['ranger_solr_collection_name']
 ranger_solr_shards = config['configurations']['ranger-env']['ranger_solr_shards']
-zookeeper_hosts_list = config['clusterHostInfo']['zookeeper_hosts']
-zookeeper_hosts_list.sort()
-zookeeper_hosts = ",".join(zookeeper_hosts_list)
-logsearch_solr_znode = config['configurations']['logsearch-solr-env']['logsearch_solr_znode']
+replication_factor = config['configurations']['ranger-env']['ranger_solr_replication_factor']
 ranger_solr_conf = format('{ranger_home}/contrib/solr_for_audit_setup/conf')
 logsearch_solr_hosts = default("/clusterHostInfo/logsearch_solr_hosts", [])
-replication_factor = 2 if len(logsearch_solr_hosts) > 1 else 1
 has_logsearch = len(logsearch_solr_hosts) > 0
 is_solrCloud_enabled = default('/configurations/ranger-env/is_solrCloud_enabled', False)
+solr_znode = '/ranger_audits'
+if is_solrCloud_enabled:
+  solr_znode = config['configurations']['ranger-admin-site']['ranger.audit.solr.zookeepers']
+  if solr_znode != '' and solr_znode.upper() != 'NONE':
+    solr_znode = solr_znode.split('/')[1]
+    solr_znode = format('/{solr_znode}')
+  if has_logsearch:
+    solr_znode = config['configurations']['logsearch-solr-env']['logsearch_solr_znode']
+solr_user = default('/configurations/logsearch-solr-env/logsearch_solr_user', unix_user)
+custom_log4j = True if has_logsearch else False
+
+# get comma separated list of zookeeper hosts
 zookeeper_port = default('/configurations/zoo.cfg/clientPort', None)
-# get comma separated list of zookeeper hosts from clusterHostInfo
+zookeeper_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
 index = 0
 zookeeper_quorum = ""
-for host in config['clusterHostInfo']['zookeeper_hosts']:
+for host in zookeeper_hosts:
   zookeeper_quorum += host + ":" + str(zookeeper_port)
   index += 1
-  if index < len(config['clusterHostInfo']['zookeeper_hosts']):
+  if index < len(zookeeper_hosts):
     zookeeper_quorum += ","
+solr_jaas_file = None
 
 if security_enabled:
   if has_ranger_tagsync:
     ranger_tagsync_principal = config['configurations']['ranger-tagsync-site']['ranger.tagsync.kerberos.principal']
     tagsync_jaas_principal = ranger_tagsync_principal.replace('_HOST', current_host.lower())
     tagsync_keytab_path = config['configurations']['ranger-tagsync-site']['ranger.tagsync.kerberos.keytab']
 
+  if stack_supports_ranger_kerberos:
+    ranger_admin_principal = config['configurations']['ranger-admin-site']['ranger.admin.kerberos.principal']
+    ranger_admin_jaas_principal = ranger_admin_principal.replace('_HOST', ranger_host.lower())
+    ranger_admin_keytab = config['configurations']['ranger-admin-site']['ranger.admin.kerberos.keytab']
+
+    if not is_empty(ranger_admin_principal) and ranger_admin_principal != '':
+      if stack_supports_logsearch_client and is_solrCloud_enabled:
+        solr_jaas_file = format('{ranger_home}/conf/ranger_solr_jass.conf')
+        solr_kerberos_principal = ranger_admin_jaas_principal
+        solr_kerberos_keytab = ranger_admin_keytab
+    else:
+      solr_jaas_file = None
+
 # logic to create core-site.xml if hdfs not installed
 if stack_supports_ranger_kerberos and not has_namenode:
   core_site_property = {

diff --git a/...ri-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py b/...ri-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py
@@ -82,8 +82,6 @@ def start(self, env, upgrade_type=None):
     env.set_params(params)
     self.configure(env, upgrade_type=upgrade_type)
 
-    if params.stack_supports_logsearch and params.has_logsearch and params.is_solrCloud_enabled:
-      setup_ranger_audit_solr()
     ranger_service('ranger_admin')
 
 

diff --git a/...erver/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/...erver/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py
@@ -22,7 +22,7 @@
 import random
 from resource_management.core.logger import Logger
 from resource_management.core.resources.system import File, Directory, Execute, Link
-from resource_management.core.source import DownloadSource, InlineTemplate
+from resource_management.core.source import DownloadSource, InlineTemplate, Template
 from resource_management.libraries.resources.xml_config import XmlConfig
 from resource_management.libraries.resources.modify_properties_file import ModifyPropertiesFile
 from resource_management.libraries.resources.properties_file import PropertiesFile
@@ -191,6 +191,10 @@ def setup_ranger_admin(upgrade_type=None):
 
   create_core_site_xml(ranger_conf)
 
+  if params.stack_supports_logsearch_client and params.is_solrCloud_enabled:
+    solr_cloud_util.setup_solr_client(params.config, user = params.solr_user, custom_log4j = params.custom_log4j)
+    setup_ranger_audit_solr()
+
 def setup_ranger_db(stack_version=None):
   import params
 
@@ -557,9 +561,15 @@ def setup_ranger_audit_solr():
   random_num = random.random()
   tmp_config_set_folder = format('{tmp_dir}/ranger_config_{ranger_solr_config_set}_{random_num}')
 
+  if params.security_enabled and params.stack_supports_ranger_kerberos:
+    File(format("{solr_jaas_file}"),
+      content=Template("ranger_solr_jass_conf.j2"),
+      owner=params.unix_user
+    )
+
   solr_cloud_util.upload_configuration_to_zk(
     zookeeper_quorum = params.zookeeper_quorum,
-    solr_znode = params.logsearch_solr_znode,
+    solr_znode = params.solr_znode,
     config_set = params.ranger_solr_config_set,
     config_set_dir = params.ranger_solr_conf,
     tmp_config_set_dir = tmp_config_set_folder,
@@ -569,10 +579,11 @@ def setup_ranger_audit_solr():
 
   solr_cloud_util.create_collection(
     zookeeper_quorum = params.zookeeper_quorum,
-    solr_znode = params.logsearch_solr_znode,
+    solr_znode = params.solr_znode,
     collection = params.ranger_solr_collection_name,
     config_set = params.ranger_solr_config_set,
     java64_home = params.java_home,
     user = params.unix_user,
     shards = params.ranger_solr_shards,
-    replication_factor = params.replication_factor)
+    replication_factor = int(params.replication_factor),
+    jaas_file = params.solr_jaas_file)
diff --git a/...rc/main/resources/common-services/RANGER/0.4.0/package/templates/ranger_solr_jass_conf.j2 b/...rc/main/resources/common-services/RANGER/0.4.0/package/templates/ranger_solr_jass_conf.j2
@@ -0,0 +1,26 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+Client {
+  com.sun.security.auth.module.Krb5LoginModule required
+  useKeyTab=true
+  storeKey=true
+  useTicketCache=false
+  keyTab="{{solr_kerberos_keytab}}"
+  principal="{{solr_kerberos_principal}}";
+};
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-env.xml b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-env.xml
@@ -39,6 +39,11 @@
     <value>1</value>
     <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>ranger_solr_replication_factor</name>
+    <value>1</value>
+    <on-ambari-upgrade add="true"/>
+  </property>
   <property>
     <name>ranger-atlas-plugin-enabled</name>
     <value>No</value>

diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.6.0/metainfo.xml b/ambari-server/src/main/resources/common-services/RANGER/0.6.0/metainfo.xml
@@ -53,6 +53,47 @@
         </theme>
       </themes>
 
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily>
+          <packages>
+            <package>
+              <name>ranger_${stack_version}-admin</name>
+            </package>
+            <package>
+              <name>ranger_${stack_version}-usersync</name>
+            </package>
+            <package>
+              <name>ranger_${stack_version}-tagsync</name>
+              <condition>should_install_ranger_tagsync</condition>
+            </package>
+            <package>
+              <name>ambari-logsearch-solr-client</name>
+              <condition>should_install_logsearch_solr_client</condition>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily>
+          <packages>
+            <package>
+              <name>ranger-${stack_version}-admin</name>
+            </package>
+            <package>
+              <name>ranger-${stack_version}-usersync</name>
+            </package>
+            <package>
+              <name>ranger-${stack_version}-tagsync</name>
+              <condition>should_install_ranger_tagsync</condition>
+            </package>
+            <package>
+              <name>ambari-logsearch-solr-client</name>
+              <condition>should_install_logsearch_solr_client</condition>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+
       <configuration-dependencies>
         <config-type>admin-log4j</config-type>
         <config-type>usersync-log4j</config-type>

diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -227,7 +227,7 @@
       "min_version": "2.5.0.0"
     },
     {
-      "name": "logsearch_support",
+      "name": "ranger_install_logsearch_client",
       "description": "LogSearch Service support",
       "min_version": "2.5.0.0"
     },