From e767022e6c87922f7a0c3ca143b7482548a0908c Mon Sep 17 00:00:00 2001 From: Anand Subramanian Date: Fri, 5 Jan 2018 11:40:04 +0530 Subject: [PATCH 1/3] Add Shiro authentication support --- .../package/scripts/indexing_master.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index c79dfd5efd..2232881dc6 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -15,6 +15,7 @@ """ import os +import requests from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute @@ -150,12 +151,36 @@ def zeppelin_notebook_import(self, env): env.set_params(params) Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) + + # With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password + # for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details). + ses = requests.session() + + # Check if authentication is enabled on the Zeppelin server + try: + conn = ses.get(ambari_format('http://{zeppelin_server_url}/api/login')) + + # Establish connection if authentication is enabled + try: + # The following credentials are created at install time by Ambari at /etc/zeppelin/conf/shiro.ini + # when Shiro auth is enabled on the Zeppelin server + zeppelin_payload = {'userName': 'admin', 'password' : 'admin'} + conn = ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload) + except: + pass + + # If authentication is not enabled, fall back to default method of imporing notebooks + except requests.exceptions.RequestException: + conn = ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook')) + for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"): - zeppelin_cmd = ambari_format( - 'curl -s -XPOST http://{zeppelin_server_url}/api/notebook/import -d "@' + os.path.join(dirName, fileName) + '"') - Execute(zeppelin_cmd, logoutput=True) + Logger.info("Importing notebook: " + fileName) + zeppelin_import_url = ambari_format('http://{zeppelin_server_url}/api/notebook/import') + zeppelin_notebook = {'file' : open(os.path.join(dirName, fileName), 'rb')} + res = ses.post(zeppelin_import_url, files=zeppelin_notebook) + Logger.info("Result: " + res.text) if __name__ == "__main__": Indexing().execute() From b8780df0a0f1231a18597302302a3bd247b7dfd6 Mon Sep 17 00:00:00 2001 From: Anand Subramanian Date: Tue, 16 Jan 2018 17:22:02 +0530 Subject: [PATCH 2/3] Get admin user creds from Zeppelin shiro config in Ambari --- .../package/scripts/indexing_master.py | 19 ++++++++++++++++--- .../package/scripts/params/params_linux.py | 1 + .../package/scripts/params/status_params.py | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index 2232881dc6..d66e0dbe26 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -15,6 +15,7 @@ """ import os +import re import requests from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger @@ -162,9 +163,21 @@ def zeppelin_notebook_import(self, env): # Establish connection if authentication is enabled try: - # The following credentials are created at install time by Ambari at /etc/zeppelin/conf/shiro.ini - # when Shiro auth is enabled on the Zeppelin server - zeppelin_payload = {'userName': 'admin', 'password' : 'admin'} + Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.") + # Read the Shiro admin user credentials from Zeppelin config in Ambari + seen_users = False + username = None + password = None + if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE): + seen_users = True + tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group() + userpassword = tokens.split(',')[0].strip() + username = userpassword.split('=')[0].strip() + password = userpassword.split('=')[1].strip() + else: + Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.") + + zeppelin_payload = {'userName': username, 'password' : password} conn = ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload) except: pass diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index de53e38c15..9da2487ee0 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -202,6 +202,7 @@ # Zeppelin Notebooks metron_config_zeppelin_path = format("{metron_config_path}/zeppelin") +zeppelin_shiro_ini_content = status_params.zeppelin_shiro_ini_content # kafka_security kafka_security_protocol = config['configurations']['kafka-broker'].get('security.inter.broker.protocol', 'PLAINTEXT') diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 4351814215..6f37a8532e 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -94,6 +94,7 @@ # Zeppelin zeppelin_server_url = config['configurations']['metron-env']['zeppelin_server_url'] +zeppelin_shiro_ini_content = config['configurations']['zeppelin-shiro-ini']['shiro_ini_content'] # Security stack_version_unformatted = str(config['hostLevelParams']['stack_version']) From 3a799da66ad1c9568f5fcb85cdc8ea8de8c9872a Mon Sep 17 00:00:00 2001 From: Anand Subramanian Date: Mon, 22 Jan 2018 14:46:19 +0530 Subject: [PATCH 3/3] Feedback from Nick - refactor auth check into a separate method --- .../package/scripts/indexing_commands.py | 48 +++++++++++++++++++ .../package/scripts/indexing_master.py | 36 ++------------ 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index 5a2b0f45c3..5d5cf4dec5 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -16,12 +16,15 @@ """ import os +import re +import requests import time from datetime import datetime from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute, File +from resource_management.libraries.functions import format as ambari_format import metron_service import metron_security @@ -284,3 +287,48 @@ def service_check(self, env): raise Fail("Indexing topology not running") Logger.info("Indexing service check completed successfully") + + def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env): + """ + With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password + for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details). + + This method checks if Shiro authentication is enabled on the Zeppelin server. And if enabled, it returns the + session connection details to be used for importing Zeppelin notebooks. + :param ses: Session handle + :param zeppelin_server_url: Zeppelin Server URL + :return: ses + """ + from params import params + env.set_params(params) + + # Check if authentication is enabled on the Zeppelin server + try: + ses.get(ambari_format('http://{zeppelin_server_url}/api/login')) + + # Establish connection if authentication is enabled + try: + Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.") + # Read the Shiro admin user credentials from Zeppelin config in Ambari + seen_users = False + username = None + password = None + if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE): + seen_users = True + tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group() + userpassword = tokens.split(',')[0].strip() + username = userpassword.split('=')[0].strip() + password = userpassword.split('=')[1].strip() + else: + Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.") + + zeppelin_payload = {'userName': username, 'password' : password} + ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload) + except: + pass + + # If authentication is not enabled, fall back to default method of imporing notebooks + except requests.exceptions.RequestException: + ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook')) + + return ses diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index d66e0dbe26..f4816c3c0b 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -15,7 +15,6 @@ """ import os -import re import requests from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger @@ -150,42 +149,13 @@ def elasticsearch_template_delete(self, env): def zeppelin_notebook_import(self, env): from params import params env.set_params(params) + commands = IndexingCommands(params) Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) - # With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password - # for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details). + # Check if authentication is configured on Zeppelin server, and fetch details if enabled. ses = requests.session() - - # Check if authentication is enabled on the Zeppelin server - try: - conn = ses.get(ambari_format('http://{zeppelin_server_url}/api/login')) - - # Establish connection if authentication is enabled - try: - Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.") - # Read the Shiro admin user credentials from Zeppelin config in Ambari - seen_users = False - username = None - password = None - if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE): - seen_users = True - tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group() - userpassword = tokens.split(',')[0].strip() - username = userpassword.split('=')[0].strip() - password = userpassword.split('=')[1].strip() - else: - Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.") - - zeppelin_payload = {'userName': username, 'password' : password} - conn = ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload) - except: - pass - - # If authentication is not enabled, fall back to default method of imporing notebooks - except requests.exceptions.RequestException: - conn = ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook')) - + ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env) for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"):