From f88ac6966eff575b18563ef8829967546cf3cc3f Mon Sep 17 00:00:00 2001 From: Melchizedek13 Date: Thu, 26 Dec 2019 14:50:05 +0300 Subject: [PATCH 1/4] The protocol scheme was added in phase of finding the active RM. --- yarn_api_client/hadoop_conf.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/yarn_api_client/hadoop_conf.py b/yarn_api_client/hadoop_conf.py index e975666..99dfae5 100644 --- a/yarn_api_client/hadoop_conf.py +++ b/yarn_api_client/hadoop_conf.py @@ -29,7 +29,9 @@ def _is_https_only(): def _get_resource_manager(hadoop_conf_path, rm_id=None): # compose property name based on policy (and rm_id) - if _is_https_only(): + is_https_only = _is_https_only() + + if is_https_only: prop_name = 'yarn.resourcemanager.webapp.https.address' else: prop_name = 'yarn.resourcemanager.webapp.address' @@ -38,7 +40,7 @@ def _get_resource_manager(hadoop_conf_path, rm_id=None): if rm_id: prop_name = "{name}.{rm_id}".format(name=prop_name, rm_id=rm_id) - rm_webapp_address = parse(os.path.join(hadoop_conf_path, 'yarn-site.xml'), prop_name) + rm_webapp_address = ('https://' if is_https_only else 'http://') + parse(os.path.join(hadoop_conf_path, 'yarn-site.xml'), prop_name) return rm_webapp_address or None @@ -46,7 +48,8 @@ def _get_resource_manager(hadoop_conf_path, rm_id=None): def check_is_active_rm(url, timeout=30, auth=None, verify=True): try: response = requests.get(url + "/cluster", timeout=timeout, auth=auth, verify=verify) - except: + except requests.RequestException as e: + print("Error to access RM: {}".format(e)) return False if response.status_code != 200: From 662b0bf758e3e74bcd5ba0efea8b4dbc45bcbf0f Mon Sep 17 00:00:00 2001 From: Melchizedek13 Date: Thu, 26 Dec 2019 16:47:05 +0300 Subject: [PATCH 2/4] Fixed test "Emulate requests library exception". --- tests/test_hadoop_conf.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_hadoop_conf.py b/tests/test_hadoop_conf.py index 347ad7d..b323834 100644 --- a/tests/test_hadoop_conf.py +++ b/tests/test_hadoop_conf.py @@ -3,6 +3,7 @@ import mock from mock import patch +from requests import RequestException from tests import TestCase import requests_mock @@ -171,10 +172,8 @@ def test_check_is_active_rm(self, is_https_only_mock): # Emulate requests library exception (socket timeout, etc) with requests_mock.mock() as requests_get_mock: - requests_get_mock.side_effect = Exception('error') - # requests_get_mock.get('https://example2:8022/cluster', status_code=200) - requests_get_mock.return_value = None - self.assertFalse(hadoop_conf.check_is_active_rm('https://example2:8022')) + requests_get_mock.get('example2:8022/cluster', exc=RequestException) + self.assertFalse(hadoop_conf.check_is_active_rm('example2:8022')) def test_get_resource_manager(self): with patch('yarn_api_client.hadoop_conf.parse') as parse_mock: From 7ab2db018133fc7299148998baea5bf1e2b497a8 Mon Sep 17 00:00:00 2001 From: Melchizedek13 Date: Thu, 26 Dec 2019 17:20:24 +0300 Subject: [PATCH 3/4] The check of rm_address value for None was added. --- yarn_api_client/hadoop_conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn_api_client/hadoop_conf.py b/yarn_api_client/hadoop_conf.py index 99dfae5..9fe95dc 100644 --- a/yarn_api_client/hadoop_conf.py +++ b/yarn_api_client/hadoop_conf.py @@ -40,9 +40,9 @@ def _get_resource_manager(hadoop_conf_path, rm_id=None): if rm_id: prop_name = "{name}.{rm_id}".format(name=prop_name, rm_id=rm_id) - rm_webapp_address = ('https://' if is_https_only else 'http://') + parse(os.path.join(hadoop_conf_path, 'yarn-site.xml'), prop_name) + rm_address = parse(os.path.join(hadoop_conf_path, 'yarn-site.xml'), prop_name) - return rm_webapp_address or None + return ('https://' if is_https_only else 'http://') + rm_address if rm_address else None def check_is_active_rm(url, timeout=30, auth=None, verify=True): From fd0ebbc2c967b4d963865f64a730580f67be8316 Mon Sep 17 00:00:00 2001 From: Melchizedek13 Date: Thu, 26 Dec 2019 17:20:58 +0300 Subject: [PATCH 4/4] The protocol scheme was added in the tests. --- tests/test_hadoop_conf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_hadoop_conf.py b/tests/test_hadoop_conf.py index b323834..820768f 100644 --- a/tests/test_hadoop_conf.py +++ b/tests/test_hadoop_conf.py @@ -104,7 +104,7 @@ def test_get_resource_endpoint(self): endpoint = hadoop_conf.get_resource_manager_endpoint() - self.assertEqual('example.com:8022', endpoint) + self.assertEqual('http://example.com:8022', endpoint) parse_mock.assert_called_with(hadoop_conf_path + 'yarn-site.xml', 'yarn.resourcemanager.webapp.address') @@ -123,7 +123,7 @@ def test_get_resource_endpoint_with_ha(self, check_is_active_rm_mock, parse_mock check_is_active_rm_mock.return_value = True endpoint = hadoop_conf.get_resource_manager_endpoint() - self.assertEqual('example.com:8022', endpoint) + self.assertEqual('http://example.com:8022', endpoint) parse_mock.assert_called_with(hadoop_conf_path + 'yarn-site.xml', 'yarn.resourcemanager.webapp.address.rm1') @@ -181,12 +181,12 @@ def test_get_resource_manager(self): endpoint = hadoop_conf._get_resource_manager(hadoop_conf.CONF_DIR, None) - self.assertEqual('example.com:8022', endpoint) + self.assertEqual('http://example.com:8022', endpoint) parse_mock.assert_called_with(hadoop_conf_path + 'yarn-site.xml', 'yarn.resourcemanager.webapp.address') endpoint = hadoop_conf._get_resource_manager(hadoop_conf.CONF_DIR, 'rm1') - self.assertEqual(('example.com:8022'), endpoint) + self.assertEqual(('http://example.com:8022'), endpoint) parse_mock.assert_called_with(hadoop_conf_path + 'yarn-site.xml', 'yarn.resourcemanager.webapp.address.rm1') parse_mock.reset_mock()