From 67800b11e40e3ca4f51ab96a6006c17b98af4bad Mon Sep 17 00:00:00 2001 From: "QUALISYSTEMS\\nahum-t" Date: Tue, 20 Apr 2021 09:28:34 +0300 Subject: [PATCH 1/3] allow custom script to ignore SSL if configured on service --- drivers/customscript_shell/drivermetadata.xml | 2 +- .../DataModel/datamodel.xml | 11 +++-- .../cm/customscript/customscript_shell.py | 6 +-- .../domain/script_configuration.py | 2 + .../customscript/domain/script_downloader.py | 10 +++-- package/requirements.txt | 4 +- package/tests/test_script_configuration.py | 42 +++++++++++++++++++ 7 files changed, 64 insertions(+), 13 deletions(-) diff --git a/drivers/customscript_shell/drivermetadata.xml b/drivers/customscript_shell/drivermetadata.xml index 6f68967..c66b5ab 100644 --- a/drivers/customscript_shell/drivermetadata.xml +++ b/drivers/customscript_shell/drivermetadata.xml @@ -1,4 +1,4 @@ - + diff --git a/drivers/customscript_shellPackage/DataModel/datamodel.xml b/drivers/customscript_shellPackage/DataModel/datamodel.xml index f734405..cf93870 100644 --- a/drivers/customscript_shellPackage/DataModel/datamodel.xml +++ b/drivers/customscript_shellPackage/DataModel/datamodel.xml @@ -1,9 +1,12 @@ - - - + + + + + @@ -20,10 +23,12 @@ + + diff --git a/package/cloudshell/cm/customscript/customscript_shell.py b/package/cloudshell/cm/customscript/customscript_shell.py index be8b265..38f81ef 100644 --- a/package/cloudshell/cm/customscript/customscript_shell.py +++ b/package/cloudshell/cm/customscript/customscript_shell.py @@ -41,7 +41,7 @@ def execute_script(self, command_context, script_conf_json, cancellation_context output_writer = ReservationOutputWriter(api, command_context) logger.info('Downloading file from \'%s\' ...' % script_conf.script_repo.url) - script_file = self._download_script(script_conf.script_repo, logger, cancel_sampler) + script_file = self._download_script(script_conf.script_repo, logger, cancel_sampler, script_conf.verify_certificate) logger.info('Done (%s, %s chars).' % (script_file.name, len(script_file.text))) service = ScriptExecutorSelector.get(script_conf.host_conf, logger, cancel_sampler) @@ -54,7 +54,7 @@ def execute_script(self, command_context, script_conf_json, cancellation_context service.execute(script_file, script_conf.host_conf.parameters, output_writer, script_conf.print_output) - def _download_script(self, script_repo, logger, cancel_sampler): + def _download_script(self, script_repo, logger, cancel_sampler, verify_certificate): """ :type script_repo: ScriptRepository :type logger: Logger @@ -65,7 +65,7 @@ def _download_script(self, script_repo, logger, cancel_sampler): auth = None if script_repo.username or script_repo.token: auth = HttpAuth(script_repo.username, script_repo.password, script_repo.token) - return ScriptDownloader(logger, cancel_sampler).download(url, auth) + return ScriptDownloader(logger, cancel_sampler).download(url, auth, verify_certificate) def _warn_for_unexpected_file_type(self, target_host, service, script_file, output_writer): """ diff --git a/package/cloudshell/cm/customscript/domain/script_configuration.py b/package/cloudshell/cm/customscript/domain/script_configuration.py index 0c652b3..e4aa3b2 100644 --- a/package/cloudshell/cm/customscript/domain/script_configuration.py +++ b/package/cloudshell/cm/customscript/domain/script_configuration.py @@ -14,6 +14,7 @@ def __init__(self, script_repo = None, host_conf = None, timeout_minutes = None, self.script_repo = script_repo or ScriptRepository() self.host_conf = host_conf or HostConfiguration() self.print_output = print_output + self.verify_certificate = True class ScriptRepository(object): @@ -55,6 +56,7 @@ def json_to_object(self, json_str): script_conf = ScriptConfiguration() script_conf.timeout_minutes = json_obj.get('timeoutMinutes', 0.0) script_conf.print_output = bool_parse(json_obj.get('printOutput', True)) + script_conf.verify_certificate = json_obj.get('verifyCertificate', 'true').lower()=='true' repo = json_obj['repositoryDetails'] script_conf.script_repo.url = repo.get('url') diff --git a/package/cloudshell/cm/customscript/domain/script_downloader.py b/package/cloudshell/cm/customscript/domain/script_downloader.py index 753132f..4f37a34 100644 --- a/package/cloudshell/cm/customscript/domain/script_downloader.py +++ b/package/cloudshell/cm/customscript/domain/script_downloader.py @@ -33,7 +33,7 @@ def __init__(self, logger, cancel_sampler): } - def download(self, url, auth): + def download(self, url, auth, verify_certificate): """ :type url: str :type auth: HttpAuth @@ -44,7 +44,9 @@ def download(self, url, auth): # assume repo is public, try to download without credentials self.logger.info("Starting download script as public...") - response = requests.get(url, auth=None, stream=True) + if not verify_certificate: + self.logger.info("Skipping server certificate") + response = requests.get(url, auth=None, stream=True, verify=verify_certificate) response_valid = self._is_response_valid(response, "public") if response_valid: @@ -54,7 +56,7 @@ def download(self, url, auth): if not response_valid and auth.token is not None: self.logger.info("Token provided. Starting download script with Token...") headers = {"Authorization": "Bearer %s" % auth.token } - response = requests.get(url, stream=True, headers=headers) + response = requests.get(url, stream=True, headers=headers, verify=verify_certificate) response_valid = self._is_response_valid(response, "Token") @@ -64,7 +66,7 @@ def download(self, url, auth): # repo is private and credentials provided, and Token did not provided or did not work. this will NOT work for github. github require Token if not response_valid and (auth.username is not None and auth.password is not None): self.logger.info("username\password provided, Starting download script with username\password...") - response = requests.get(url, auth=(auth.username, auth.password) , stream=True) + response = requests.get(url, auth=(auth.username, auth.password) , stream=True, verify=verify_certificate) file_name = self._get_filename(response) response_valid = self._is_response_valid(response, "username\password") diff --git a/package/requirements.txt b/package/requirements.txt index f0eb125..34cc2c6 100644 --- a/package/requirements.txt +++ b/package/requirements.txt @@ -1,7 +1,7 @@ -cloudshell-automation-api>=2021.1.0,<2021.2.1 +cloudshell-automation-api>=2021.2.0,<2021.2.1 cloudshell-shell-core>=5.0.0,<6.0.0 cloudshell-core>=2.2.0,<2.3.0 pywinrm>=0.2.2 paramiko==2.7.2 scpclient>=0.7 -requests>=2.16.2 \ No newline at end of file +requests>=2.25.0,<2.26.0 \ No newline at end of file diff --git a/package/tests/test_script_configuration.py b/package/tests/test_script_configuration.py index 8aa4a7f..b54f494 100644 --- a/package/tests/test_script_configuration.py +++ b/package/tests/test_script_configuration.py @@ -28,6 +28,48 @@ def test_cannot_parse_json_without_repository_details(self): self.parser.json_to_object(json) self.assertIn('Missing "repositoryDetails" node.', str(context.exception)) + def test_verify_certificate_false(self): + json = """ + { + "verifyCertificate": "nope", + "timeoutMinutes": 12.3, + "repositoryDetails" : { + "url": "B", + "username": "C", + "password": "D" + }, + "hostsDetails": [{ + "ip": "E", + "username": "F", + "password": "G", + "accessKey": "H", + "connectionMethod": "IiIiI" + }] + }""" + res = self.parser.json_to_object(json) + self.assertEqual(res.verify_certificate, False) + + def test_verify_certificate_true(self): + json = """ + { + "verifyCertificate": "TrUe", + "timeoutMinutes": 12.3, + "repositoryDetails" : { + "url": "B", + "username": "C", + "password": "D" + }, + "hostsDetails": [{ + "ip": "E", + "username": "F", + "password": "G", + "accessKey": "H", + "connectionMethod": "IiIiI" + }] + }""" + res = self.parser.json_to_object(json) + self.assertEqual(res.verify_certificate, True) + def test_cannot_parse_json_without_repository_url(self): json = '{"repositoryDetails":{}}' with self.assertRaises(SyntaxError) as context: From 10818acb64358d36bbf286a00471e2821b47d970 Mon Sep 17 00:00:00 2001 From: "QUALISYSTEMS\\nahum-t" Date: Tue, 20 Apr 2021 09:44:09 +0300 Subject: [PATCH 2/3] fixed some tests --- package/tests/test_script_configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/tests/test_script_configuration.py b/package/tests/test_script_configuration.py index b54f494..787e85e 100644 --- a/package/tests/test_script_configuration.py +++ b/package/tests/test_script_configuration.py @@ -156,7 +156,7 @@ def wrapIt(x): self.assertEqual("decrypted-G", conf.host_conf.password) self.assertEqual("decrypted-H", conf.host_conf.access_key) self.assertEqual("iiiii", conf.host_conf.connection_method) - self.assertCountEqual('K12', conf.host_conf.parameters['K11']) - self.assertCountEqual('K22', conf.host_conf.parameters['K21']) + self.assertEqual('K12', conf.host_conf.parameters['K11']) + self.assertEqual('K22', conf.host_conf.parameters['K21']) self.api.DecryptPassword.assert_any_call('G') self.api.DecryptPassword.assert_any_call('H') \ No newline at end of file From 3cbc68553ebba37079d8fe9a993c7308ca0b2b57 Mon Sep 17 00:00:00 2001 From: "QUALISYSTEMS\\nahum-t" Date: Tue, 20 Apr 2021 09:49:27 +0300 Subject: [PATCH 3/3] fixed some tests --- package/tests/test_customscript_shell.py | 4 ++-- package/tests/test_script_downloader.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package/tests/test_customscript_shell.py b/package/tests/test_customscript_shell.py index 16b2874..ca41b7d 100644 --- a/package/tests/test_customscript_shell.py +++ b/package/tests/test_customscript_shell.py @@ -54,7 +54,7 @@ def test_download_script_without_auth(self): CustomScriptShell().execute_script(self.context, '', self.cancel_context) - self.downloader.assert_called_with('some url', None) + self.downloader.assert_called_with('some url', None, True) def test_download_script_with_auth(self): self.script_conf.script_repo.url = 'some url' @@ -63,7 +63,7 @@ def test_download_script_with_auth(self): CustomScriptShell().execute_script(self.context, '', self.cancel_context) - self.downloader.assert_called_with('some url', Any(lambda x: x.username == 'admin' and x.password=='1234')) + self.downloader.assert_called_with('some url', Any(lambda x: x.username == 'admin' and x.password=='1234'), True) def test_selector_is_called_with_host_details(self): CustomScriptShell().execute_script(self.context, '', self.cancel_context) diff --git a/package/tests/test_script_downloader.py b/package/tests/test_script_downloader.py index 1c3094c..c2c4190 100644 --- a/package/tests/test_script_downloader.py +++ b/package/tests/test_script_downloader.py @@ -35,7 +35,7 @@ def test_download_as_public(self, mock_requests): # set downloaded and downaload self.logger.info = print_logs script_downloader = ScriptDownloader(self.logger, self.cancel_sampler) - script_file = script_downloader.download(public_repo_url, self.auth) + script_file = script_downloader.download(public_repo_url, self.auth, True) # assert name and content self.assertEqual(script_file.name, "bashScript.sh") @@ -50,7 +50,7 @@ def test_download_as_private_with_token(self, mocked_requests_get): # set downloaded and downaload self.logger.info = print_logs script_downloader = ScriptDownloader(self.logger, self.cancel_sampler) - script_file = script_downloader.download(private_repo_url, self.auth) + script_file = script_downloader.download(private_repo_url, self.auth, True) # assert name and content self.assertEqual(script_file.name, "bashScript.sh") @@ -65,7 +65,7 @@ def test_download_as_private_with_credentials_and_failed_token(self, mocked_requ # set downloaded and downaload self.logger.info = print_logs script_downloader = ScriptDownloader(self.logger, self.cancel_sampler) - script_file = script_downloader.download(private_repo_url, self.auth) + script_file = script_downloader.download(private_repo_url, self.auth, True) # assert name and content self.assertEqual(script_file.name, "bashScript.sh")