Merge pull request #56 from QualiSystems/wip/adam.s/179924_support_token

Wip/adam.s/179924 support token

.gitignore

@@ -7,3 +7,7 @@ drivers/customscript_shellPackage/Resource Drivers - Python/*.zip
 .eggs/
 cloudshell_cm_customscript.egg-info/
 package/cloudshell_cm_customscript.egg-info/
+.vscode
+venv
+dist
+*.stackdump

drivers/customscript_shell/drivermetadata.xml

@@ -1,4 +1,4 @@
-<Driver Description="" MainClass="driver.CustomScriptShellDriver" Name="Custom Script Shell Driver" Version="1.5.0">
+<Driver Description="" MainClass="driver.CustomScriptShellDriver" Name="Custom Script Shell Driver" Version="1.6.0">
     <Layout>
         <Category Name="General">
             <Command Description="" DisplayName="Execute Script" EnableCancellation="true" Name="execute_script" Tags="allow_unreserved" />

drivers/customscript_shell/requirements.txt

@@ -1,2 +1,2 @@
 cloudshell-shell-core>=3.1.0,<3.2.0
-cloudshell-cm-customscript>=1.5.0,<1.6.0
+cloudshell-cm-customscript>=1.6.0,<1.7.0

drivers/version.txt

@@ -1 +1 @@
-1.5.0
+1.6.0

package/cloudshell/cm/customscript/customscript_shell.py

@@ -38,6 +38,7 @@ def execute_script(self, command_context, script_conf_json, cancellation_context
                 with CloudShellSessionContext(command_context) as api:
                     cancel_sampler = CancellationSampler(cancellation_context)
                     script_conf = ScriptConfigurationParser(api).json_to_object(script_conf_json)
+
                     output_writer = ReservationOutputWriter(api, command_context)
 
                     logger.info('Downloading file from \'%s\' ...' % script_conf.script_repo.url)
@@ -64,7 +65,7 @@ def _download_script(self, script_repo, logger, cancel_sampler):
         url = script_repo.url
         auth = None
         if script_repo.username:
-            auth = HttpAuth(script_repo.username, script_repo.password)
+            auth = HttpAuth(script_repo.username, script_repo.password, script_repo.token)
         return ScriptDownloader(logger, cancel_sampler).download(url, auth)
 
     def _warn_for_unexpected_file_type(self, target_host, service, script_file, output_writer):

package/cloudshell/cm/customscript/domain/script_configuration.py

@@ -21,6 +21,7 @@ def __init__(self):
         self.url = None
         self.username = None
         self.password = None
+        self.token = None
 
 
 class HostConfiguration(object):
@@ -59,6 +60,7 @@ def json_to_object(self, json_str):
         script_conf.script_repo.url = repo.get('url')
         script_conf.script_repo.username = repo.get('username')
         script_conf.script_repo.password = repo.get('password')
+        script_conf.script_repo.token = repo.get('token')
 
         host = json_obj['hostsDetails'][0]
         script_conf.host_conf = HostConfiguration()

package/cloudshell/cm/customscript/domain/script_downloader.py

@@ -6,12 +6,14 @@
 
 from cloudshell.cm.customscript.domain.cancellation_sampler import CancellationSampler
 from cloudshell.cm.customscript.domain.script_file import ScriptFile
+from requests.models import HTTPBasicAuth
 
 
 class HttpAuth(object):
-    def __init__(self, username, password):
+    def __init__(self, username, password, token):
         self.username = username
         self.password = password
+        self.token = token
 
 
 class ScriptDownloader(object):
@@ -23,53 +25,100 @@ def __init__(self, logger, cancel_sampler):
         :type cancel_sampler: CancellationSampler
         """
         self.logger = logger
-        self.cancel_sampler = cancel_sampler
-        self.filename_pattern = r'(?P<filename>^.*\.(sh|bash|ps1)$)'
+        self.cancel_sampler = cancel_sampler        
+        self.filename_pattern = r"(?P<filename>^.*\.?[^/\\&\?]+\.(sh|bash|ps1)(?=([\?&].*$|$)))" #this regex is to extract the filename from the url, works for cases: filename is at the end, parameter token is at the end
         self.filename_patterns = {
             "content-disposition": "\s*((?i)inline|attachment|extension-token)\s*;\s*filename=" + self.filename_pattern,
             "x-artifactory-filename": self.filename_pattern
         }
 
+
     def download(self, url, auth):
         """
         :type url: str
         :type auth: HttpAuth
         :rtype ScriptFile
         """
-        response = requests.get(url, auth=(auth.username, auth.password) if auth else None, stream=True)
-        file_name = self._get_filename(response)
         file_txt = ''
+        response_valid = False
+
+        # assume repo is public, try to download without credentials
+        self.logger.info("Starting download script as public...")
+        response = requests.get(url, auth=None, stream=True)
+        response_valid = self._is_response_valid(response, "public")
+
+        if response_valid:
+            file_name = self._get_filename(response)
+
+        # repo is private and token provided
+        if not response_valid and auth.token is not None:
+            self.logger.info("Token provided. Starting download script with Token...")
+            headers = {"Authorization": "Bearer %s" % auth.token }
+            response = requests.get(url, stream=True, headers=headers)
+
+            response_valid = self._is_response_valid(response, "Token")
+
+            if response_valid:
+                file_name = self._get_filename(response)
+
+        # repo is private and credentials provided, and Token did not provided or did not work. this will NOT work for github. github require Token
+        if not response_valid and (auth.username is not None and auth.password is not None):
+            self.logger.info("username\password provided, Starting download script with username\password...")
+            response = requests.get(url, auth=(auth.username, auth.password) , stream=True)
+            file_name = self._get_filename(response)
+
+            response_valid = self._is_response_valid(response, "username\password")
+
+            if response_valid:
+                file_name = self._get_filename(response)
+
+        if not response_valid:
+            raise Exception('Failed to download script file. please check the logs for more details.')
 
         for chunk in response.iter_content(ScriptDownloader.CHUNK_SIZE):
             if chunk:
                 file_txt += ''.join(chunk)
             self.cancel_sampler.throw_if_canceled()
 
-        self._validate_response(response, file_txt)
+        self._validate_file(file_txt)
 
         return ScriptFile(name=file_name, text=file_txt)
+
+    def _is_response_valid(self, response, request_method):
+        try:
+            self._validate_response(response)
+            response_valid = True
+        except Exception as ex:
+            failure_message = "failed to Authorize repository with %s" % request_method
+            self.logger.error(failure_message + " :" + str(ex))
+            response_valid = False
 
-    def _validate_response(self, response, content):
-        if response.status_code < 200 or response.status_code > 300:
-            raise Exception('Failed to download script file: '+str(response.status_code)+' '+response.reason+
-                            '. Please make sure the URL is valid, and the credentials are correct and necessary.')
+        return response_valid
 
+    def _validate_file(self, content):
         if content.lstrip('\n\r').lower().startswith('<!doctype html>'):
             raise Exception('Failed to download script file: url points to an html file')
 
+    def _validate_response(self, response):
+        if response.status_code < 200 or response.status_code > 300:            
+            raise Exception('Failed to download script file: '+str(response.status_code)+' '+response.reason+
+                              '. Please make sure the URL is valid, and the credentials are correct and necessary.')
+
     def _get_filename(self, response):
         file_name = None
         for header_value, pattern in self.filename_patterns.iteritems():
             matching = re.match(pattern, response.headers.get(header_value, ""))
             if matching:
                 file_name = matching.group('filename')
                 break
+
         # fallback, couldn't find file name from header, get it from url
         if not file_name:
             file_name_from_url = urllib.unquote(response.url[response.url.rfind('/') + 1:])
             matching = re.match(self.filename_pattern, file_name_from_url)
             if matching:
                 file_name = matching.group('filename')
+
         if not file_name:
             raise Exception("Script file of supported types: '.sh', '.bash', '.ps1' was not found")
         return file_name.strip()

package/test_requirements.txt

@@ -1,3 +1,3 @@
 nose
 mock
-unittest2
+unittest2

package/tests/helpers.py

@@ -1,6 +1,47 @@
-
 class Any(object):
     def __init__(self, predicate=None):
         self.predicate = predicate
     def __eq__(self, other):
         return not self.predicate or self.predicate(other)
+
+def mocked_requests_get(*args, **kwargs):
+    '''
+    Helper method for mocking requests for test_script_downloader
+    '''
+    repo_dict = {
+        "public": 'https://raw.repocontentservice.com/SomeUser/SomePublicRepo/master/bashScript.sh', 
+        "private_token": 'https://raw.repocontentservice.com/SomeUser/SomePrivateTokenRepo/master/bashScript.sh',
+        "private_cred": 'https://raw.repocontentservice.com/SomeUser/SomePrivateCredRepo/master/bashScript.sh',
+        "content": 'SomeBashScriptContent'
+    }
+
+    class MockResponse:
+        def __init__(self, json_data, status_code, headers, url):
+            self.json_data = json_data
+            self.status_code = status_code
+            self.headers = headers
+            self.url = url
+
+        def json(self):
+            return self.json_data
+
+        def iter_content(self, chunk):
+            return self.json_data
+
+    if args[0] == repo_dict['public']:
+        response = MockResponse(repo_dict['content'], 200, {"Content-Type": "text/plain"}, repo_dict['public'])
+        return response
+
+    if args[0] == repo_dict['private_token']:
+        if 'headers' in kwargs:
+            if kwargs["headers"]["Authorization"] == 'Bearer 551e48b030e1a9f334a330121863e48e43f58c55':
+                response = MockResponse(repo_dict['content'], 200, {"Content-Type": "text/plain"}, repo_dict['private_token'])
+                return response
+
+    if args[0] == repo_dict['private_cred']:
+        if 'auth' in kwargs and kwargs["auth"] is not None:
+            if kwargs["auth"][0] == 'SomeUser' and kwargs["auth"][1] == 'SomePassword':
+                response = MockResponse(repo_dict['content'], 200, {"Content-Type": "text/plain"}, repo_dict['private_cred'])
+                return response
+
+    return MockResponse(None, 404, None, None)

package/tests/test_script_downloader.py

@@ -0,0 +1,72 @@
+from unittest import TestCase
+
+from cloudshell.cm.customscript.domain.script_executor import ExcutorConnectionError
+from mock import patch, Mock
+import mock
+from cloudshell.cm.customscript.customscript_shell import CustomScriptShell
+from cloudshell.cm.customscript.domain.reservation_output_writer import ReservationOutputWriter
+from cloudshell.cm.customscript.domain.script_configuration import ScriptConfiguration
+from cloudshell.cm.customscript.domain.script_file import ScriptFile
+from cloudshell.cm.customscript.domain.script_downloader import ScriptDownloader, HttpAuth
+from cloudshell.cm.customscript.domain.script_configuration import ScriptRepository
+from tests.helpers import mocked_requests_get
+
+from tests.helpers import Any
+
+def print_logs(message):
+    print(message)
+
+class TestScriptDownloader(TestCase):
+
+    def setUp(self):
+        self.logger = Mock()
+        self.cancel_sampler = Mock()
+        self.logger_patcher = patch('cloudshell.cm.customscript.customscript_shell.LoggingSessionContext')
+        self.logger_patcher.start()
+        self.script_repo = ScriptRepository()
+        pass
+
+    @mock.patch('cloudshell.cm.customscript.domain.script_downloader.requests.get', side_effect=mocked_requests_get)
+    def test_download_as_public(self, mock_requests):
+        # public - url, no credentials
+        public_repo_url = 'https://raw.repocontentservice.com/SomeUser/SomePublicRepo/master/bashScript.sh'
+        self.auth = HttpAuth('','','')
+
+        # set downloaded and downaload
+        self.logger.info = print_logs
+        script_downloader = ScriptDownloader(self.logger, self.cancel_sampler)
+        script_file = script_downloader.download(public_repo_url, self.auth)
+
+        # assert name and content
+        self.assertEqual(script_file.name, "bashScript.sh")
+        self.assertEqual(script_file.text, "SomeBashScriptContent")
+
+    @mock.patch('cloudshell.cm.customscript.domain.script_downloader.requests.get', side_effect=mocked_requests_get)
+    def test_download_as_private_with_token(self, mocked_requests_get):
+        # private - url, with token
+        private_repo_url = 'https://raw.repocontentservice.com/SomeUser/SomePrivateTokenRepo/master/bashScript.sh'
+        self.auth = HttpAuth('','','551e48b030e1a9f334a330121863e48e43f58c55')
+
+        # set downloaded and downaload
+        self.logger.info = print_logs
+        script_downloader = ScriptDownloader(self.logger, self.cancel_sampler)
+        script_file = script_downloader.download(private_repo_url, self.auth)
+
+        # assert name and content
+        self.assertEqual(script_file.name, "bashScript.sh")
+        self.assertEqual(script_file.text, "SomeBashScriptContent")
+
+    @mock.patch('cloudshell.cm.customscript.domain.script_downloader.requests.get', side_effect=mocked_requests_get)
+    def test_download_as_private_with_credentials_and_failed_token(self, mocked_requests_get):
+        # private - url, with token that fails and user\password. note - this is will not work on GitHub repo, they require token
+        private_repo_url = 'https://raw.repocontentservice.com/SomeUser/SomePrivateCredRepo/master/bashScript.sh'
+        self.auth = HttpAuth('SomeUser','SomePassword','551e48b030e1a9f334a330121863e48e43f0000')
+
+        # set downloaded and downaload
+        self.logger.info = print_logs
+        script_downloader = ScriptDownloader(self.logger, self.cancel_sampler)
+        script_file = script_downloader.download(private_repo_url, self.auth)
+
+        # assert name and content
+        self.assertEqual(script_file.name, "bashScript.sh")
+        self.assertEqual(script_file.text, "SomeBashScriptContent")

package/version.txt

@@ -1 +1 @@
-1.5.0
+1.6.0

version.txt

@@ -1 +1 @@
-1.5.0
+1.6.0