Merge pull request #101 from QualiSystems/update_to_python3

Update to python3

.travis.yml

@@ -1,12 +1,13 @@
 language: python
 python:
-    - "2.7"
+    - "3.7.1"
 
 install:
   - pip install -r external_requirements.txt --extra-index-url https://pypi.python.org/simple
   - pip install -r test_requirements.txt --extra-index-url https://pypi.python.org/simple
-  - pip install "cloudshell-shell-core>=3.1.0,<3.2.0" --extra-index-url https://testpypi.python.org/simple
-  - pip install "cloudshell-automation-api>=8.3.0.0,<8.3.1.0" --extra-index-url https://testpypi.python.org/simple
+  - pip install "cloudshell-shell-core>=5.0.0,<6.0.0" --extra-index-url https://testpypi.python.org/simple
+  - pip install "cloudshell-automation-api>=2021.2.0,<2021.2.1" --extra-index-url https://testpypi.python.org/simple
+  - pip install "paramiko==2.7.2" --extra-index-url https://pypi.python.org/simple
 
 script:
     - pushd package

drivers/ansible_shell/drivermetadata.xml

@@ -1,4 +1,4 @@
-<Driver Description="" MainClass="driver.AnsibleShellDriver" Name="Ansible Shell Driver" Version="1.6.0">
+<Driver Description="" MainClass="driver.AnsibleShellDriver" Name="Ansible Shell Driver" Version="2.0.0" PythonVersion="3">
     <Layout>
         <Category Name="General">
             <Command Description="" DisplayName="Execute Playbook" EnableCancellation="true" Name="execute_playbook" Tags="allow_unreserved" />

drivers/ansible_shell/requirements.txt

@@ -1,2 +1,2 @@
-cloudshell-shell-core>=3.1.0,<3.2.0
-cloudshell-cm-ansible>=1.6.0,<1.7.0
+cloudshell-shell-core>=5.0.0,<6.0.0
+cloudshell-cm-ansible>=2.0.0,<2.1.0

drivers/version.txt

@@ -1 +1 @@
-1.6.0
+2.0.0

package/cloudshell/cm/ansible/ansible_shell.py

@@ -106,7 +106,7 @@ def _add_host_vars_files(self, ansi_conf, logger):
                     file.add_password(host_conf.password)
                 else:
                     file_name = host_conf.ip + '_access_key.pem'
-                    with self.file_system.create_file(file_name, 0400) as file_stream:
+                    with self.file_system.create_file(file_name, 0o400) as file_stream:
                         file_stream.write(host_conf.access_key)
                     file.add_conn_file(file_name)
 
@@ -160,7 +160,7 @@ def _wait_for_all_hosts_to_be_deployed(self, ansi_conf, logger, output_writer):
             logger.info("Trying to connect to host:" + host.ip)
             ansible_port = self.ansible_connection_helper.get_ansible_port(host)
 
-            if HostVarsFile.ANSIBLE_PORT in host.parameters.keys() and (
+            if HostVarsFile.ANSIBLE_PORT in list(host.parameters.keys()) and (
                     host.parameters[HostVarsFile.ANSIBLE_PORT] != '' and
                     host.parameters[HostVarsFile.ANSIBLE_PORT] is not None):
                 ansible_port = host.parameters[HostVarsFile.ANSIBLE_PORT]

package/cloudshell/cm/ansible/domain/ansible_command_executor.py

@@ -8,7 +8,6 @@
 from cloudshell.cm.ansible.domain.cancellation_sampler import CancellationSampler
 from cloudshell.cm.ansible.domain.output.unixToHtmlConverter import UnixToHtmlColorConverter
 from cloudshell.cm.ansible.domain.output.ansible_result import AnsibleResult
-from cloudshell.shell.core.context import ResourceCommandContext
 from cloudshell.cm.ansible.domain.stdout_accumulator import StdoutAccumulator, StderrAccumulator
 
 

package/cloudshell/cm/ansible/domain/ansible_config_file.py

@@ -1,4 +1,4 @@
-from file_system_service import FileSystemService
+from .file_system_service import FileSystemService
 from logging import Logger
 import os
 
@@ -22,7 +22,7 @@ def __enter__(self):
     def __exit__(self, type, value, traceback):
         with self.file_system.create_file(AnsibleConfigFile.FILE_NAME) as file_stream:
             lines = ['[defaults]']
-            for key, value in self.config_keys.iteritems():
+            for key, value in self.config_keys.items():
                 lines.append(key + ' = ' + value)
             file_stream.write(os.linesep.join(lines))
             self.logger.debug(os.linesep.join(lines))

package/cloudshell/cm/ansible/domain/connection_service.py

@@ -1,6 +1,6 @@
 import re
 import socket
-from StringIO import StringIO
+from io import StringIO
 from abc import ABCMeta, abstractmethod
 from multiprocessing.pool import ThreadPool
 from uuid import uuid4
@@ -15,9 +15,7 @@
 from paramiko import SSHClient, AutoAddPolicy, RSAKey
 
 
-class IVMConnectionService(object):
-    __metaclass__ = ABCMeta
-
+class IVMConnectionService(object, metaclass=ABCMeta):
     @abstractmethod
     def check_connection(self, target_host, logger, ansible_port):
         pass
@@ -87,7 +85,7 @@ def check_connection(self, target_host, logger, ansible_port):
                 raise Exception('Both password and access key are empty.')
             logger.info("Done testing connection")
         except NoValidConnectionsError as e:
-            error_code = next(e.errors.itervalues(), type('e', (object,), {'errno': 0})).errno
+            error_code = next(iter(e.errors.values()), type('e', (object,), {'errno': 0})).errno
             raise ExcutorConnectionError(error_code, e)
         except socket.error as e:
             raise ExcutorConnectionError(e.errno, e)

package/cloudshell/cm/ansible/domain/filename_extractor.py

@@ -1,4 +1,4 @@
-import urllib
+import urllib.request, urllib.parse, urllib.error
 import re
 
 from cloudshell.cm.ansible.domain.exceptions import AnsibleException
@@ -14,14 +14,14 @@ def __init__(self):
 
     def get_filename(self,response):
         file_name = None;
-        for header_value, pattern in self.filename_patterns.iteritems():
+        for header_value, pattern in self.filename_patterns.items():
             matching = re.match(pattern, response.headers.get(header_value,""))
             if matching:
                 file_name = matching.group('filename')
                 break
         #fallback, couldn't find file name from header, get it from url
         if not file_name:
-            file_name_from_url = urllib.unquote(response.url[response.url.rfind('/') + 1:])
+            file_name_from_url = urllib.parse.unquote(response.url[response.url.rfind('/') + 1:])
             matching = re.match(self._filename_pattern, file_name_from_url)
             if matching:
                 file_name = matching.group('filename')

package/cloudshell/cm/ansible/domain/host_vars_file.py

@@ -1,6 +1,6 @@
 import os
 from logging import Logger
-from file_system_service import FileSystemService
+from .file_system_service import FileSystemService
 
 
 class HostVarsFile(object):
@@ -32,7 +32,7 @@ def __exit__(self, type, value, traceback):
             self.file_system.create_folder(HostVarsFile.FOLDER_NAME)
         with self.file_system.create_file(self.file_path) as file_stream:
             lines = ['---']
-            for key, value in sorted(self.vars.iteritems()):
+            for key, value in sorted(self.vars.items()):
                 lines.append(str(key) + ': "' + str(value) + '"')
             file_stream.write(os.linesep.join(lines))
             self.logger.debug(os.linesep.join(lines))
@@ -54,7 +54,7 @@ def add_password(self, password):
         self.vars[HostVarsFile.ANSIBLE_PASSWORD] = password
 
     def add_port(self, port):
-        if HostVarsFile.ANSIBLE_PORT not in self.vars.keys() or \
+        if HostVarsFile.ANSIBLE_PORT not in list(self.vars.keys()) or \
                 (self.vars[HostVarsFile.ANSIBLE_PORT] == '') or \
                         self.vars[HostVarsFile.ANSIBLE_PORT] is None:
             self.vars[HostVarsFile.ANSIBLE_PORT] = port

package/cloudshell/cm/ansible/domain/inventory_file.py

@@ -1,6 +1,6 @@
 import os
 from collections import OrderedDict
-from file_system_service import FileSystemService
+from .file_system_service import FileSystemService
 
 
 class InventoryFile(object):

package/cloudshell/cm/ansible/domain/output/unixToHtmlConverter.py

@@ -32,7 +32,7 @@ def _add_font_tag(self, x):
 
     def convert(self, text):
         result = '<html><body><font color=white>'
-        p_object = re.compile('|'.join(self.unixToHtml.keys()))
+        p_object = re.compile('|'.join(list(self.unixToHtml.keys())))
         result += p_object.sub(lambda x: self._add_font_tag(x), text)
         result += '</font></body></html>'
         # result = '<br />'.join([line for line in result.split(os.linesep) if line])

package/cloudshell/cm/ansible/domain/stdout_accumulator.py

@@ -1,5 +1,5 @@
 import os
-from Queue import Queue, Empty
+from queue import Queue, Empty
 from threading import Thread, RLock
 
 class StreamAccumulator(object):

package/cloudshell/cm/ansible/domain/temp_folder_scope.py

@@ -1,5 +1,5 @@
 import os
-from file_system_service import FileSystemService
+from .file_system_service import FileSystemService
 from logging import Logger
 
 

package/requirements.txt

@@ -1,6 +1,7 @@
-cloudshell-automation-api>=8.3.0.0,<8.3.1.0
-cloudshell-shell-core>=3.1.0,<3.2.0
-requests==2.14.2
-pywinrm==0.2.2
-paramiko==2.1.2
-
+cloudshell-automation-api>=2021.2.0,<2021.2.1
+cloudshell-shell-core>=5.0.0,<6.0.0
+cloudshell-core>=2.2.0,<2.3.0
+pywinrm>=0.2.2
+paramiko==2.7.2
+requests>=2.16.2
+jsonpickle

package/setup.py

@@ -28,6 +28,6 @@
             "Development Status :: 4 - Beta",
             "Topic :: Software Development :: Libraries",
             "License :: OSI Approved :: Apache Software License",
-        ], requires=['jsonpickle']
-
+        ], 
+        #requires=required,
 )

package/tests/test_ansible_command_executor.py

@@ -3,7 +3,7 @@
 from mock import Mock, MagicMock, patch
 from subprocess import PIPE
 from cloudshell.cm.ansible.domain.ansible_command_executor import AnsibleCommandExecutor
-from helpers import mock_enter_exit_self
+from .helpers import mock_enter_exit_self
 
 
 class TestAnsibleCommandExecutor(TestCase):

package/tests/test_ansible_config_file.py

@@ -1,6 +1,6 @@
 from unittest import TestCase
 from cloudshell.cm.ansible.domain.ansible_config_file import AnsibleConfigFile
-from mocks.file_system_service_mock import FileSystemServiceMock
+from .mocks.file_system_service_mock import FileSystemServiceMock
 from mock import Mock
 import os
 
@@ -12,14 +12,14 @@ def setUp(self):
     def test_can_add_ignore_ssh_key_checking(self):
         with AnsibleConfigFile(self.file_system, Mock()) as f:
             f.ignore_ssh_key_checking()
-        self.assertEquals(os.linesep.join(['[defaults]', 'host_key_checking = False']), self.file_system.read_all_lines('ansible.cfg'))
+        self.assertEqual(os.linesep.join(['[defaults]', 'host_key_checking = False']), self.file_system.read_all_lines('ansible.cfg'))
 
     def test_can_add_force_color(self):
         with AnsibleConfigFile(self.file_system, Mock()) as f:
             f.force_color()
-        self.assertEquals
+        self.assertEqual
 
     def test_can_add_set_retry_path(self):
         with AnsibleConfigFile(self.file_system, Mock()) as f:
             f.set_retry_path(678)
-        self.assertEquals(os.linesep.join(['[defaults]', 'retry_files_save_path = 678']), self.file_system.read_all_lines('ansible.cfg'))
+        self.assertEqual(os.linesep.join(['[defaults]', 'retry_files_save_path = 678']), self.file_system.read_all_lines('ansible.cfg'))

package/tests/test_ansible_configutarion.py

@@ -16,43 +16,43 @@ def test_cannot_parse_json_without_repository_details(self):
         json = '{}'
         with self.assertRaises(SyntaxError) as context:
             self.parser.json_to_object(json)
-        self.assertIn('Missing "repositoryDetails" node.', context.exception.message)
+        self.assertIn('Missing "repositoryDetails" node.', str(context.exception))
 
     def test_cannot_parse_json_without_repository_url(self):
         json = '{"repositoryDetails":{}}'
         with self.assertRaises(SyntaxError) as context:
             self.parser.json_to_object(json)
-        self.assertIn('Missing "repositoryDetails.url" node.', context.exception.message)
+        self.assertIn('Missing "repositoryDetails.url" node.', str(context.exception))
 
     def test_cannot_parse_json_with_an_empty_repository_url(self):
         json = '{"repositoryDetails":{"url":""}}'
         with self.assertRaises(SyntaxError) as context:
             self.parser.json_to_object(json)
-        self.assertIn('"repositoryDetails.url" node cannot be empty.', context.exception.message)
+        self.assertIn('"repositoryDetails.url" node cannot be empty.', str(context.exception))
 
     def test_cannot_parse_json_without_hosts_detalis(self):
         json = '{"repositoryDetails":{"url":"someurl"}}'
         with self.assertRaises(SyntaxError) as context:
             self.parser.json_to_object(json)
-        self.assertIn('Missing "hostsDetails" node.', context.exception.message)
+        self.assertIn('Missing "hostsDetails" node.', str(context.exception))
 
     def test_cannot_parse_json_with_an_empty_hosts_detalis(self):
         json = '{"repositoryDetails":{"url":"someurl"},"hostsDetails":[]}'
         with self.assertRaises(SyntaxError) as context:
             self.parser.json_to_object(json)
-        self.assertIn('"hostsDetails" node cannot be empty.', context.exception.message)
+        self.assertIn('"hostsDetails" node cannot be empty.', str(context.exception))
 
     def test_cannot_parse_json_with_host_with_an_empty_address(self):
         json = '{"repositoryDetails":{"url":"someurl"},"hostsDetails":[{}]}'
         with self.assertRaises(SyntaxError) as context:
             self.parser.json_to_object(json)
-        self.assertIn('Missing "ip" node in 1 hosts', context.exception.message)
+        self.assertIn('Missing "ip" node in 1 hosts', str(context.exception))
 
     def test_cannot_parse_json_with_host_with_an_empty_connection_method(self):
         json = '{"repositoryDetails":{"url":"someurl"},"hostsDetails":[{"ip":"x.x.x.x"}]}'
         with self.assertRaises(SyntaxError) as context:
             self.parser.json_to_object(json)
-        self.assertIn('Missing "connectionMethod" node in 1 hosts', context.exception.message)
+        self.assertIn('Missing "connectionMethod" node in 1 hosts', str(context.exception))
 
     def test_sanity(self):
         def wrapIt(x):
@@ -84,19 +84,19 @@ def wrapIt(x):
     }]
 }"""
         conf = self.parser.json_to_object(json)
-        self.assertEquals("A", conf.additional_cmd_args)
-        self.assertEquals("B", conf.playbook_repo.url)
-        self.assertEquals("C", conf.playbook_repo.username)
-        self.assertEquals("D", conf.playbook_repo.password)
+        self.assertEqual("A", conf.additional_cmd_args)
+        self.assertEqual("B", conf.playbook_repo.url)
+        self.assertEqual("C", conf.playbook_repo.username)
+        self.assertEqual("D", conf.playbook_repo.password)
         host1 = next((h for h in conf.hosts_conf if h.ip == 'E'), None)
-        self.assertEquals("F", host1.username)
-        self.assertEquals("decrypted-G", host1.password)
-        self.assertEquals("decrypted-H", host1.access_key)
-        self.assertEquals("iiiii", host1.connection_method)
-        self.assertEquals(False, host1.connection_secured)
-        self.assertItemsEqual(['J1','J2'], host1.groups)
-        self.assertItemsEqual('K12', host1.parameters['K11'])
-        self.assertItemsEqual('K22', host1.parameters['K21'])
+        self.assertEqual("F", host1.username)
+        self.assertEqual("decrypted-G", host1.password)
+        self.assertEqual("decrypted-H", host1.access_key)
+        self.assertEqual("iiiii", host1.connection_method)
+        self.assertEqual(False, host1.connection_secured)
+        self.assertEquals(['J1','J2'], host1.groups)
+        self.assertEquals('K12', host1.parameters['K11'])
+        self.assertEquals('K22', host1.parameters['K21'])
         host1 = next((h for h in conf.hosts_conf if h.ip == 'E2'), None)
         self.assertIsNotNone(host1)
         self.api.DecryptPassword.assert_any_call('G')

package/tests/test_ansible_result.py

@@ -17,7 +17,7 @@ def test_result_should_fail_on_general_ansible_error(self):
 \033[0;31mERROR! 'tasks_SFSDFEG' is not a valid attribute for a Play"""+os.linesep+"""The error appears to have\033[0m"""
         result = AnsibleResult('', resultTxt, ['192.168.85.11'])
         self.assertFalse(result.success)
-        self.assertEquals('Did not run / no information for this host.'+os.linesep+'\'tasks_SFSDFEG\' is not a valid attribute for a Play'+os.linesep+'The error appears to have',
+        self.assertEqual('Did not run / no information for this host.'+os.linesep+'\'tasks_SFSDFEG\' is not a valid attribute for a Play'+os.linesep+'The error appears to have',
                           get_error_for(result, '192.168.85.11'))
 
     def test_result_should_fail_on_unreachable(self):
@@ -32,7 +32,7 @@ def test_result_should_fail_on_unreachable(self):
 \033[0;31m192.168.85.11\033[0m              : ok=0    changed=0    \033[1;31munreachable=1   \033[0m failed=0"""
         result = AnsibleResult(resultTxt, '', ['192.168.85.11'])
         self.assertFalse(result.success)
-        self.assertEquals('{"changed": false, "msg": "Authentication failure.", "unreachable": true}',
+        self.assertEqual('{"changed": false, "msg": "Authentication failure.", "unreachable": true}',
                           get_error_for(result, '192.168.85.11'))
 
     def test_result_should_fail_on_failed(self):
@@ -50,7 +50,7 @@ def test_result_should_fail_on_failed(self):
 \033[0;31m192.168.85.11\033[0m              : \033[0;32mok=1   \033[0m changed=0    unreachable=0    \033[0;31mfailed=1   \033[0m"""
         result = AnsibleResult(resultTxt, '', ['192.168.85.11'])
         self.assertFalse(result.success)
-        self.assertEquals('{"changed": false, "cmd": "tauch /tmp/f", "failed": true, "msg": "[Errno 2] No such file or directory", "rc": 2}',
+        self.assertEqual('{"changed": false, "cmd": "tauch /tmp/f", "failed": true, "msg": "[Errno 2] No such file or directory", "rc": 2}',
                           get_error_for(result, '192.168.85.11'))
 
     def test_result_should_be_true(self):
@@ -85,7 +85,7 @@ def test_result_should_contain_stderr_in_no_error_msg_found(self):
 \033[0;31m192.168.85.11\033[0m              : \033[0;32mok=1   \033[0m changed=0    unreachable=0    \033[0;31mfailed=1   \033[0m"""
             result = AnsibleResult(resultTxt, 'general error', ['192.168.85.11'])
             self.assertFalse(result.success)
-            self.assertEquals('general error',
+            self.assertEqual('general error',
                               get_error_for(result, '192.168.85.11'))
 
         def test_result_should_contain_error_for_unrun_hosts(self):