Merge pull request #4662 from StackStorm/python3_flag_python_versions…

…_attribute

Update pack install action so it works correctly when --python3 flag and "python_versions" pack.yaml attribute is used

CHANGELOG.rst

@@ -11,6 +11,8 @@ Fixed
   file if ``ssh_runner.use_ssh_config`` parameter is set to ``True`` and if a custom (non-default)
   value for SSH port is specified in the configured SSH config file
   (``ssh_runner.ssh_config_file_path``). (bug fix) #4660 #4661
+* Update pack install action so it works correctly when ``python_versions`` ``pack.yaml`` metadata
+  attribute is used in combination with ``--python3`` pack install flag. (bug fix) #4654 #4662
 
 3.0.0 - April 18, 2019
 ----------------------

contrib/packs/actions/download.yaml

@@ -22,3 +22,8 @@
       description: "Set to True to force install the pack and skip StackStorm version compatibility check"
       required: false
       default: false
+    python3:
+      type: "boolean"
+      description: "True to use Python 3 binary for this pack."
+      required: false
+      default: false

contrib/packs/actions/pack_mgmt/download.py

@@ -62,14 +62,15 @@ def __init__(self, config=None, action_service=None):
         if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None):
             os.environ['no_proxy'] = self.no_proxy
 
-    def run(self, packs, abs_repo_base, verifyssl=True, force=False):
+    def run(self, packs, abs_repo_base, verifyssl=True, force=False, python3=False):
         result = {}
 
         for pack in packs:
             pack_result = download_pack(pack=pack, abs_repo_base=abs_repo_base,
                                         verify_ssl=verifyssl, force=force,
                                         proxy_config=self.proxy_config,
                                         force_permissions=True,
+                                        use_python3=python3,
                                         logger=self.logger)
             pack_url, pack_ref, pack_result = pack_result
             result[pack_ref] = pack_result

contrib/packs/actions/workflows/install.yaml

@@ -6,6 +6,7 @@
       parameters:
         packs: "{{packs}}"
         force: "{{force}}"
+        python3: "{{python3}}"
       on-success: "make a prerun"
     -
       name: "make a prerun"

contrib/packs/tests/test_action_download.py

@@ -375,6 +375,57 @@ def test_download_pack_python_version_check(self):
             result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False)
             self.assertEqual(result['test3'], 'Success.')
 
+        # StackStorm is running under Python 2, Pack requires Python 3 and --python3 flag is used
+        with mock.patch('st2common.util.pack_management.get_pack_metadata') as \
+                mock_get_pack_metadata:
+            mock_get_pack_metadata.return_value = {
+                'name': 'test3',
+                'stackstorm_version': '',
+                'python_versions': ['3']
+            }
+
+            st2common.util.pack_management.six.PY2 = True
+            st2common.util.pack_management.six.PY3 = False
+            st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5'
+
+            result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False,
+                                python3=True)
+            self.assertEqual(result['test3'], 'Success.')
+
+        with mock.patch('st2common.util.pack_management.get_pack_metadata') as \
+                mock_get_pack_metadata:
+            mock_get_pack_metadata.return_value = {
+                'name': 'test3',
+                'stackstorm_version': '',
+                'python_versions': ['2', '3']
+            }
+
+            st2common.util.pack_management.six.PY2 = True
+            st2common.util.pack_management.six.PY3 = False
+            st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5'
+
+            result = action.run(packs=['test3'], abs_repo_base=self.repo_base, force=False,
+                                python3=True)
+            self.assertEqual(result['test3'], 'Success.')
+
+        # StackStorm is running under Python 2, pack requires Python 3 and --python3 flag is used
+        with mock.patch('st2common.util.pack_management.get_pack_metadata') as \
+                mock_get_pack_metadata:
+            mock_get_pack_metadata.return_value = {
+                'name': 'test3',
+                'stackstorm_version': '',
+                'python_versions': ['2']
+            }
+
+            st2common.util.pack_management.six.PY2 = True
+            st2common.util.pack_management.six.PY3 = False
+            st2common.util.pack_management.CURRENT_PYTHON_VERSION = '2.7.5'
+
+            expected_msg = (r'Pack "test3" requires Python 2.x, but --python3 flag is used')
+            self.assertRaisesRegexp(ValueError, expected_msg, action.run,
+                                    packs=['test3'], abs_repo_base=self.repo_base, force=False,
+                                    python3=True)
+
     def test_resolve_urls(self):
         url = eval_repo_url(
             "https://github.com/StackStorm-Exchange/stackstorm-test")

st2common/st2common/cmd/download_pack.py

@@ -40,6 +40,8 @@ def _register_cli_opts():
         cfg.BoolOpt('force', default=False,
                     help='True to force pack download and ignore download '
                          'lock file if it exists.'),
+        cfg.BoolOpt('use-python3', default=False,
+                    help='True to use Python3 binary.')
     ]
     do_register_cli_opts(cli_opts)
 
@@ -54,13 +56,15 @@ def main(argv):
     packs = cfg.CONF.pack
     verify_ssl = cfg.CONF.verify_ssl
     force = cfg.CONF.force
+    use_python3 = cfg.CONF.use_python3
 
     proxy_config = get_and_set_proxy_config()
 
     for pack in packs:
         LOG.info('Installing pack "%s"' % (pack))
         result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force,
-                               proxy_config=proxy_config, force_permissions=True)
+                               proxy_config=proxy_config, force_permissions=True,
+                               use_python3=use_python3)
 
         # Raw pack name excluding the version
         pack_name = result[1]

st2common/st2common/cmd/install_pack.py

@@ -66,7 +66,8 @@ def main(argv):
         # 1. Download the pack
         LOG.info('Installing pack "%s"' % (pack))
         result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force,
-                               proxy_config=proxy_config, force_permissions=True)
+                               proxy_config=proxy_config, force_permissions=True,
+                               use_python3=use_python3)
 
         # Raw pack name excluding the version
         pack_name = result[1]

st2common/st2common/util/pack_management.py

@@ -26,6 +26,7 @@
 import re
 
 import six
+from oslo_config import cfg
 from git.repo import Repo
 from gitdb.exc import BadName, BadObject
 from lockfile import LockFile
@@ -64,7 +65,8 @@
 
 
 def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True, force=False,
-                  proxy_config=None, force_owner_group=True, force_permissions=True, logger=LOG):
+                  proxy_config=None, force_owner_group=True, force_permissions=True,
+                  use_python3=False, logger=LOG):
     """
     Download the pack and move it to /opt/stackstorm/packs.
 
@@ -84,11 +86,21 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True,
     :param force: Force the installation and ignore / delete the lock file if it already exists.
     :type force: ``bool``
 
+    :param use_python3: True if a python3 binary should be used for this pack.
+    :type use_python3: ``bool``
+
     :return: (pack_url, pack_ref, result)
     :rtype: tuple
     """
     proxy_config = proxy_config or {}
 
+    # Python3 binary check
+    python3_binary = cfg.CONF.actionrunner.python3_binary
+    if use_python3 and not python3_binary:
+        msg = ('Requested to use Python 3, but python3 binary not found on the system or '
+               'actionrunner.python3 config option is not configured correctly.')
+        raise ValueError(msg)
+
     try:
         pack_url, pack_version = get_repo_url(pack, proxy_config=proxy_config)
     except Exception as e:
@@ -125,7 +137,7 @@ def download_pack(pack, abs_repo_base='/opt/stackstorm/packs', verify_ssl=True,
 
             # 2. Verify that the pack version if compatible with current StackStorm version
             if not force:
-                verify_pack_version(pack_dir=abs_local_path)
+                verify_pack_version(pack_dir=abs_local_path, use_python3=use_python3)
 
             # 3. Move pack to the final location
             move_result = move_pack(abs_repo_base=abs_repo_base, pack_name=pack_ref,
@@ -370,7 +382,7 @@ def is_desired_pack(abs_pack_path, pack_name):
     return (True, '')
 
 
-def verify_pack_version(pack_dir):
+def verify_pack_version(pack_dir, use_python3=False):
     """
     Verify that the pack works with the currently running StackStorm version.
     """
@@ -390,12 +402,17 @@ def verify_pack_version(pack_dir):
             raise ValueError(msg)
 
     if supported_python_versions:
-        if set(supported_python_versions) == set(['2']) and not six.PY2:
-            msg = ('Pack "%s" requires Python 2.x, but current Python version is "%s". '
-                   'You can override this restriction by providing the "force" flag, but '
-                   'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION))
+        if set(supported_python_versions) == set(['2']) and (not six.PY2 or use_python3):
+            if use_python3:
+                msg = ('Pack "%s" requires Python 2.x, but --python3 flag is used. '
+                       'You can override this restriction by providing the "force" flag, but '
+                       'the pack is not guaranteed to work.' % (pack_name))
+            else:
+                msg = ('Pack "%s" requires Python 2.x, but current Python version is "%s". '
+                       'You can override this restriction by providing the "force" flag, but '
+                       'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION))
             raise ValueError(msg)
-        elif set(supported_python_versions) == set(['3']) and not six.PY3:
+        elif set(supported_python_versions) == set(['3']) and (not six.PY3 and not use_python3):
             msg = ('Pack "%s" requires Python 3.x, but current Python version is "%s". '
                    'You can override this restriction by providing the "force" flag, but '
                    'the pack is not guaranteed to work.' % (pack_name, CURRENT_PYTHON_VERSION))