Feature: road to pack management command (#69)

requirements.in

@@ -62,7 +62,7 @@ uvicorn
 gunicorn
 cryptography
 
-# SOURCE: https://github.com/esnme/ultrajson
+# SOURCE: https://github.com/ultrajson/ultrajson
 # Ultra fast JSON decoder and encoder written in C with Python binding
 ujson
 
@@ -126,3 +126,5 @@ semver
 eventlet
 ipaddr
 jsonpath_rw
+gitpython
+lockfile

requirements.txt

@@ -44,6 +44,8 @@ eventlet==0.25.2          # via -r requirements.in
 factory-boy==2.12.0       # via -r requirements.in
 faker==4.0.0              # via -r requirements.in, factory-boy
 fastapi==0.48.0           # via -r requirements.in
+gitdb==4.0.5              # via gitpython
+gitpython==3.1.3          # via -r requirements.in
 greenlet==0.4.16          # via eventlet
 gunicorn==20.0.4          # via -r requirements.in
 h11==0.9.0                # via uvicorn
@@ -57,6 +59,7 @@ jinja2-cli[yaml]==0.7.0   # via -r requirements.in
 jinja2==2.11.1            # via aiohttp-jinja2, jinja2-cli
 jsonpath-rw==1.4.0        # via -r requirements.in
 kombu==4.6.7              # via celery
+lockfile==0.12.2          # via -r requirements.in
 loggerfactory==0.0.5      # via -r requirements.in
 logging-tree==1.8.1       # via -r requirements.in
 lxml==4.5.0               # via emails, premailer
@@ -96,6 +99,7 @@ ruamel.yaml==0.16.9       # via -r requirements.in, ruamel.yaml.cmd, ruamel.yaml
 semver==2.10.2            # via -r requirements.in
 simplejson==3.17.0        # via pytool
 six==1.14.0               # via bcrypt, configobj, cryptography, ecdsa, eventlet, jsonpath-rw, pyconfig, python-dateutil, python-jose, python-multipart, pytool, sqlalchemy-utils, tenacity
+smmap==3.0.4              # via gitdb
 sqlalchemy-utils==0.36.1  # via -r requirements.in
 sqlalchemy==1.3.13        # via -r requirements.in, alembic, databases, sqlalchemy-utils
 starlette-prometheus==0.5.0  # via -r requirements.in

tasks/ci.py

@@ -330,7 +330,7 @@ def pytest(
     if pdb:
         _cmd += r" --pdb "
 
-    _cmd += r" --cov-config=setup.cfg --verbose --cov-append --cov-report=term-missing --cov-report=xml:cov.xml --cov-report=html:htmlcov --cov-report=annotate:cov_annotate --mypy --showlocals --tb=short --cov=ultron8 tests"
+    _cmd += r" --cov-config=setup.cfg -vv --cov-append --cov-report=term-missing --cov-report=xml:cov.xml --cov-report=html:htmlcov --cov-report=annotate:cov_annotate --mypy --showlocals --tb=short --cov=ultron8 tests"
 
     ctx.run(_cmd)
 
@@ -673,6 +673,7 @@ def clean_pyi(ctx, loc="local", verbose=0, dry_run=False):
         # call(pytest, loc="local", mockedfs=True),
         # call(pytest, loc="local", clionly=True),
         # call(pytest, loc="local", usersonly=True),
+        # call(pytest, loc="local", utilsonly=True),
         # call(pytest, loc="local", convertingtotestclientstarlette=True),
         # call(pytest, loc="local", loggeronly=True),
         call(pytest, loc="local"),

tests/content/test_content_loader.py

@@ -0,0 +1,77 @@
+# st2
+import os
+
+import pytest
+
+from ultron8.content.loader import LOG, ContentPackLoader
+
+CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
+RESOURCES_DIR = os.path.abspath(os.path.join(CURRENT_DIR, "../resources"))
+
+
+class ContentLoaderTest:
+    def test_get_sensors(self):
+        packs_base_path = os.path.join(RESOURCES_DIR, "packs/")
+        loader = ContentPackLoader()
+        pack_sensors = loader.get_content(
+            base_dirs=[packs_base_path], content_type="sensors"
+        )
+        assert pack_sensors.get("pack1", None) is not None
+
+    def test_get_sensors_pack_missing_sensors(self):
+        loader = ContentPackLoader()
+        fail_pack_path = os.path.join(RESOURCES_DIR, "packs/pack2")
+        assert os.path.exists(fail_pack_path)
+        assert loader._get_sensors(fail_pack_path) == None
+
+    def test_invalid_content_type(self):
+        packs_base_path = os.path.join(RESOURCES_DIR, "packs/")
+        loader = ContentPackLoader()
+        with pytest.raises(ValueError):
+            loader.get_content(base_dirs=[packs_base_path], content_type="stuff")
+
+    def test_get_content_multiple_directories(self, mocker):
+        packs_base_path_1 = os.path.join(RESOURCES_DIR, "packs/")
+        packs_base_path_2 = os.path.join(RESOURCES_DIR, "packs2/")
+        base_dirs = [packs_base_path_1, packs_base_path_2]
+
+        LOG.warning = mocker.Mock()
+
+        loader = ContentPackLoader()
+        sensors = loader.get_content(base_dirs=base_dirs, content_type="sensors")
+        assert "pack1" in sensors  # from packs/
+        assert "pack3" in sensors  # from packs2/
+
+        # Assert that a warning is emitted when a duplicated pack is found
+        expected_msg = (
+            'Pack "pack1" already found in '
+            '"%s/packs/", ignoring content from '
+            '"%s/packs2/"' % (RESOURCES_DIR, RESOURCES_DIR)
+        )
+        LOG.warning.assert_called_once_with(expected_msg)
+
+    def test_get_content_from_pack_success(self):
+        loader = ContentPackLoader()
+        pack_path = os.path.join(RESOURCES_DIR, "packs/pack1")
+
+        sensors = loader.get_content_from_pack(
+            pack_dir=pack_path, content_type="sensors"
+        )
+        assert sensors.endswith("packs/pack1/sensors")
+
+    def test_get_content_from_pack_directory_doesnt_exist(self):
+        loader = ContentPackLoader()
+        pack_path = os.path.join(RESOURCES_DIR, "packs/pack100")
+
+        message_regex = "Directory .*? doesn't exist"
+        with pytest.raises(ValueError, match=message_regex):
+            loader.get_content_from_pack(pack_dir=pack_path, content_type="sensors")
+
+    def test_get_content_from_pack_no_sensors(self):
+        loader = ContentPackLoader()
+        pack_path = os.path.join(RESOURCES_DIR, "packs/pack2")
+
+        result = loader.get_content_from_pack(
+            pack_dir=pack_path, content_type="sensors"
+        )
+        assert result == None

tests/content/test_content_utils.py

@@ -0,0 +1,231 @@
+# # st2common
+# # from __future__ import absolute_import
+# import os
+# import os.path
+
+# # import unittest2
+# # from oslo_config import cfg
+
+# from st2common.constants.action import LIBS_DIR as ACTION_LIBS_DIR
+# from st2common.content.utils import get_pack_base_path
+# from st2common.content.utils import get_packs_base_paths
+# from st2common.content.utils import get_aliases_base_paths
+# from st2common.content.utils import get_pack_resource_file_abs_path
+# from st2common.content.utils import get_pack_file_abs_path
+# from st2common.content.utils import get_entry_point_abs_path
+# from st2common.content.utils import get_action_libs_abs_path
+# from st2common.content.utils import get_relative_path_to_pack_file
+# from st2tests import config as tests_config
+# from st2tests.fixturesloader import get_fixtures_packs_base_path
+
+# # TODO: We need to create fixures for everything inside of st2tests.config
+
+
+# class ContentUtilsTestCase(unittest2.TestCase):
+#     @classmethod
+#     def setUpClass(cls):
+#         tests_config.parse_args()
+
+#     def test_get_pack_base_paths(self):
+#         cfg.CONF.content.system_packs_base_path = ''
+#         cfg.CONF.content.packs_base_paths = '/opt/path1'
+#         result = get_packs_base_paths()
+#         self.assertEqual(result, ['/opt/path1'])
+
+#         # Multiple paths, no trailing colon
+#         cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2'
+#         result = get_packs_base_paths()
+#         self.assertEqual(result, ['/opt/path1', '/opt/path2'])
+
+#         # Multiple paths, trailing colon
+#         cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:'
+#         result = get_packs_base_paths()
+#         self.assertEqual(result, ['/opt/path1', '/opt/path2'])
+
+#         # Multiple same paths
+#         cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:/opt/path1:/opt/path2'
+#         result = get_packs_base_paths()
+#         self.assertEqual(result, ['/opt/path1', '/opt/path2'])
+
+#         # Assert system path is always first
+#         cfg.CONF.content.system_packs_base_path = '/opt/system'
+#         cfg.CONF.content.packs_base_paths = '/opt/path2:/opt/path1'
+#         result = get_packs_base_paths()
+#         self.assertEqual(result, ['/opt/system', '/opt/path2', '/opt/path1'])
+
+#         # More scenarios
+#         orig_path = cfg.CONF.content.system_packs_base_path
+#         cfg.CONF.content.system_packs_base_path = '/tests/packs'
+
+#         names = [
+#             'test_pack_1',
+#             'test_pack_2',
+#             'ma_pack'
+#         ]
+
+#         for name in names:
+#             actual = get_pack_base_path(pack_name=name)
+#             expected = os.path.join(cfg.CONF.content.system_packs_base_path,
+#                                     name)
+#             self.assertEqual(actual, expected)
+
+#         cfg.CONF.content.system_packs_base_path = orig_path
+
+#     def test_get_aliases_base_paths(self):
+#         cfg.CONF.content.aliases_base_paths = '/opt/path1'
+#         result = get_aliases_base_paths()
+#         self.assertEqual(result, ['/opt/path1'])
+
+#         # Multiple paths, no trailing colon
+#         cfg.CONF.content.aliases_base_paths = '/opt/path1:/opt/path2'
+#         result = get_aliases_base_paths()
+#         self.assertEqual(result, ['/opt/path1', '/opt/path2'])
+
+#         # Multiple paths, trailing colon
+#         cfg.CONF.content.aliases_base_paths = '/opt/path1:/opt/path2:'
+#         result = get_aliases_base_paths()
+#         self.assertEqual(result, ['/opt/path1', '/opt/path2'])
+
+#         # Multiple same paths
+#         cfg.CONF.content.aliases_base_paths = '/opt/path1:/opt/path2:/opt/path1:/opt/path2'
+#         result = get_aliases_base_paths()
+#         self.assertEqual(result, ['/opt/path1', '/opt/path2'])
+
+#     def test_get_pack_resource_file_abs_path(self):
+#         # Mock the packs path to point to the fixtures directory
+#         cfg.CONF.content.packs_base_paths = get_fixtures_packs_base_path()
+
+#         # Invalid resource type
+#         expected_msg = 'Invalid resource type: fooo'
+#         self.assertRaisesRegexp(ValueError, expected_msg, get_pack_resource_file_abs_path,
+#                                 pack_ref='dummy_pack_1',
+#                                 resource_type='fooo',
+#                                 file_path='test.py')
+
+#         # Invalid paths (directory traversal and absolute paths)
+#         file_paths = ['/tmp/foo.py', '../foo.py', '/etc/passwd', '../../foo.py',
+#                      '/opt/stackstorm/packs/invalid_pack/actions/my_action.py',
+#                      '../../foo.py']
+#         for file_path in file_paths:
+#             # action resource_type
+#             expected_msg = (r'Invalid file path: ".*%s"\. File path needs to be relative to the '
+#                             r'pack actions directory (.*). For example "my_action.py"\.' %
+#                             (file_path))
+#             self.assertRaisesRegexp(ValueError, expected_msg, get_pack_resource_file_abs_path,
+#                                     pack_ref='dummy_pack_1',
+#                                     resource_type='action',
+#                                     file_path=file_path)
+
+#             # sensor resource_type
+#             expected_msg = (r'Invalid file path: ".*%s"\. File path needs to be relative to the '
+#                             r'pack sensors directory (.*). For example "my_sensor.py"\.' %
+#                             (file_path))
+#             self.assertRaisesRegexp(ValueError, expected_msg, get_pack_resource_file_abs_path,
+#                                     pack_ref='dummy_pack_1',
+#                                     resource_type='sensor',
+#                                     file_path=file_path)
+
+#             # no resource type
+#             expected_msg = (r'Invalid file path: ".*%s"\. File path needs to be relative to the '
+#                             r'pack directory (.*). For example "my_action.py"\.' %
+#                             (file_path))
+#             self.assertRaisesRegexp(ValueError, expected_msg, get_pack_file_abs_path,
+#                                     pack_ref='dummy_pack_1',
+#                                     file_path=file_path)
+
+#         # Valid paths
+#         file_paths = ['foo.py', 'a/foo.py', 'a/b/foo.py']
+#         for file_path in file_paths:
+#             expected = os.path.join(get_fixtures_packs_base_path(),
+#                                     'dummy_pack_1/actions', file_path)
+#             result = get_pack_resource_file_abs_path(pack_ref='dummy_pack_1',
+#                                                      resource_type='action',
+#                                                      file_path=file_path)
+#             self.assertEqual(result, expected)
+
+#     def test_get_entry_point_absolute_path(self):
+#         orig_path = cfg.CONF.content.system_packs_base_path
+#         cfg.CONF.content.system_packs_base_path = '/tests/packs'
+#         acutal_path = get_entry_point_abs_path(
+#             pack='foo',
+#             entry_point='/tests/packs/foo/bar.py')
+#         self.assertEqual(acutal_path, '/tests/packs/foo/bar.py', 'Entry point path doesn\'t match.')
+#         cfg.CONF.content.system_packs_base_path = orig_path
+
+#     def test_get_entry_point_absolute_path_empty(self):
+#         orig_path = cfg.CONF.content.system_packs_base_path
+#         cfg.CONF.content.system_packs_base_path = '/tests/packs'
+#         acutal_path = get_entry_point_abs_path(pack='foo', entry_point=None)
+#         self.assertEqual(acutal_path, None, 'Entry point path doesn\'t match.')
+#         acutal_path = get_entry_point_abs_path(pack='foo', entry_point='')
+#         self.assertEqual(acutal_path, None, 'Entry point path doesn\'t match.')
+#         cfg.CONF.content.system_packs_base_path = orig_path
+
+#     def test_get_entry_point_relative_path(self):
+#         orig_path = cfg.CONF.content.system_packs_base_path
+#         cfg.CONF.content.system_packs_base_path = '/tests/packs'
+#         acutal_path = get_entry_point_abs_path(pack='foo', entry_point='foo/bar.py')
+#         expected_path = os.path.join(cfg.CONF.content.system_packs_base_path, 'foo', 'actions',
+#                                      'foo/bar.py')
+#         self.assertEqual(acutal_path, expected_path, 'Entry point path doesn\'t match.')
+#         cfg.CONF.content.system_packs_base_path = orig_path
+
+#     def test_get_action_libs_abs_path(self):
+#         orig_path = cfg.CONF.content.system_packs_base_path
+#         cfg.CONF.content.system_packs_base_path = '/tests/packs'
+
+#         # entry point relative.
+#         acutal_path = get_action_libs_abs_path(pack='foo', entry_point='foo/bar.py')
+#         expected_path = os.path.join(cfg.CONF.content.system_packs_base_path, 'foo', 'actions',
+#                                      os.path.join('foo', ACTION_LIBS_DIR))
+#         self.assertEqual(acutal_path, expected_path, 'Action libs path doesn\'t match.')
+
+#         # entry point absolute.
+#         acutal_path = get_action_libs_abs_path(
+#             pack='foo',
+#             entry_point='/tests/packs/foo/tmp/foo.py')
+#         expected_path = os.path.join('/tests/packs/foo/tmp', ACTION_LIBS_DIR)
+#         self.assertEqual(acutal_path, expected_path, 'Action libs path doesn\'t match.')
+#         cfg.CONF.content.system_packs_base_path = orig_path
+
+#     def test_get_relative_path_to_pack_file(self):
+#         packs_base_paths = get_fixtures_packs_base_path()
+
+#         pack_ref = 'dummy_pack_1'
+
+#         # 1. Valid paths
+#         file_path = os.path.join(packs_base_paths, 'dummy_pack_1/pack.yaml')
+#         result = get_relative_path_to_pack_file(pack_ref=pack_ref, file_path=file_path)
+#         self.assertEqual(result, 'pack.yaml')
+
+#         file_path = os.path.join(packs_base_paths, 'dummy_pack_1/actions/action.meta.yaml')
+#         result = get_relative_path_to_pack_file(pack_ref=pack_ref, file_path=file_path)
+#         self.assertEqual(result, 'actions/action.meta.yaml')
+
+#         file_path = os.path.join(packs_base_paths, 'dummy_pack_1/actions/lib/foo.py')
+#         result = get_relative_path_to_pack_file(pack_ref=pack_ref, file_path=file_path)
+#         self.assertEqual(result, 'actions/lib/foo.py')
+
+#         # Already relative
+#         file_path = 'actions/lib/foo2.py'
+#         result = get_relative_path_to_pack_file(pack_ref=pack_ref, file_path=file_path)
+#         self.assertEqual(result, 'actions/lib/foo2.py')
+
+#         # 2. Invalid path - outside pack directory
+#         expected_msg = r'file_path (.*?) is not located inside the pack directory (.*?)'
+
+#         file_path = os.path.join(packs_base_paths, 'dummy_pack_2/actions/lib/foo.py')
+#         self.assertRaisesRegexp(ValueError, expected_msg, get_relative_path_to_pack_file,
+#                                 pack_ref=pack_ref, file_path=file_path)
+
+#         file_path = '/tmp/foo/bar.py'
+#         self.assertRaisesRegexp(ValueError, expected_msg, get_relative_path_to_pack_file,
+#                                 pack_ref=pack_ref, file_path=file_path)
+
+#         file_path = os.path.join(packs_base_paths, '../dummy_pack_1/pack.yaml')
+#         self.assertRaisesRegexp(ValueError, expected_msg, get_relative_path_to_pack_file,
+#                                 pack_ref=pack_ref, file_path=file_path)
+
+#         file_path = os.path.join(packs_base_paths, '../../dummy_pack_1/pack.yaml')
+#         self.assertRaisesRegexp(ValueError, expected_msg, get_relative_path_to_pack_file,
+#                                 pack_ref=pack_ref, file_path=file_path)

tests/utils/test_casts.py

@@ -1,9 +1,10 @@
-# st2
+# st2common
 import json
 
-from ultron8.utils.casts import get_cast
 import pytest
 
+from ultron8.utils.casts import get_cast
+
 
 class CastsTestCase:
     def test_cast_string(self):