StackStorm Packs - Support for Git Submodules

CHANGELOG.rst

@@ -28,6 +28,8 @@ Added
 * Expose environment variable ST2_ACTION_DEBUG to all StackStorm actions.
   Contributed by @maxfactor1
 
+* Added option to checkout git submodules when downloading/installing packs #5814
+  Contributed by @jk464
 
 3.8.0 - November 18, 2022
 -------------------------

contrib/packs/actions/download.yaml

@@ -28,3 +28,8 @@
       items:
         type: "string"
       required: false
+    checkout_submodules:
+      type: "boolean"
+      description: "Set to True to checkout git submodules present in the pack"
+      required: false
+      default: false

contrib/packs/actions/install.meta.yaml

@@ -37,4 +37,9 @@
       required: false
       description: Action timeout in seconds. Action will get killed if it doesn't finish in timeout
       type: integer
+    checkout_submodules:
+      type: "boolean"
+      description: "Set to True to checkout git submodules present in the pack"
+      required: false
+      default: false
 

contrib/packs/actions/pack_mgmt/download.py

@@ -71,7 +71,13 @@ def __init__(self, config=None, action_service=None):
             os.environ["no_proxy"] = self.no_proxy
 
     def run(
-        self, packs, abs_repo_base, verifyssl=True, force=False, dependency_list=None
+        self,
+        packs,
+        abs_repo_base,
+        verifyssl=True,
+        force=False,
+        dependency_list=None,
+        checkout_submodules=False,
     ):
         result = {}
         pack_url = None
@@ -86,6 +92,7 @@ def run(
                     proxy_config=self.proxy_config,
                     force_permissions=True,
                     logger=self.logger,
+                    checkout_submodules=checkout_submodules,
                 )
                 pack_url, pack_ref, pack_result = pack_result
                 result[pack_ref] = pack_result
@@ -99,6 +106,7 @@ def run(
                     proxy_config=self.proxy_config,
                     force_permissions=True,
                     logger=self.logger,
+                    checkout_submodules=checkout_submodules,
                 )
                 pack_url, pack_ref, pack_result = pack_result
                 result[pack_ref] = pack_result

contrib/packs/actions/workflows/install.yaml

@@ -9,6 +9,7 @@ input:
   - force
   - skip_dependencies
   - timeout
+  - checkout_submodules
 
 vars:
   - packs_list: null
@@ -30,6 +31,7 @@ tasks:
       packs: <% ctx().packs %>
       force: <% ctx().force %>
       dependency_list: <% ctx().dependency_list %>
+      checkout_submodules: <% ctx().checkout_submodules %>
     next:
       - when: <% succeeded() %>
         do: make_a_prerun

st2api/st2api/controllers/v1/packs.py

@@ -103,6 +103,9 @@ def post(self, pack_install_request, requester_user=None):
         if pack_install_request.skip_dependencies:
             parameters["skip_dependencies"] = True
 
+        if pack_install_request.checkout_submodules:
+            parameters["checkout_submodules"] = True
+
         if not requester_user:
             requester_user = UserDB(name=cfg.CONF.system_user.user)
 

st2api/tests/unit/controllers/v1/test_packs.py

@@ -210,6 +210,18 @@ def test_install_with_skip_dependencies_parameter(self, _handle_schedule_executi
         self.assertEqual(resp.status_int, 202)
         self.assertEqual(resp.json, {"execution_id": "123"})
 
+    @mock.patch.object(ActionExecutionsControllerMixin, "_handle_schedule_execution")
+    def test_install_with_checkout_submodules_parameter(
+        self, _handle_schedule_execution
+    ):
+        _handle_schedule_execution.return_value = Response(json={"id": "123"})
+        payload = {"packs": ["some"], "checkout_submodules": True}
+
+        resp = self.app.post_json("/v1/packs/install", payload)
+
+        self.assertEqual(resp.status_int, 202)
+        self.assertEqual(resp.json, {"execution_id": "123"})
+
     @mock.patch.object(ActionExecutionsControllerMixin, "_handle_schedule_execution")
     def test_uninstall(self, _handle_schedule_execution):
         _handle_schedule_execution.return_value = Response(json={"id": "123"})

st2client/st2client/commands/pack.py

@@ -266,6 +266,12 @@ def __init__(self, resource, *args, **kwargs):
             default=False,
             help="Skip pack dependency installation.",
         )
+        self.parser.add_argument(
+            "--checkout-submodules",
+            action="store_true",
+            default=False,
+            help="Checkout git submodules present in the pack.",
+        )
 
     def run(self, args, **kwargs):
         is_structured_output = args.json or args.yaml
@@ -279,6 +285,7 @@ def run(self, args, **kwargs):
             args.packs,
             force=args.force,
             skip_dependencies=args.skip_dependencies,
+            checkout_submodules=args.checkout_submodules,
             **kwargs,
         )
 

st2client/st2client/models/core.py

@@ -628,12 +628,20 @@ class AsyncRequest(Resource):
 
 class PackResourceManager(ResourceManager):
     @add_auth_token_to_kwargs_from_env
-    def install(self, packs, force=False, skip_dependencies=False, **kwargs):
+    def install(
+        self,
+        packs,
+        force=False,
+        skip_dependencies=False,
+        checkout_submodules=False,
+        **kwargs,
+    ):
         url = "/%s/install" % (self.resource.get_url_path_name())
         payload = {
             "packs": packs,
             "force": force,
             "skip_dependencies": skip_dependencies,
+            "checkout_submodules": checkout_submodules,
         }
         response = self.client.post(url, payload, **kwargs)
         if response.status_code != http_client.OK:

st2common/st2common/models/api/pack.py

@@ -394,6 +394,11 @@ class PackInstallRequestAPI(BaseAPI):
                 "description": "Set to True to skip pack dependency installations.",
                 "default": False,
             },
+            "checkout_submodules": {
+                "type": "boolean",
+                "description": "Checkout git submodules present in the pack.",
+                "default": False,
+            },
         },
     }
 

st2common/st2common/openapi.yaml

@@ -5173,6 +5173,10 @@ definitions:
         type: boolean
         description: Set to True to skip pack dependency installations.
         default: false
+      checkout_submodules:
+        type: boolean
+        description: Set to True to checkout git submodules present in the pack.
+        default: false
     required:
       - packs
   PacksUninstall:

st2common/st2common/openapi.yaml.j2

@@ -5169,6 +5169,10 @@ definitions:
         type: boolean
         description: Set to True to skip pack dependency installations.
         default: false
+      checkout_submodules:
+        type: boolean
+        description: Set to True to checkout git submodules present in the pack.
+        default: false
     required:
       - packs
   PacksUninstall:

st2common/st2common/util/pack_management.py

@@ -79,6 +79,7 @@ def download_pack(
     force_owner_group=True,
     force_permissions=True,
     logger=LOG,
+    checkout_submodules=False,
 ):
     """
     Download the pack and move it to /opt/stackstorm/packs.
@@ -99,6 +100,10 @@ def download_pack(
     :param force: Force the installation and ignore / delete the lock file if it already exists.
     :type force: ``bool``
 
+    :param checkout_submodules: Whether to also checkout git submodules present
+                                in the pack
+    :type checkout_submodules: ``bool``
+
     :return: (pack_url, pack_ref, result)
     :rtype: tuple
     """
@@ -163,6 +168,7 @@ def download_pack(
                     repo_url=pack_url,
                     verify_ssl=verify_ssl,
                     ref=pack_version,
+                    checkout_submodules=checkout_submodules,
                 )
 
             pack_metadata = get_pack_metadata(pack_dir=abs_local_path)
@@ -190,7 +196,9 @@ def download_pack(
     return tuple(result)
 
 
-def clone_repo(temp_dir, repo_url, verify_ssl=True, ref="master"):
+def clone_repo(
+    temp_dir, repo_url, verify_ssl=True, ref="master", checkout_submodules=False
+):
     # Switch to non-interactive mode
     os.environ["GIT_TERMINAL_PROMPT"] = "0"
     os.environ["GIT_ASKPASS"] = "/bin/echo"
@@ -204,6 +212,10 @@ def clone_repo(temp_dir, repo_url, verify_ssl=True, ref="master"):
     # future.
     repo = Repo.clone_from(repo_url, temp_dir)
 
+    # Checkout any Git Submodules if requested.
+    if checkout_submodules:
+        repo.submodule_update(recursive=False)
+
     is_local_repo = repo_url.startswith("file://")
 
     try: