Merge pull request #1175 from kapicorp/move-to-pytest

Incremental changes from OmegaConf work #1173

.github/workflows/test-build-publish.yml

@@ -50,33 +50,36 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ['3.10', '3.11']
-
     steps:
       - name: Checkout kapitan recursively
         uses: actions/checkout@v4
         with:
           submodules: recursive
-
+      - name: Install poetry
+        run: pipx install poetry
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          cache: 'pip'
+          cache: 'poetry'
           python-version: ${{ matrix.python-version }}
-
-      - name: Install testing dependencies
+      - name: Install libraries dependencies
+        run: |
+          poetry install --no-root 
+      - name: Install testing dependencies (Helm)
         run: |
-          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 762E3157
           sudo apt-get -qq update
           sudo apt-get install -y gnupg2 git curl
-          curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-          chmod 700 get_helm.sh
-          sudo ./get_helm.sh
-          pip3 install --editable ".[test]"
-          pip3 install coverage black
-      - name: Run tests
-        run: |-
-          # includes make test
-          make test_coverage 
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+      - name: Run pytest
+        uses: pavelzw/pytest-action@v2
+        with:
+          verbose: true
+          emoji: false
+          job-summary: true
+          custom-pytest: poetry run pytest
+          custom-arguments: '-q'
+          click-to-expand: true
+          report-title: 'Kapitan tests'
 
   build:
     name: build ${{ matrix.platform }} image

Dockerfile

@@ -10,7 +10,8 @@ RUN apt-get update \
     && apt-get install --no-install-recommends -y \
         curl \
         build-essential \
-        git
+        git \
+        default-jre
 
 ENV POETRY_VERSION=1.7.1
 ENV VIRTUAL_ENV=/opt/venv

kapitan/cached.py

@@ -6,6 +6,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 "cached module"
+from argparse import Namespace
 
 inv = {}
 inv_cache = {}
@@ -16,7 +17,7 @@
 dot_kapitan = {}
 ref_controller_obj = None
 revealer_obj = None
-args = {}  # args won't need resetting
+args = args = Namespace() # args won't need resetting
 inv_sources = set()
 
 

kapitan/cli.py

@@ -92,9 +92,9 @@ def trigger_compile(args):
         validate=args.validate,
         schemas_path=args.schemas_path,
         jinja2_filters=args.jinja2_filters,
-        verbose=hasattr(args, "verbose") and args.verbose,
+        verbose=args.verbose,
         use_go_jsonnet=args.use_go_jsonnet,
-        compose_target_name=args.compose_target_name,
+        compose_target_name=args.compose_target_name
     )
 
 
@@ -111,6 +111,12 @@ def build_parser():
         choices=AVAILABLE_BACKENDS.keys(),
         help="Select the inventory backend to use (default=reclass)",
     )
+    inventory_backend_parser.add_argument(
+        "--migrate",
+        action="store_true",
+        default=from_dot_kapitan("inventory_backend", "migrate", False),
+        help="Migrate your inventory to your selected inventory backend.",
+    )
 
     inventory_backend_parser.add_argument(
         "--compose-target-name", "--compose-target-name",
@@ -595,6 +601,7 @@ def build_parser():
         "validate",
         aliases=["v"],
         help="validates the compile output against schemas as specified in inventory",
+        parents=[inventory_backend_parser]
     )
     validate_parser.set_defaults(func=schema_validate_compiled, name="validate")
 
@@ -651,26 +658,19 @@ def main():
 
     logger.debug("Running with args: %s", args)
 
-    try:
-        cmd = sys.argv[1]
-    except IndexError:
+    if len(sys.argv) < 2:
         parser.print_help()
         sys.exit(1)
 
-    # cache args where key is subcommand
-    assert "name" in args, "All cli commands must have provided default name"
-    cached.args[args.name] = args
-    if "inventory_backend" in args:
-        cached.args["inventory-backend"] = args.inventory_backend
-        cached.args.setdefault("global", {}).setdefault("inventory-backend", args.inventory_backend)
+    cached.args = args  
 
-    if "compose_target_name" in args:
-        cached.args.setdefault("global", {}).setdefault("compose_target_name", args.compose_target_name)
-
     if hasattr(args, "verbose") and args.verbose:
-        setup_logging(level=logging.DEBUG, force=True)
+        logging_level = logging.DEBUG
     elif hasattr(args, "quiet") and args.quiet:
-        setup_logging(level=logging.CRITICAL, force=True)
+        logging_level = logging.CRITICAL
+    else:
+        logging_level = logging.INFO
+    setup_logging(level=logging_level, force=True)
 
     # call chosen command
     args.func(args)

kapitan/inputs/base.py

@@ -89,9 +89,9 @@ def make_compile_dirs(self, target_name, output_path, **kwargs):
         """make compile dirs, skips if dirs exist"""
         _compile_path = os.path.join(self.compile_path, target_name, output_path)
         if kwargs.get("compose_target_name", False):
-            os.makedirs(_compile_path.replace(".", "/"), exist_ok=True)
-        else:
-            os.makedirs(_compile_path, exist_ok=True)
+            _compile_path = _compile_path.replace(".", "/")
+
+        os.makedirs(_compile_path, exist_ok=True)
 
     def compile_file(self, file_path, compile_path, ext_vars, **kwargs):
         """implements compilation for file_path to compile_path with ext_vars"""

kapitan/inputs/jinja2_filters.py

@@ -79,7 +79,7 @@ def load_jinja2_filters_from_file(env, jinja2_filters):
 # Custom filters
 def reveal_maybe(ref_tag):
     "Will reveal ref_tag if valid and --reveal flag is used"
-    if cached.args["compile"].reveal:
+    if cached.args.reveal:
         return cached.revealer_obj.reveal_raw(ref_tag)
     else:
         return ref_tag

kapitan/inputs/kadet.py

@@ -25,9 +25,8 @@
 kadet.ABORT_EXCEPTION_TYPE = CompileError
 
 logger = logging.getLogger(__name__)
-inventory_path = cached.args.get(
-    "inventory_path"
-)  # XXX think about this as it probably breaks usage as library
+inventory_path = vars(cached.args).get("inventory_path")
+# XXX think about this as it probably breaks usage as library
 search_paths = contextvars.ContextVar("current search_paths in thread")
 current_target = contextvars.ContextVar("current target in thread")
 

kapitan/inventory/inv_reclass.py

@@ -10,13 +10,14 @@
 
 from kapitan.errors import InventoryError
 
-from .inventory import Inventory
+from .inventory import Inventory, InventoryTarget
 
 logger = logging.getLogger(__name__)
 
 
 class ReclassInventory(Inventory):
-    def render_targets(self, targets: list = None, ignore_class_notfound: bool = False):
+
+    def render_targets(self, targets: list[InventoryTarget] = None, ignore_class_notfound: bool = False) -> None:
         """
         Runs a reclass inventory in inventory_path
         (same output as running ./reclass.py -b inv_base_uri/ --inventory)

kapitan/inventory/inventory.py

@@ -22,7 +22,6 @@
 class InventoryTarget:
     name: str
     path: str
-    composed_name: str
     parameters: dict = field(default_factory=dict)
     classes: list = field(default_factory=list)
     applications: list = field(default_factory=list)
@@ -48,47 +47,43 @@ def inventory(self) -> dict:
         get all targets from inventory
         targets will be rendered
         """
-        if not self.targets:
-            self.search_targets()
-
-        inventory = self.get_targets([*self.targets.keys()])
 
         return {
-            target_name: {"parameters": target.parameters, "classes": target.classes}
-            for target_name, target in inventory.items()
+            target.name: {"parameters": target.parameters, "classes": target.classes}
+            for target in self.get_targets().values()
         }
 
     def search_targets(self) -> dict:
         """
         look for targets at '<inventory_path>/targets/' and return targets without rendering parameters
         """
+
         for root, dirs, files in os.walk(self.targets_path):
             for file in files:
                 # split file extension and check if yml/yaml
-                path = os.path.join(root, file)
-                name, ext = os.path.splitext(file)
+                path = os.path.relpath(os.path.join(root, file), self.targets_path)
+
+                if self.compose_target_name:
+                    name, ext = os.path.splitext(path)
+                    name = name.replace(os.sep, ".")
+                else:
+                    name, ext = os.path.splitext(file)
+
                 if ext not in (".yml", ".yaml"):
-                    logger.debug(f"{file}: targets have to be .yml or .yaml files.")
+                    logger.debug(f"ignoring {file}: targets have to be .yml or .yaml files.")
                     continue
 
-                # initialize target
-                composed_name = (
-                    os.path.splitext(os.path.relpath(path, self.targets_path))[0]
-                    .replace(os.sep, ".")
-                    .lstrip(".")
-                )
-                target = InventoryTarget(name, path, composed_name)
-                if self.compose_target_name:
-                    target.name = target.composed_name
+                target = InventoryTarget(name, path)
+
+
 
-                # check for same name
                 if self.targets.get(target.name):
                     raise InventoryError(
-                        f"Conflicting targets {target.name}: {target.path} and {self.targets[target.name].path}"
+                        f"Conflicting targets {target.name}: {target.path} and {self.targets[target.name].path}. "
+                        f"Consider using '--compose-target-name'."
                     )
-
+                
                 self.targets[target.name] = target
-
         return self.targets
 
     def get_target(self, target_name: str, ignore_class_not_found: bool = False) -> InventoryTarget:
@@ -97,28 +92,35 @@ def get_target(self, target_name: str, ignore_class_not_found: bool = False) ->
         """
         return self.get_targets([target_name], ignore_class_not_found)[target_name]
 
-    def get_targets(self, target_names: list, ignore_class_not_found: bool = False) -> dict:
+    def get_targets(self, target_names: list[str] = [], ignore_class_not_found: bool = False) -> dict:
         """
         helper function to get rendered InventoryTarget objects for multiple targets
         """
+        if not self.targets:
+            self.search_targets()
+
         targets_to_render = []
-
-        for target_name in target_names:
-            target = self.targets.get(target_name)
-            if not target:
-                if ignore_class_not_found:
-                    continue
-                raise InventoryError(f"target '{target_name}' not found")
-
+        targets = {}
+
+        if not target_names:
+            targets = self.targets
+        else:
+            try:
+                targets = { target_name : self.targets[target_name] for target_name in target_names }
+            except KeyError as e:
+                if not ignore_class_not_found:
+                    raise InventoryError(f"targets not found: {set(target_names)-set(self.targets)}" )
+
+        for target in targets.values():
             if not target.parameters:
                 targets_to_render.append(target)
 
         if targets_to_render:
             self.render_targets(targets_to_render, ignore_class_not_found)
 
-        return {name: target for name, target in self.targets.items() if name in target_names}
+        return self.targets
 
-    def get_parameters(self, target_names: Union[str, list], ignore_class_not_found: bool = False) -> dict:
+    def get_parameters(self, target_names: str | list[str], ignore_class_not_found: bool = False) -> dict:
         """
         helper function to get rendered parameters for single target or multiple targets
         """
@@ -129,12 +131,18 @@ def get_parameters(self, target_names: Union[str, list], ignore_class_not_found:
         return {name: target.parameters for name, target in self.get_targets(target_names)}
 
     @abstractmethod
-    def render_targets(self, targets: list = None, ignore_class_notfound: bool = False):
+    def render_targets(self, targets: list[InventoryTarget] = None, ignore_class_notfound: bool = False) -> None:
         """
         create the inventory depending on which backend gets used
         """
         raise NotImplementedError
 
+    def migrate(self):
+        """
+        migrate the inventory, e.g. change interpolation syntax to new syntax
+        """
+        pass
+
     def __getitem__(self, key):
         return self.inventory[key]
 

kapitan/resources.py

@@ -253,10 +253,7 @@ def inventory(search_paths: list, target_name: str = None, inventory_path: str =
     set inventory_path to read custom path. None defaults to value set via cli
     Returns a dictionary with the inventory for target
     """
-    if inventory_path is None:
-        # grab inventory_path value from cli subcommand
-        inventory_path_arg = cached.args.get("compile") or cached.args.get("inventory")
-        inventory_path = inventory_path_arg.inventory_path
+    inventory_path = inventory_path or cached.args.inventory_path
 
     inv_path_exists = False
 
@@ -317,19 +314,28 @@ def get_inventory(inventory_path) -> Inventory:
     if cached.inv and cached.inv.targets:
         return cached.inv
 
+    compose_target_name = hasattr(cached.args, "compose_target_name") and cached.args.compose_target_name
+    if hasattr(cached.args, "compose_node_name") and cached.args.compose_node_name:
+        logger.warning(
+            "inventory flag '--compose-node-name' is deprecated and scheduled to be dropped with the next release. "
+            "Please use '--compose-target-name' instead."
+        )
+        compose_target_name = True
+
     # select inventory backend
-    backend_id = cached.args.get("inventory-backend")
-    compose_target_name = cached.args["global"].get("compose_target_name")
-    backend = AVAILABLE_BACKENDS.get(backend_id)
+    backend_id = hasattr(cached.args, "inventory_backend") and cached.args.inventory_backend
+    compose_target_name = hasattr(cached.args, "compose_target_name") and cached.args.compose_target_name
+    backend = AVAILABLE_BACKENDS.get(backend_id, AVAILABLE_BACKENDS.get("reclass"))
     inventory_backend: Inventory = None
-    if backend != None:
-        logger.debug(f"Using {backend_id} as inventory backend")
-        inventory_backend = backend(inventory_path, compose_target_name)
-    else:
-        logger.debug(f"Backend {backend_id} is unknown, falling back to reclass as inventory backend")
-        inventory_backend = ReclassInventory(inventory_path, compose_target_name)
+
+    logger.debug(f"Using {backend_id} as inventory backend")
+    inventory_backend = backend(inventory_path, compose_target_name)
+
+    cached.inv = inventory_backend
+    # migrate inventory to selected inventory backend
+    if hasattr(cached.args, "migrate") and cached.args.migrate:
+        inventory_backend.migrate()
 
     inventory_backend.search_targets()
 
-    cached.inv = inventory_backend
     return inventory_backend