Add HelmRelease images to cluster manifest (#445)

Follow on from #443 by also adding HelmRelease images to the manifest.
This also contains cleanup of other unused code from the initial PR
adding container images.

Issue #434

Example usage
```
$ flux-local get cluster --path tests/testdata/cluster/ -o yaml --enable-images
---
clusters:
- path: tests/testdata/cluster
  kustomizations:
  - name: apps
    namespace: flux-system
    path: tests/testdata/cluster/apps/prod
    helm_repos: []
    helm_releases:
    - name: podinfo
      namespace: podinfo
      chart:
        name: podinfo
        repo_name: podinfo
        repo_namespace: flux-system
      images:
      - public.ecr.aws/docker/library/redis:7.0.6
      - ghcr.io/stefanprodan/podinfo:6.3.2
    cluster_policies: []
  - name: flux-system
    namespace: flux-system
    path: tests/testdata/cluster/clusters/prod
    helm_repos: []
    helm_releases: []
    cluster_policies: []
  - name: infra-configs
    namespace: flux-system
    path: tests/testdata/cluster/infrastructure/configs
    helm_repos:
    - name: bitnami
      namespace: flux-system
      url: https://charts.bitnami.com/bitnami
      repo_type: default
    - name: podinfo
      namespace: flux-system
      url: oci://ghcr.io/stefanprodan/charts
      repo_type: oci
    - name: weave-charts
      namespace: flux-system
      url: oci://ghcr.io/weaveworks/charts
      repo_type: oci
    helm_releases: []
    cluster_policies:
    - name: test-allow-policy
  - name: infra-controllers
    namespace: flux-system
    path: tests/testdata/cluster/infrastructure/controllers
    helm_repos: []
    helm_releases:
    - name: metallb
      namespace: metallb
      chart:
        name: metallb
        repo_name: bitnami
        repo_namespace: flux-system
      images:
      - docker.io/bitnami/metallb-controller:0.13.7-debian-11-r29
      - docker.io/bitnami/metallb-speaker:0.13.7-debian-11-r28
    - name: weave-gitops
      namespace: flux-system
      chart:
        name: weave-gitops
        repo_name: weave-charts
        repo_namespace: flux-system
      images:
      - ghcr.io/weaveworks/wego-app:v0.24.0
    cluster_policies: []
```

flux_local/git_repo.py

@@ -568,7 +568,7 @@ async def build_kustomization(
         docs = await cmd.grep(regexp).objects(
             target_namespace=kustomization.target_namespace
         )
-        
+
         if selector.doc_visitor:
             doc_kinds = set(selector.doc_visitor.kinds)
             for doc in docs:

flux_local/image.py

@@ -72,3 +72,4 @@ def update_manifest(self, manifest: manifest.Manifest) -> None:
             for kustomization in cluster.kustomizations:
                 if images := self.images.get(kustomization.namespaced_name):
                     kustomization.images = list(images)
+                    kustomization.images.sort()

flux_local/manifest.py

@@ -57,7 +57,7 @@ class BaseManifest(BaseModel):
 
     _COMPACT_EXCLUDE_FIELDS: dict[str, Any] = {}
 
-    def compact_dict(self, exclude: dict[str, Any] | None = None, include: dict[str, Any] | None = None) -> dict[str, Any]:
+    def compact_dict(self, exclude: dict[str, Any] | None = None) -> dict[str, Any]:
         """Return a compact dictionary representation of the object.
 
         This is similar to `dict()` but with a specific implementation for serializing
@@ -143,6 +143,9 @@ class HelmRelease(BaseManifest):
     values: Optional[dict[str, Any]] = None
     """The values to install in the chart."""
 
+    images: list[str] = Field(default_factory=list)
+    """The list of images referenced in the HelmRelease."""
+
     @classmethod
     def parse_doc(cls, doc: dict[str, Any]) -> "HelmRelease":
         """Parse a HelmRelease from a kubernetes resource object."""
@@ -171,6 +174,11 @@ def repo_name(self) -> str:
         """Identifier for the HelmRepository identified in the HelmChart."""
         return f"{self.chart.repo_namespace}-{self.chart.repo_name}"
 
+    @property
+    def namespaced_name(self) -> str:
+        """Return the namespace and name concatenated as an id."""
+        return f"{self.namespace}/{self.name}"
+
     _COMPACT_EXCLUDE_FIELDS = {
         "values": True,
         "chart": HelmChart._COMPACT_EXCLUDE_FIELDS,
@@ -329,9 +337,9 @@ def id_name(self) -> str:
         return f"{self.path}"
 
     @property
-    def namespaced_name(self, sep: str = "/") -> str:
+    def namespaced_name(self) -> str:
         """Return the namespace and name concatenated as an id."""
-        return f"{self.namespace}{sep}{self.name}"
+        return f"{self.namespace}/{self.name}"
 
     _COMPACT_EXCLUDE_FIELDS = {
         "helm_releases": {

flux_local/tool/get.py

@@ -1,14 +1,21 @@
 """Flux-local get action."""
 
 import logging
-from argparse import ArgumentParser, BooleanOptionalAction, _SubParsersAction as SubParsersAction
+from argparse import (
+    ArgumentParser,
+    BooleanOptionalAction,
+    _SubParsersAction as SubParsersAction,
+)
 from typing import cast, Any
 import sys
+import pathlib
+import tempfile
 
-from flux_local import git_repo, image
+from flux_local import git_repo, image, helm
 
 from .format import PrintFormatter, YamlFormatter
 from . import selector
+from .visitor import HelmVisitor, ImageOutput
 
 
 _LOGGER = logging.getLogger(__name__)
@@ -178,6 +185,7 @@ async def run(  # type: ignore[no-untyped-def]
         query.helm_release.enabled = output == "yaml"
 
         image_visitor: image.ImageVisitor | None = None
+        helm_content: ImageOutput | None = None
         if enable_images:
             if output != "yaml":
                 print(
@@ -188,24 +196,27 @@ async def run(  # type: ignore[no-untyped-def]
             image_visitor = image.ImageVisitor()
             query.doc_visitor = image_visitor.repo_visitor()
 
+            helm_content = ImageOutput()
+            helm_visitor = HelmVisitor()
+            query.helm_repo.visitor = helm_visitor.repo_visitor()
+            query.helm_release.visitor = helm_visitor.release_visitor()
+
         manifest = await git_repo.build_manifest(
             selector=query, options=selector.options(**kwargs)
         )
         if output == "yaml":
-            include: dict[str, Any] | None = None
             if image_visitor:
                 image_visitor.update_manifest(manifest)
-                include = {
-                    "clusters": {
-                        "__all__": {
-                            "kustomizations": {
-                                "__all__": True,
-                                #"images": True,
-                            }
-                        }
-                    }
-                }
-            YamlFormatter().print([manifest.compact_dict(include=include)])
+            if helm_content:
+                with tempfile.TemporaryDirectory() as helm_cache_dir:
+                    await helm_visitor.inflate(
+                        pathlib.Path(helm_cache_dir),
+                        helm_content.visitor(),
+                        helm.Options(),
+                    )
+                    helm_content.update_manifest(manifest)
+
+            YamlFormatter().print([manifest.compact_dict()])
             return
 
         cols = ["path", "kustomizations"]

flux_local/tool/visitor.py

@@ -9,14 +9,15 @@
 import yaml
 from typing import Any
 
-from flux_local import git_repo
+from flux_local import git_repo, image
 from flux_local.helm import Helm, Options
 from flux_local.kustomize import Kustomize
 from flux_local.manifest import (
     HelmRelease,
     Kustomization,
     HelmRepository,
     ClusterPolicy,
+    Manifest,
 )
 
 
@@ -153,6 +154,42 @@ def strip_attrs(metadata: dict[str, Any], strip_attributes: list[str]) -> None:
                 break
 
 
+class ImageOutput(ResourceOutput):
+    """Resource visitor that builds outputs for objects within the kustomization."""
+
+    def __init__(self) -> None:
+        """Initialize ObjectOutput."""
+        # Map of kustomizations to the map of built objects as lines of the yaml string
+        self.content: dict[ResourceKey, dict[ResourceKey, list[str]]] = {}
+        self.image_visitor = image.ImageVisitor()
+        self.repo_visitor = self.image_visitor.repo_visitor()
+
+    async def call_async(
+        self,
+        cluster_path: pathlib.Path,
+        kustomization_path: pathlib.Path,
+        doc: ResourceType,
+        cmd: Kustomize | None,
+    ) -> None:
+        """Visitor function invoked to build and record resource objects."""
+        if cmd and isinstance(doc, HelmRelease):
+            objects = await cmd.objects()
+            for obj in objects:
+                if obj.get("kind") in self.repo_visitor.kinds:
+                    self.repo_visitor.func(doc.namespaced_name, obj)
+
+    def update_manifest(self, manifest: Manifest) -> None:
+        """Update the manifest with the images found in the repo."""
+        for cluster in manifest.clusters:
+            for kustomization in cluster.kustomizations:
+                for helm_release in kustomization.helm_releases:
+                    if images := self.image_visitor.images.get(
+                        helm_release.namespaced_name
+                    ):
+                        helm_release.images = list(images)
+                        helm_release.images.sort()
+
+
 class ObjectOutput(ResourceOutput):
     """Resource visitor that builds outputs for objects within the kustomization."""
 

tests/tool/__snapshots__/test_get_cluster.ambr

@@ -62,28 +62,79 @@
 
   '''
 # ---
-# name: test_get_cluster[yaml-cluster8-images]
+# name: test_get_cluster[yaml-cluster-images]
   '''
   ---
   clusters:
-  - path: tests/testdata/cluster8
+  - path: tests/testdata/cluster
     kustomizations:
     - name: apps
       namespace: flux-system
-      path: tests/testdata/cluster8/apps
+      path: tests/testdata/cluster/apps/prod
       helm_repos: []
-      helm_releases: []
+      helm_releases:
+      - name: podinfo
+        namespace: podinfo
+        chart:
+          name: podinfo
+          repo_name: podinfo
+          repo_namespace: flux-system
+        images:
+        - ghcr.io/stefanprodan/podinfo:6.3.2
+        - public.ecr.aws/docker/library/redis:7.0.6
       cluster_policies: []
     - name: flux-system
       namespace: flux-system
-      path: tests/testdata/cluster8/cluster
+      path: tests/testdata/cluster/clusters/prod
       helm_repos: []
       helm_releases: []
       cluster_policies: []
+    - name: infra-configs
+      namespace: flux-system
+      path: tests/testdata/cluster/infrastructure/configs
+      helm_repos:
+      - name: bitnami
+        namespace: flux-system
+        url: https://charts.bitnami.com/bitnami
+        repo_type: default
+      - name: podinfo
+        namespace: flux-system
+        url: oci://ghcr.io/stefanprodan/charts
+        repo_type: oci
+      - name: weave-charts
+        namespace: flux-system
+        url: oci://ghcr.io/weaveworks/charts
+        repo_type: oci
+      helm_releases: []
+      cluster_policies:
+      - name: test-allow-policy
+    - name: infra-controllers
+      namespace: flux-system
+      path: tests/testdata/cluster/infrastructure/controllers
+      helm_repos: []
+      helm_releases:
+      - name: metallb
+        namespace: metallb
+        chart:
+          name: metallb
+          repo_name: bitnami
+          repo_namespace: flux-system
+        images:
+        - docker.io/bitnami/metallb-controller:0.13.7-debian-11-r29
+        - docker.io/bitnami/metallb-speaker:0.13.7-debian-11-r28
+      - name: weave-gitops
+        namespace: flux-system
+        chart:
+          name: weave-gitops
+          repo_name: weave-charts
+          repo_namespace: flux-system
+        images:
+        - ghcr.io/weaveworks/wego-app:v0.24.0
+      cluster_policies: []
 
   '''
 # ---
-# name: test_get_cluster[yaml]
+# name: test_get_cluster[yaml-cluster-no-images]
   '''
   ---
   clusters:
@@ -147,3 +198,48 @@
 
   '''
 # ---
+# name: test_get_cluster[yaml-cluster8-images]
+  '''
+  ---
+  clusters:
+  - path: tests/testdata/cluster8
+    kustomizations:
+    - name: apps
+      namespace: flux-system
+      path: tests/testdata/cluster8/apps
+      helm_repos: []
+      helm_releases: []
+      cluster_policies: []
+      images:
+      - alpine
+      - busybox
+    - name: flux-system
+      namespace: flux-system
+      path: tests/testdata/cluster8/cluster
+      helm_repos: []
+      helm_releases: []
+      cluster_policies: []
+
+  '''
+# ---
+# name: test_get_cluster[yaml-cluster8-no-images]
+  '''
+  ---
+  clusters:
+  - path: tests/testdata/cluster8
+    kustomizations:
+    - name: apps
+      namespace: flux-system
+      path: tests/testdata/cluster8/apps
+      helm_repos: []
+      helm_releases: []
+      cluster_policies: []
+    - name: flux-system
+      namespace: flux-system
+      path: tests/testdata/cluster8/cluster
+      helm_repos: []
+      helm_releases: []
+      cluster_policies: []
+
+  '''
+# ---

tests/tool/test_get_cluster.py

@@ -27,7 +27,9 @@
         (["--path", "tests/testdata/cluster7"]),
         (["--all-namespaces", "--path", "tests/testdata/cluster/"]),
         (["--path", "tests/testdata/cluster", "-o", "yaml"]),
+        (["--path", "tests/testdata/cluster", "-o", "yaml", "--enable-images"]),
         (["--path", "tests/testdata/cluster8", "-o", "yaml"]),
+        (["--path", "tests/testdata/cluster8", "-o", "yaml", "--enable-images"]),
     ],
     ids=[
         "cluster",
@@ -39,7 +41,9 @@
         "cluster6",
         "cluster7",
         "all-namespaces",
-        "yaml",
+        "yaml-cluster-no-images",
+        "yaml-cluster-images",
+        "yaml-cluster8-no-images",
         "yaml-cluster8-images"
     ],
 )