From af813cd45c901b6b8c8865671da2e993846c358d Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Thu, 21 Dec 2023 06:47:24 +0000 Subject: [PATCH 1/4] Add HelmRelease images to cluster manifest --- flux_local/manifest.py | 14 ++- flux_local/tool/get.py | 32 +++--- flux_local/tool/visitor.py | 39 ++++++- .../tool/__snapshots__/test_get_cluster.ambr | 108 +++++++++++++++++- tests/tool/test_get_cluster.py | 6 +- 5 files changed, 175 insertions(+), 24 deletions(-) diff --git a/flux_local/manifest.py b/flux_local/manifest.py index a26c5998..ce1595e5 100644 --- a/flux_local/manifest.py +++ b/flux_local/manifest.py @@ -57,7 +57,7 @@ class BaseManifest(BaseModel): _COMPACT_EXCLUDE_FIELDS: dict[str, Any] = {} - def compact_dict(self, exclude: dict[str, Any] | None = None, include: dict[str, Any] | None = None) -> dict[str, Any]: + def compact_dict(self, exclude: dict[str, Any] | None = None) -> dict[str, Any]: """Return a compact dictionary representation of the object. This is similar to `dict()` but with a specific implementation for serializing @@ -143,6 +143,9 @@ class HelmRelease(BaseManifest): values: Optional[dict[str, Any]] = None """The values to install in the chart.""" + images: list[str] = Field(default_factory=list) + """The list of images referenced in the HelmRelease.""" + @classmethod def parse_doc(cls, doc: dict[str, Any]) -> "HelmRelease": """Parse a HelmRelease from a kubernetes resource object.""" @@ -171,6 +174,11 @@ def repo_name(self) -> str: """Identifier for the HelmRepository identified in the HelmChart.""" return f"{self.chart.repo_namespace}-{self.chart.repo_name}" + @property + def namespaced_name(self) -> str: + """Return the namespace and name concatenated as an id.""" + return f"{self.namespace}/{self.name}" + _COMPACT_EXCLUDE_FIELDS = { "values": True, "chart": HelmChart._COMPACT_EXCLUDE_FIELDS, @@ -329,9 +337,9 @@ def id_name(self) -> str: return f"{self.path}" @property - def namespaced_name(self, sep: str = "/") -> str: + def namespaced_name(self) -> str: """Return the namespace and name concatenated as an id.""" - return f"{self.namespace}{sep}{self.name}" + return f"{self.namespace}/{self.name}" _COMPACT_EXCLUDE_FIELDS = { "helm_releases": { diff --git a/flux_local/tool/get.py b/flux_local/tool/get.py index 3d2024d5..7a88d6d0 100644 --- a/flux_local/tool/get.py +++ b/flux_local/tool/get.py @@ -4,11 +4,15 @@ from argparse import ArgumentParser, BooleanOptionalAction, _SubParsersAction as SubParsersAction from typing import cast, Any import sys +import pathlib +import tempfile +import yaml -from flux_local import git_repo, image +from flux_local import git_repo, image, helm from .format import PrintFormatter, YamlFormatter from . import selector +from .visitor import HelmVisitor, ObjectOutput, ImageOutput _LOGGER = logging.getLogger(__name__) @@ -178,6 +182,7 @@ async def run( # type: ignore[no-untyped-def] query.helm_release.enabled = output == "yaml" image_visitor: image.ImageVisitor | None = None + helm_content: ImageOutput | None = None if enable_images: if output != "yaml": print( @@ -188,24 +193,25 @@ async def run( # type: ignore[no-untyped-def] image_visitor = image.ImageVisitor() query.doc_visitor = image_visitor.repo_visitor() + helm_content = ImageOutput() + helm_visitor = HelmVisitor() + query.helm_repo.visitor = helm_visitor.repo_visitor() + query.helm_release.visitor = helm_visitor.release_visitor() + manifest = await git_repo.build_manifest( selector=query, options=selector.options(**kwargs) ) if output == "yaml": - include: dict[str, Any] | None = None if image_visitor: image_visitor.update_manifest(manifest) - include = { - "clusters": { - "__all__": { - "kustomizations": { - "__all__": True, - #"images": True, - } - } - } - } - YamlFormatter().print([manifest.compact_dict(include=include)]) + if helm_content: + with tempfile.TemporaryDirectory() as helm_cache_dir: + await helm_visitor.inflate( + pathlib.Path(helm_cache_dir), helm_content.visitor(), helm.Options(), + ) + helm_content.update_manifest(manifest) + + YamlFormatter().print([manifest.compact_dict()]) return cols = ["path", "kustomizations"] diff --git a/flux_local/tool/visitor.py b/flux_local/tool/visitor.py index 9dc01f4b..b16fc0cc 100644 --- a/flux_local/tool/visitor.py +++ b/flux_local/tool/visitor.py @@ -9,7 +9,7 @@ import yaml from typing import Any -from flux_local import git_repo +from flux_local import git_repo, image from flux_local.helm import Helm, Options from flux_local.kustomize import Kustomize from flux_local.manifest import ( @@ -17,6 +17,7 @@ Kustomization, HelmRepository, ClusterPolicy, + Manifest, ) @@ -152,6 +153,42 @@ def strip_attrs(metadata: dict[str, Any], strip_attributes: list[str]) -> None: del metadata[attr_key] break +class ImageOutput(ResourceOutput): + """Resource visitor that builds outputs for objects within the kustomization.""" + + def __init__(self) -> None: + """Initialize ObjectOutput.""" + # Map of kustomizations to the map of built objects as lines of the yaml string + self.content: dict[ResourceKey, dict[ResourceKey, list[str]]] = {} + self.image_visitor = image.ImageVisitor() + self.repo_visitor = self.image_visitor.repo_visitor() + + async def call_async( + self, + cluster_path: pathlib.Path, + kustomization_path: pathlib.Path, + doc: ResourceType, + cmd: Kustomize | None, + ) -> None: + """Visitor function invoked to build and record resource objects.""" + if cmd: + objects = await cmd.objects() + name = doc.namespaced_name + for obj in objects: + if obj.get("kind") in self.repo_visitor.kinds: + #_LOGGER.debug("Looking for image %s", doc) + self.repo_visitor.func(doc.namespaced_name, obj) + + + def update_manifest(self, manifest: Manifest) -> None: + """Update the manifest with the images found in the repo.""" + #_LOGGER.debug(self.image_visitor.images) + for cluster in manifest.clusters: + for kustomization in cluster.kustomizations: + for helm_release in kustomization.helm_releases: + if images := self.image_visitor.images.get(f"{helm_release.namespace}/{helm_release.name}"): + helm_release.images = list(images) + class ObjectOutput(ResourceOutput): """Resource visitor that builds outputs for objects within the kustomization.""" diff --git a/tests/tool/__snapshots__/test_get_cluster.ambr b/tests/tool/__snapshots__/test_get_cluster.ambr index fc0a4bac..01bc5852 100644 --- a/tests/tool/__snapshots__/test_get_cluster.ambr +++ b/tests/tool/__snapshots__/test_get_cluster.ambr @@ -62,28 +62,79 @@ ''' # --- -# name: test_get_cluster[yaml-cluster8-images] +# name: test_get_cluster[yaml-cluster-images] ''' --- clusters: - - path: tests/testdata/cluster8 + - path: tests/testdata/cluster kustomizations: - name: apps namespace: flux-system - path: tests/testdata/cluster8/apps + path: tests/testdata/cluster/apps/prod helm_repos: [] - helm_releases: [] + helm_releases: + - name: podinfo + namespace: podinfo + chart: + name: podinfo + repo_name: podinfo + repo_namespace: flux-system + images: + - ghcr.io/stefanprodan/podinfo:6.3.2 + - public.ecr.aws/docker/library/redis:7.0.6 cluster_policies: [] - name: flux-system namespace: flux-system - path: tests/testdata/cluster8/cluster + path: tests/testdata/cluster/clusters/prod helm_repos: [] helm_releases: [] cluster_policies: [] + - name: infra-configs + namespace: flux-system + path: tests/testdata/cluster/infrastructure/configs + helm_repos: + - name: bitnami + namespace: flux-system + url: https://charts.bitnami.com/bitnami + repo_type: default + - name: podinfo + namespace: flux-system + url: oci://ghcr.io/stefanprodan/charts + repo_type: oci + - name: weave-charts + namespace: flux-system + url: oci://ghcr.io/weaveworks/charts + repo_type: oci + helm_releases: [] + cluster_policies: + - name: test-allow-policy + - name: infra-controllers + namespace: flux-system + path: tests/testdata/cluster/infrastructure/controllers + helm_repos: [] + helm_releases: + - name: metallb + namespace: metallb + chart: + name: metallb + repo_name: bitnami + repo_namespace: flux-system + images: + - docker.io/bitnami/metallb-speaker:0.13.7-debian-11-r28 + - docker.io/bitnami/metallb-controller:0.13.7-debian-11-r29 + - name: weave-gitops + namespace: flux-system + chart: + name: weave-gitops + repo_name: weave-charts + repo_namespace: flux-system + images: + - ghcr.io/weaveworks/wego-app:v0.24.0 + cluster_policies: [] ''' # --- -# name: test_get_cluster[yaml] +# name: test_get_cluster[yaml-cluster-no-images] ''' --- clusters: @@ -147,3 +198,48 @@ ''' # --- +# name: test_get_cluster[yaml-cluster8-images] + ''' + --- + clusters: + - path: tests/testdata/cluster8 + kustomizations: + - name: apps + namespace: flux-system + path: tests/testdata/cluster8/apps + helm_repos: [] + helm_releases: [] + cluster_policies: [] + images: + - busybox + - alpine + - name: flux-system + namespace: flux-system + path: tests/testdata/cluster8/cluster + helm_repos: [] + helm_releases: [] + cluster_policies: [] + + ''' +# --- +# name: test_get_cluster[yaml-cluster8-no-images] + ''' + --- + clusters: + - path: tests/testdata/cluster8 + kustomizations: + - name: apps + namespace: flux-system + path: tests/testdata/cluster8/apps + helm_repos: [] + helm_releases: [] + cluster_policies: [] + - name: flux-system + namespace: flux-system + path: tests/testdata/cluster8/cluster + helm_repos: [] + helm_releases: [] + cluster_policies: [] + + ''' +# --- diff --git a/tests/tool/test_get_cluster.py b/tests/tool/test_get_cluster.py index 5ebeb9f0..a4d18dc4 100644 --- a/tests/tool/test_get_cluster.py +++ b/tests/tool/test_get_cluster.py @@ -27,7 +27,9 @@ (["--path", "tests/testdata/cluster7"]), (["--all-namespaces", "--path", "tests/testdata/cluster/"]), (["--path", "tests/testdata/cluster", "-o", "yaml"]), + (["--path", "tests/testdata/cluster", "-o", "yaml", "--enable-images"]), (["--path", "tests/testdata/cluster8", "-o", "yaml"]), + (["--path", "tests/testdata/cluster8", "-o", "yaml", "--enable-images"]), ], ids=[ "cluster", @@ -39,7 +41,9 @@ "cluster6", "cluster7", "all-namespaces", - "yaml", + "yaml-cluster-no-images", + "yaml-cluster-images", + "yaml-cluster8-no-images", "yaml-cluster8-images" ], ) From b084bcd62ea613617a19d71abeffe5e0c6b164d0 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Thu, 21 Dec 2023 06:51:20 +0000 Subject: [PATCH 2/4] Fix lint errors --- flux_local/git_repo.py | 2 +- flux_local/tool/get.py | 13 +++++++++---- flux_local/tool/visitor.py | 13 +++++++------ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/flux_local/git_repo.py b/flux_local/git_repo.py index e56857a1..3ef5902b 100644 --- a/flux_local/git_repo.py +++ b/flux_local/git_repo.py @@ -568,7 +568,7 @@ async def build_kustomization( docs = await cmd.grep(regexp).objects( target_namespace=kustomization.target_namespace ) - + if selector.doc_visitor: doc_kinds = set(selector.doc_visitor.kinds) for doc in docs: diff --git a/flux_local/tool/get.py b/flux_local/tool/get.py index 7a88d6d0..cab92628 100644 --- a/flux_local/tool/get.py +++ b/flux_local/tool/get.py @@ -1,18 +1,21 @@ """Flux-local get action.""" import logging -from argparse import ArgumentParser, BooleanOptionalAction, _SubParsersAction as SubParsersAction +from argparse import ( + ArgumentParser, + BooleanOptionalAction, + _SubParsersAction as SubParsersAction, +) from typing import cast, Any import sys import pathlib import tempfile -import yaml from flux_local import git_repo, image, helm from .format import PrintFormatter, YamlFormatter from . import selector -from .visitor import HelmVisitor, ObjectOutput, ImageOutput +from .visitor import HelmVisitor, ImageOutput _LOGGER = logging.getLogger(__name__) @@ -207,7 +210,9 @@ async def run( # type: ignore[no-untyped-def] if helm_content: with tempfile.TemporaryDirectory() as helm_cache_dir: await helm_visitor.inflate( - pathlib.Path(helm_cache_dir), helm_content.visitor(), helm.Options(), + pathlib.Path(helm_cache_dir), + helm_content.visitor(), + helm.Options(), ) helm_content.update_manifest(manifest) diff --git a/flux_local/tool/visitor.py b/flux_local/tool/visitor.py index b16fc0cc..bfccc471 100644 --- a/flux_local/tool/visitor.py +++ b/flux_local/tool/visitor.py @@ -153,6 +153,7 @@ def strip_attrs(metadata: dict[str, Any], strip_attributes: list[str]) -> None: del metadata[attr_key] break + class ImageOutput(ResourceOutput): """Resource visitor that builds outputs for objects within the kustomization.""" @@ -171,22 +172,22 @@ async def call_async( cmd: Kustomize | None, ) -> None: """Visitor function invoked to build and record resource objects.""" - if cmd: + if cmd and isinstance(doc, HelmRelease): objects = await cmd.objects() - name = doc.namespaced_name for obj in objects: if obj.get("kind") in self.repo_visitor.kinds: - #_LOGGER.debug("Looking for image %s", doc) + # _LOGGER.debug("Looking for image %s", doc) self.repo_visitor.func(doc.namespaced_name, obj) - def update_manifest(self, manifest: Manifest) -> None: """Update the manifest with the images found in the repo.""" - #_LOGGER.debug(self.image_visitor.images) + # _LOGGER.debug(self.image_visitor.images) for cluster in manifest.clusters: for kustomization in cluster.kustomizations: for helm_release in kustomization.helm_releases: - if images := self.image_visitor.images.get(f"{helm_release.namespace}/{helm_release.name}"): + if images := self.image_visitor.images.get( + f"{helm_release.namespace}/{helm_release.name}" + ): helm_release.images = list(images) From e13bc6aa21932225caea5d3b8fee4b365e35bd72 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Thu, 21 Dec 2023 15:18:41 +0000 Subject: [PATCH 3/4] Consistent sorting on image names --- flux_local/image.py | 1 + flux_local/tool/visitor.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flux_local/image.py b/flux_local/image.py index 0b5f9fc6..65d647f6 100644 --- a/flux_local/image.py +++ b/flux_local/image.py @@ -72,3 +72,4 @@ def update_manifest(self, manifest: manifest.Manifest) -> None: for kustomization in cluster.kustomizations: if images := self.images.get(kustomization.namespaced_name): kustomization.images = list(images) + kustomization.images.sort() diff --git a/flux_local/tool/visitor.py b/flux_local/tool/visitor.py index bfccc471..03214ced 100644 --- a/flux_local/tool/visitor.py +++ b/flux_local/tool/visitor.py @@ -176,19 +176,18 @@ async def call_async( objects = await cmd.objects() for obj in objects: if obj.get("kind") in self.repo_visitor.kinds: - # _LOGGER.debug("Looking for image %s", doc) self.repo_visitor.func(doc.namespaced_name, obj) def update_manifest(self, manifest: Manifest) -> None: """Update the manifest with the images found in the repo.""" - # _LOGGER.debug(self.image_visitor.images) for cluster in manifest.clusters: for kustomization in cluster.kustomizations: for helm_release in kustomization.helm_releases: if images := self.image_visitor.images.get( - f"{helm_release.namespace}/{helm_release.name}" + helm_release.namespaced_name ): helm_release.images = list(images) + helm_release.images.sort() class ObjectOutput(ResourceOutput): From f02a9eb3f519c0addab6d7434b1fb980c3f624a4 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Thu, 21 Dec 2023 15:21:15 +0000 Subject: [PATCH 4/4] Update snapshots with sort order --- tests/tool/__snapshots__/test_get_cluster.ambr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tool/__snapshots__/test_get_cluster.ambr b/tests/tool/__snapshots__/test_get_cluster.ambr index 01bc5852..38d77fa1 100644 --- a/tests/tool/__snapshots__/test_get_cluster.ambr +++ b/tests/tool/__snapshots__/test_get_cluster.ambr @@ -120,8 +120,8 @@ repo_name: bitnami repo_namespace: flux-system images: - - docker.io/bitnami/metallb-speaker:0.13.7-debian-11-r28 - docker.io/bitnami/metallb-controller:0.13.7-debian-11-r29 + - docker.io/bitnami/metallb-speaker:0.13.7-debian-11-r28 - name: weave-gitops namespace: flux-system chart: @@ -211,8 +211,8 @@ helm_releases: [] cluster_policies: [] images: - - busybox - alpine + - busybox - name: flux-system namespace: flux-system path: tests/testdata/cluster8/cluster